Skip to content

Commit 34f11e7

Browse files
acudzelig
acud
authored and
zelig
committed
cmd/swarm/swarm-snapshot: swarm snapshot generator (#18453)
* cmd/swarm/swarm-snapshot: add binary to create network snapshots * cmd/swarm/swarm-snapshot: refactor and extend tests * p2p/simulations: remove unused triggerChecks func and fix linter * internal/cmdtest: raise the timeout for killing TestCmd * cmd/swarm/swarm-snapshot: add more comments and other minor adjustments * cmd/swarm/swarm-snapshot: remove redundant check in createSnapshot * cmd/swarm/swarm-snapshot: change comment wording * p2p/simulations: revert Simulation.Run from master https://github.com/ethersphere/go-ethereum/pull/1077/files#r247078904 * cmd/swarm/swarm-snapshot: address pr comments * swarm/network/simulations/discovery: removed snapshot write to file * cmd/swarm/swarm-snapshot, swarm/network/simulations: removed redundant connection event check, fixed lint error
1 parent f728837 commit 34f11e7

File tree

6 files changed

+437
-76
lines changed

6 files changed

+437
-76
lines changed

‎cmd/swarm/swarm-snapshot/create.go‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
// Copyright 2018 The go-ethereum Authors
2+
// This file is part of go-ethereum.
3+
//
4+
// go-ethereum is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// go-ethereum is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU General Public License
15+
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package main
18+
19+
import (
20+
"context"
21+
"encoding/json"
22+
"errors"
23+
"fmt"
24+
"io/ioutil"
25+
"os"
26+
"path"
27+
"path/filepath"
28+
"strings"
29+
"sync"
30+
"time"
31+
32+
"github.com/ethereum/go-ethereum/log"
33+
"github.com/ethereum/go-ethereum/node"
34+
"github.com/ethereum/go-ethereum/p2p/simulations"
35+
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
36+
"github.com/ethereum/go-ethereum/swarm/network"
37+
"github.com/ethereum/go-ethereum/swarm/network/simulation"
38+
cli "gopkg.in/urfave/cli.v1"
39+
)
40+
41+
// create is used as the entry function for "create" app command.
42+
func create(ctx *cli.Context) error {
43+
log.PrintOrigins(true)
44+
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
45+
46+
if len(ctx.Args()) < 1 {
47+
return errors.New("argument should be the filename to verify or write-to")
48+
}
49+
filename, err := touchPath(ctx.Args()[0])
50+
if err != nil {
51+
return err
52+
}
53+
return createSnapshot(filename, ctx.Int("nodes"), strings.Split(ctx.String("services"), ","))
54+
}
55+
56+
// createSnapshot creates a new snapshot on filesystem with provided filename,
57+
// number of nodes and service names.
58+
func createSnapshot(filename string, nodes int, services []string) (err error) {
59+
log.Debug("create snapshot", "filename", filename, "nodes", nodes, "services", services)
60+
61+
sim := simulation.New(map[string]simulation.ServiceFunc{
62+
"bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
63+
addr := network.NewAddr(ctx.Config.Node())
64+
kad := network.NewKademlia(addr.Over(), network.NewKadParams())
65+
hp := network.NewHiveParams()
66+
hp.KeepAliveInterval = time.Duration(200) * time.Millisecond
67+
hp.Discovery = true // discovery must be enabled when creating a snapshot
68+
69+
config := &network.BzzConfig{
70+
OverlayAddr: addr.Over(),
71+
UnderlayAddr: addr.Under(),
72+
HiveParams: hp,
73+
}
74+
return network.NewBzz(config, kad, nil, nil, nil), nil, nil
75+
},
76+
})
77+
defer sim.Close()
78+
79+
_, err = sim.AddNodes(nodes)
80+
if err != nil {
81+
return fmt.Errorf("add nodes: %v", err)
82+
}
83+
84+
err = sim.Net.ConnectNodesRing(nil)
85+
if err != nil {
86+
return fmt.Errorf("connect nodes: %v", err)
87+
}
88+
89+
ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute)
90+
defer cancelSimRun()
91+
if _, err := sim.WaitTillHealthy(ctx); err != nil {
92+
return fmt.Errorf("wait for healthy kademlia: %v", err)
93+
}
94+
95+
var snap *simulations.Snapshot
96+
if len(services) > 0 {
97+
// If service names are provided, include them in the snapshot.
98+
// But, check if "bzz" service is not among them to remove it
99+
// form the snapshot as it exists on snapshot creation.
100+
var removeServices []string
101+
var wantBzz bool
102+
for _, s := range services {
103+
if s == "bzz" {
104+
wantBzz = true
105+
break
106+
}
107+
}
108+
if !wantBzz {
109+
removeServices = []string{"bzz"}
110+
}
111+
snap, err = sim.Net.SnapshotWithServices(services, removeServices)
112+
} else {
113+
snap, err = sim.Net.Snapshot()
114+
}
115+
if err != nil {
116+
return fmt.Errorf("create snapshot: %v", err)
117+
}
118+
jsonsnapshot, err := json.Marshal(snap)
119+
if err != nil {
120+
return fmt.Errorf("json encode snapshot: %v", err)
121+
}
122+
return ioutil.WriteFile(filename, jsonsnapshot, 0666)
123+
}
124+
125+
// touchPath creates an empty file and all subdirectories
126+
// that are missing.
127+
func touchPath(filename string) (string, error) {
128+
if path.IsAbs(filename) {
129+
if _, err := os.Stat(filename); err == nil {
130+
// path exists, overwrite
131+
return filename, nil
132+
}
133+
}
134+
135+
d, f := path.Split(filename)
136+
dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
137+
if err != nil {
138+
return "", err
139+
}
140+
141+
_, err = os.Stat(path.Join(dir, filename))
142+
if err == nil {
143+
// path exists, overwrite
144+
return filename, nil
145+
}
146+
147+
dirPath := path.Join(dir, d)
148+
filePath := path.Join(dirPath, f)
149+
if d != "" {
150+
err = os.MkdirAll(dirPath, os.ModeDir)
151+
if err != nil {
152+
return "", err
153+
}
154+
}
155+
156+
return filePath, nil
157+
}

‎cmd/swarm/swarm-snapshot/create_test.go‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
// Copyright 2018 The go-ethereum Authors
2+
// This file is part of go-ethereum.
3+
//
4+
// go-ethereum is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// go-ethereum is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU General Public License
15+
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package main
18+
19+
import (
20+
"encoding/json"
21+
"fmt"
22+
"io/ioutil"
23+
"os"
24+
"sort"
25+
"strconv"
26+
"strings"
27+
"testing"
28+
29+
"github.com/ethereum/go-ethereum/p2p/simulations"
30+
)
31+
32+
// TestSnapshotCreate is a high level e2e test that tests for snapshot generation.
33+
// It runs a few "create" commands with different flag values and loads generated
34+
// snapshot files to validate their content.
35+
func TestSnapshotCreate(t *testing.T) {
36+
for _, v := range []struct {
37+
name string
38+
nodes int
39+
services string
40+
}{
41+
{
42+
name: "defaults",
43+
},
44+
{
45+
name: "more nodes",
46+
nodes: defaultNodes + 5,
47+
},
48+
{
49+
name: "services",
50+
services: "stream,pss,zorglub",
51+
},
52+
{
53+
name: "services with bzz",
54+
services: "bzz,pss",
55+
},
56+
} {
57+
t.Run(v.name, func(t *testing.T) {
58+
t.Parallel()
59+
60+
file, err := ioutil.TempFile("", "swarm-snapshot")
61+
if err != nil {
62+
t.Fatal(err)
63+
}
64+
defer os.Remove(file.Name())
65+
66+
if err = file.Close(); err != nil {
67+
t.Error(err)
68+
}
69+
70+
args := []string{"create"}
71+
if v.nodes > 0 {
72+
args = append(args, "--nodes", strconv.Itoa(v.nodes))
73+
}
74+
if v.services != "" {
75+
args = append(args, "--services", v.services)
76+
}
77+
testCmd := runSnapshot(t, append(args, file.Name())...)
78+
79+
testCmd.ExpectExit()
80+
if code := testCmd.ExitStatus(); code != 0 {
81+
t.Fatalf("command exit code %v, expected 0", code)
82+
}
83+
84+
f, err := os.Open(file.Name())
85+
if err != nil {
86+
t.Fatal(err)
87+
}
88+
defer func() {
89+
err := f.Close()
90+
if err != nil {
91+
t.Error("closing snapshot file", "err", err)
92+
}
93+
}()
94+
95+
b, err := ioutil.ReadAll(f)
96+
if err != nil {
97+
t.Fatal(err)
98+
}
99+
var snap simulations.Snapshot
100+
err = json.Unmarshal(b, &snap)
101+
if err != nil {
102+
t.Fatal(err)
103+
}
104+
105+
wantNodes := v.nodes
106+
if wantNodes == 0 {
107+
wantNodes = defaultNodes
108+
}
109+
gotNodes := len(snap.Nodes)
110+
if gotNodes != wantNodes {
111+
t.Errorf("got %v nodes, want %v", gotNodes, wantNodes)
112+
}
113+
114+
if len(snap.Conns) == 0 {
115+
t.Error("no connections in a snapshot")
116+
}
117+
118+
var wantServices []string
119+
if v.services != "" {
120+
wantServices = strings.Split(v.services, ",")
121+
} else {
122+
wantServices = []string{"bzz"}
123+
}
124+
// sort service names so they can be comparable
125+
// as strings to every node sorted services
126+
sort.Strings(wantServices)
127+
128+
for i, n := range snap.Nodes {
129+
gotServices := n.Node.Config.Services
130+
sort.Strings(gotServices)
131+
if fmt.Sprint(gotServices) != fmt.Sprint(wantServices) {
132+
t.Errorf("got services %v for node %v, want %v", gotServices, i, wantServices)
133+
}
134+
}
135+
136+
})
137+
}
138+
}

‎cmd/swarm/swarm-snapshot/main.go‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Copyright 2018 The go-ethereum Authors
2+
// This file is part of go-ethereum.
3+
//
4+
// go-ethereum is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// go-ethereum is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU General Public License
15+
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package main
18+
19+
import (
20+
"os"
21+
22+
"github.com/ethereum/go-ethereum/cmd/utils"
23+
"github.com/ethereum/go-ethereum/log"
24+
cli "gopkg.in/urfave/cli.v1"
25+
)
26+
27+
var gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
28+
29+
// default value for "create" command --nodes flag
30+
const defaultNodes = 10
31+
32+
func main() {
33+
err := newApp().Run(os.Args)
34+
if err != nil {
35+
log.Error(err.Error())
36+
os.Exit(1)
37+
}
38+
}
39+
40+
// newApp construct a new instance of Swarm Snapshot Utility.
41+
// Method Run is called on it in the main function and in tests.
42+
func newApp() (app *cli.App) {
43+
app = utils.NewApp(gitCommit, "Swarm Snapshot Utility")
44+
45+
app.Name = "swarm-snapshot"
46+
app.Usage = ""
47+
48+
// app flags (for all commands)
49+
app.Flags = []cli.Flag{
50+
cli.IntFlag{
51+
Name: "verbosity",
52+
Value: 1,
53+
Usage: "verbosity level",
54+
},
55+
}
56+
57+
app.Commands = []cli.Command{
58+
{
59+
Name: "create",
60+
Aliases: []string{"c"},
61+
Usage: "create a swarm snapshot",
62+
Action: create,
63+
// Flags only for "create" command.
64+
// Allow app flags to be specified after the
65+
// command argument.
66+
Flags: append(app.Flags,
67+
cli.IntFlag{
68+
Name: "nodes",
69+
Value: defaultNodes,
70+
Usage: "number of nodes",
71+
},
72+
cli.StringFlag{
73+
Name: "services",
74+
Value: "bzz",
75+
Usage: "comma separated list of services to boot the nodes with",
76+
},
77+
),
78+
},
79+
}
80+
81+
return app
82+
}

0 commit comments

Comments
 (0)