1.Windows compatibility. 2. Conflicts merged. 3.All discv4 tests

.gitignore

@@ -0,0 +1,2 @@
+.vscode/
+workspace

README.md

@@ -29,6 +29,69 @@ $ go get github.com/karalabe/hive
 of resource files to build the corrent docker images and containers. This requirement will be removed
 in the future.*
 
+# Running on Windows
+
+The following information assumes Docker for Windows (CE) is installed on Windows 10 Pro. 
+
+## Docker daemon
+`hive` uses the Docker API to connect to the Docker Daemon to dynamically create and run containers. At the time of writing, the daemon is disabled by default. This must be enabled. 
+
+To enable the daemon, right click on the Docker Whale in the system tray and press Settings. Under 'General' select "Expose daemon on tcp.... without TLS".
+
+To run `hive`, use the following command line option --docker-endpoint tcp://localhost:2375 Alternatively, if using VSCode simply run using the supplied launch.json (see below)
+
+## Shell container
+Currently, the Windows version must be run from the Host. To achieve this run with the --docker-noshell command line option. 
+
+## Debugging or executing from Visual Studio Code
+As described above, golang must be installed on the machine. The golang extension for VSCode is then required, along with Delve and the standard tools recommended by the Golang extension.
+
+When VS Code is configured for general go development, `hive` may be run simply by launching with F5 with the following `launch.json`. This `launch.json` includes example parameters that limit the client to `geth` as the full client suite may take significant time to build initial docker images.
+```json
+{
+
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch",
+            "type": "go",
+            "request": "launch",
+            "mode": "debug",
+            "remotePath": "",
+            "port": 2345,
+            "host": "127.0.0.1",
+            "program": "${workspaceFolder}",
+            "env": {},
+            "args": [
+
+                "--docker-endpoint","tcp://localhost:2375",
+                "--docker-noshell",
+                "--client","go-ethereum_master" , 
+                "--loglevel","6",
+                "--smoke"
+
+
+            ],
+            "showLog": true
+        }
+    ]
+}
+```
+
+## Access to the local drive
+Docker will need access to the `workspace` folder. This will either be requested automatically in an Windows notification, or permission can be set in the docker settings in advance.
+
+To set the permissions to access your drive, right click on the Docker Whale in the system tray and press Settings. Under 'Shared Drives' select the drive where the `workspace` folder is for sharing.
+
+## Host access to the docker network
+
+`hive` requires network access to the docker containers it creates. While this is automatically available on Linux, at the time of writing because of virtualisation there needs to be some further network configuration so that the `hive` host can connect. The following is dependent on your docker configuration, and there may be other ways to achieve the same result, but a typical setting may be:
+
+'route /P add 172.17.0.0 MASK 255.255.0.0 10.0.75.2'
+
+An administrator level command prompt must be opened and the target IPs of the containers routed to HyperV's IP address for the docker containers.
+
+
 # Validating clients
 
 You can run the full suite of `hive` validation tests against all the known implementations tagged

clients/aleth:nightly/Dockerfile → clients/aleth_nightly/Dockerfile

@@ -24,6 +24,6 @@ RUN apk add --no-cache jq bc bash
 RUN cat /usr/share/aleth/buildinfo.json |jq "del(.is_prerelease)"  > /version.json
 
 # Export the usual networking ports to allow outside access to the node
-EXPOSE 8545 30303
+EXPOSE 8545 30303 30303/udp
 
 ENTRYPOINT ["/aleth.sh"]

clients/devp2p/ethereumj_devp2p/Dockerfile

@@ -0,0 +1,4 @@
+# Dockerfile allowing devp2p tests on geth master
+FROM hive/clients/ethereumj_master
+
+ENV HIVE_BOOTNODE enode://158f8aab45f6d19c6cbf4a089c2670541a8da11978a2f90dbf6a502a4a3bab80d288afdbeb7ec0ef6d92de563767f3b1ea9e8e334ca711e9f8e2df5a0385e8e6@13.75.154.138:30303

clients/devp2p/go-ethereum_devp2p/Dockerfile

@@ -0,0 +1,8 @@
+# Dockerfile allowing devp2p tests on geth master
+FROM hive/clients/go-ethereum_master
+
+#Live bootnode
+#ENV HIVE_BOOTNODE enode://158f8aab45f6d19c6cbf4a089c2670541a8da11978a2f90dbf6a502a4a3bab80d288afdbeb7ec0ef6d92de563767f3b1ea9e8e334ca711e9f8e2df5a0385e8e6@13.75.154.138:30303
+
+#Dead bootnode
+ENV HIVE_BOOTNODE enode://158f8aab45f6d19c6cbf4a089c2670541a8da11978a2f90dbf6a502a4a3bab80d288afdbeb7ec0ef6d92de563767f3b1ea9e8e334ca711e9f8e2df5a0385e8e6@1.2.3.4:30000

clients/devp2p/parity_devp2p/Dockerfile

@@ -0,0 +1,4 @@
+# Dockerfile allowing devp2p tests on geth master
+FROM hive/clients/parity_master
+
+ENV HIVE_BOOTNODE enode://158f8aab45f6d19c6cbf4a089c2670541a8da11978a2f90dbf6a502a4a3bab80d288afdbeb7ec0ef6d92de563767f3b1ea9e8e334ca711e9f8e2df5a0385e8e6@13.75.154.138:30303

clients/ethereumj:develop/Dockerfile → clients/ethereumj_develop/Dockerfile

@@ -28,7 +28,7 @@ ADD harmony.sh /harmony.sh
 RUN chmod +x /harmony.sh
 
 # Export the usual networking ports to allow outside access to the node
-EXPOSE 8545 8546 30303
+EXPOSE 8545 8546 30303 30303/udp
 
 ENTRYPOINT ["/harmony.sh"]
 

clients/go-ethereum:local/Dockerfile → clients/go-ethereum_local/Dockerfile

@@ -15,7 +15,7 @@ ADD geth.sh /geth.sh
 RUN chmod +x /geth.sh
 
 # Export the usual networking ports to allow outside access to the node
-EXPOSE 8545 8546 30303
+EXPOSE 8545 8546 30303 30303/udp
 
 ADD version.json /version.json
 ENTRYPOINT ["/geth.sh"]

clients/go-ethereum:master/Dockerfile → clients/go-ethereum_master/Dockerfile

@@ -25,8 +25,12 @@ RUN \
 ADD geth.sh /geth.sh
 RUN chmod +x /geth.sh
 
+# Inject the enode id retriever script
+ADD enode.sh /enode.sh
+
+
 # Export the usual networking ports to allow outside access to the node
-EXPOSE 8545 8546 30303
+EXPOSE 8545 8546 30303 30303/udp
 
 # Generate the ethash verification caches
 RUN \

clients/go-ethereum_master/enode.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Script to retrieve the enode
+# 
+# This is copied into the validator container by Hive
+# and used to provide a client-specific enode id retriever
+#
+
+# Immediately abort the script on any error encountered
+set -e
+echo "Trying to get enode."
+
+TARGET_RESPONSE=$(curl -s -X POST  -H "Content-Type: application/json"  --data '{"jsonrpc":"2.0","method":"admin_nodeInfo","params":[],"id":1}' "$HIVE_CLIENT_IP:8545" )
+
+echo "Got admin enode info response: $TARGET_RESPONSE"
+TARGET_ENODE=$(echo ${TARGET_RESPONSE}| jq -r '.result.enode')
+echo "Target enode identified as $TARGET_ENODE"

clients/go-ethereum:stable/geth.sh → clients/go-ethereum_master/geth.sh

@@ -28,7 +28,7 @@
 # Immediately abort the script on any error encountered
 set -e
 
-# It doesn't make sense to dial out, use only a pre-set bootnode
+#It doesn't make sense to dial out, use only a pre-set bootnode
 if [ "$HIVE_BOOTNODE" != "" ]; then
 	FLAGS="$FLAGS --bootnodes $HIVE_BOOTNODE"
 else
@@ -123,4 +123,4 @@ fi
 
 # Run the go-ethereum implementation with the requested flags
 echo "Running go-ethereum..."
-/geth $FLAGS --nat=none --rpc --rpcaddr "0.0.0.0" --rpcapi "admin,debug,eth,miner,net,personal,shh,txpool,web3" --ws --wsaddr "0.0.0.0" --wsapi "admin,debug,eth,miner,net,personal,shh,txpool,web3" --wsorigins "*"
+/geth --verbosity 6 $FLAGS --nat=none --rpc --rpcaddr "0.0.0.0" --rpcapi "admin,debug,eth,miner,net,personal,shh,txpool,web3" --ws --wsaddr "0.0.0.0" --wsapi "admin,debug,eth,miner,net,personal,shh,txpool,web3" --wsorigins "*"

clients/go-ethereum:stable/Dockerfile → clients/go-ethereum_stable/Dockerfile

@@ -27,6 +27,6 @@ ADD geth.sh /geth.sh
 RUN chmod +x /geth.sh
 
 # Export the usual networking ports to allow outside access to the node
-EXPOSE 8545 8546 30303
+EXPOSE 8545 8546 30303 30303/udp
 
 ENTRYPOINT ["/geth.sh"]

clients/parity:beta/Dockerfile → clients/parity_beta/Dockerfile

@@ -40,6 +40,6 @@ RUN chmod +x /parity.sh
 ADD chain.json /chain.json
 
 # Export the usual networking ports to allow outside access to the node
-EXPOSE 8545 30303
+EXPOSE 8545 30303 30303/udp
 
 ENTRYPOINT ["/parity.sh"]

clients/parity:master/Dockerfile → clients/parity_master/Dockerfile

@@ -37,7 +37,11 @@ ADD parity.sh /parity.sh
 RUN chmod +x /parity.sh
 ADD chain.json /chain.json
 
+# Inject the enode id retriever script
+ADD enode.sh /enode.sh
+
+
 # Export the usual networking ports to allow outside access to the node
-EXPOSE 8545 30303
+EXPOSE 8545 30303 30303/udp
 
 ENTRYPOINT ["/parity.sh"]

clients/parity_master/enode.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Script to retrieve the enode
+# 
+# This is copied into the validator container by Hive
+# and used to provide a client-specific enode id retriever
+#
+
+# Immediately abort the script on any error encountered
+set -e
+
+set -e
+echo "Trying to get enode."
+
+TARGET_RESPONSE=$(curl --data '{"method":"parity_enode","params":[],"id":1,"jsonrpc":"2.0"}' -H "Content-Type: application/json" -X POST "$HIVE_CLIENT_IP:8545" )
+
+echo "Got admin enode info response: $TARGET_RESPONSE"
+TARGET_ENODE=$(echo ${TARGET_RESPONSE}| jq -r '.result.enode')
+
+echo "Target enode identified as $TARGET_ENODE"

hive.go

@@ -13,11 +13,12 @@ import (
 )
 
 var (
-	dockerEndpoint   = flag.String("docker-endpoint", "unix:///var/run/docker.sock", "Unix socket to the local Docker daemon")
+	dockerEndpoint   = flag.String("docker-endpoint", "unix:///var/run/docker.sock", "Endpoint to the local Docker daemon")
+	dockerHostAlias  = flag.String("docker-hostalias", "unix:///var/run/docker.sock", "Endpoint to the host Docket daemon from within a validator")
 	noShellContainer = flag.Bool("docker-noshell", false, "Disable outer docker shell, running directly on the host")
 	noCachePattern   = flag.String("docker-nocache", "", "Regexp selecting the docker images to forcibly rebuild")
 
-	clientPattern = flag.String("client", ":master", "Regexp selecting the client(s) to run against")
+	clientPattern = flag.String("client", "_master", "Regexp selecting the client(s) to run against")
 	overrideFiles = flag.String("override", "", "Comma separated regexp:files to override in client containers")
 	smokeFlag     = flag.Bool("smoke", false, "Whether to only smoke test or run full test suite")
 
@@ -101,15 +102,15 @@ func mainInHost(daemon *docker.Client, overrides []string, cacher *buildCacher)
 	}
 	// Smoke tests are exclusive with all other flags
 	if *smokeFlag {
-		if results.Validations, err = validateClients(daemon, *clientPattern, "smoke/", overrides, cacher); err != nil {
+		if results.Validations, err = validateClients(daemon, *clientPattern, "smoke", overrides, cacher); err != nil {
 			log15.Crit("failed to smoke-validate client images", "error", err)
 			return err
 		}
-		if results.Simulations, err = simulateClients(daemon, *clientPattern, "smoke/", overrides, cacher); err != nil {
+		if results.Simulations, err = simulateClients(daemon, *clientPattern, "smoke", overrides, cacher); err != nil {
 			log15.Crit("failed to smoke-simulate client images", "error", err)
 			return err
 		}
-		if results.Benchmarks, err = benchmarkClients(daemon, *clientPattern, "smoke/", overrides, cacher); err != nil {
+		if results.Benchmarks, err = benchmarkClients(daemon, *clientPattern, "smoke", overrides, cacher); err != nil {
 			log15.Crit("failed to smoke-benchmark client images", "error", err)
 			return err
 		}

images.go

@@ -131,7 +131,7 @@ func buildNestedImages(daemon *docker.Client, root string, pattern string, kind
 			if name := filepath.Dir(path); re.MatchString(name) {
 				names = append(names, filepath.Join(strings.Split(name, string(filepath.Separator))[1:]...))
 			}
-			return filepath.SkipDir
+			//return filepath.SkipDir
 		}
 		// Continue walking the path
 		return nil
@@ -143,7 +143,7 @@ func buildNestedImages(daemon *docker.Client, root string, pattern string, kind
 	for _, name := range names {
 		var (
 			context = filepath.Join(root, name)
-			image   = filepath.Join(hiveImageNamespace, context)
+			image   = strings.Replace(filepath.Join(hiveImageNamespace, context), string(os.PathSeparator), "/", -1)
 			logger  = log15.New(kind, name)
 		)
 		if err := buildImage(daemon, image, context, cacher, logger); err != nil {
@@ -177,6 +177,11 @@ func buildImage(daemon *docker.Client, image, context string, cacher *buildCache
 	}
 	logger.Info("building new docker image", "nocache", nocache)
 
+	context, err := filepath.Abs(context)
+	if err != nil {
+		logger.Error("failed to build docker image", "error", err)
+		return err
+	}
 	stream := io.Writer(new(bytes.Buffer))
 	if *loglevelFlag > 5 {
 		stream = os.Stderr

validator.go

@@ -120,18 +120,20 @@ func validate(daemon *docker.Client, client, validator string, overrides []strin
 		}
 		// Container seems to be alive, check whether the RPC is accepting connections
 		if conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", c.NetworkSettings.IPAddress, 8545)); err == nil {
+
 			clogger.Debug("client container online", "time", time.Since(start))
 			conn.Close()
 			break
 		}
+
 		time.Sleep(100 * time.Millisecond)
 	}
 	// Create the validator container and make sure it's cleaned up afterwards
 	logger.Debug("creating validator container")
 	vc, err := daemon.CreateContainer(docker.CreateContainerOptions{
 		Config: &docker.Config{
 			Image: validator,
-			Env:   []string{"HIVE_CLIENT_IP=" + cip},
+			Env:   []string{"HIVE_CLIENT_IP=" + cip, "HIVE_CLIENT_ID=" + cc.ID, "HIVE_DOCKER_HOST_ALIAS=" + *dockerHostAlias},
 		},
 	})
 	if err != nil {
@@ -148,6 +150,12 @@ func validate(daemon *docker.Client, client, validator string, overrides []strin
 		}
 	}()
 
+	//copy the enode identifier script from the client to the validator
+	err = copyBetweenContainers(daemon, vc.ID, cc.ID, "", "/enode.sh", true)
+	if err != nil {
+		vlogger.Warn("No enode.sh provided. Discovery tests will not be able to identify their target node id.", "warning", err)
+	}
+
 	// Start the tester container and wait until it finishes
 	vlogger.Debug("running validator container")
 	vwaiter, err := runContainer(daemon, vc.ID, vlogger, filepath.Join(logdir, "validator.log"), false)
@@ -156,15 +164,23 @@ func validate(daemon *docker.Client, client, validator string, overrides []strin
 		result.Error = err
 		return result
 	}
+	v, err := daemon.InspectContainer(vc.ID)
+	if err != nil {
+		vlogger.Error("failed to inspect validator", "error", err)
+		result.Error = err
+		return result
+	}
+	vlogger.Info("validator ip address:" + v.NetworkSettings.IPAddress)
 	vwaiter.Wait()
 
 	// Retrieve the exist status to report pass of fail
-	v, err := daemon.InspectContainer(vc.ID)
+	v, err = daemon.InspectContainer(vc.ID)
 	if err != nil {
 		vlogger.Error("failed to inspect validator", "error", err)
 		result.Error = err
 		return result
 	}
+
 	result.Success = v.State.ExitCode == 0
 	return result
 }

validators/devp2p/Dockerfile

@@ -0,0 +1,47 @@
+# This validation runs various devp2p tests against the client
+
+FROM golang:1-alpine
+
+# Build a geth bootnode, regular node from the develop branch and unit test binary
+RUN apk add --update git make gcc musl-dev curl jq linux-headers
+
+# Get master branch of geth
+RUN (git clone -b master --single-branch https://github.com/ethereum/go-ethereum /go/src/github.com/ethereum/go-ethereum)
+RUN (cd /go/src/github.com/ethereum/go-ethereum )
+
+# Build geth
+RUN (cd /go/src/github.com/ethereum/go-ethereum && GOPATH=/go make all)
+RUN (cd /go/src/github.com/ethereum/go-ethereum && GOPATH=/go go get)
+
+# Add the docker client
+RUN (git clone -b master https://github.com/fsouza/go-dockerclient /go/src/github.com/fsouza/go-dockerclient)
+
+RUN (cd /go/src/github.com/fsouza/go-dockerclient && GOPATH=/go go get)
+
+
+
+#RUN (cd / && GOPATH=/go go get github.com/fsouza/go-dockerclient)
+
+# Add the local test stuff
+ADD devp2p_test.go /devp2p_test.go
+ADD node.go /node.go
+ADD udp.go /udp.go
+
+
+#go test -c -o means compile the tests (-c) and rename (-o) to 
+RUN (cd / && GOPATH=/go go test -c -o /devp2p.test)
+
+# Cleanup previous stuff 
+RUN apk del git make gcc musl-dev linux-headers
+RUN (rm -rf /var/cache/apk/* && rm -rf /go)
+
+# Add shell
+RUN apk add --update bash curl jq
+# Add the entry script
+ADD tests.sh /tests.sh
+#mandatory init file
+ADD genesis.json /genesis.json
+
+EXPOSE 8545 8546 30303 30303/udp
+
+ENTRYPOINT ["/tests.sh"]