diff --git a/.github/actions/nimbus-build-system/action.yml b/.github/actions/nimbus-build-system/action.yml index 249d6fbad..fc71de246 100644 --- a/.github/actions/nimbus-build-system/action.yml +++ b/.github/actions/nimbus-build-system/action.yml @@ -29,6 +29,7 @@ runs: shell: ${{ inputs.shell }} {0} run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs/ | sh -s -- --default-toolchain=${{ inputs.rust_version }} -y + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - name: APT (Linux amd64/arm64) if: inputs.os == 'linux' && (inputs.cpu == 'amd64' || inputs.cpu == 'arm64') @@ -89,7 +90,7 @@ runs: - name: Install gcc 14 on Linux # We don't want to install gcc 14 for coverage (Ubuntu 20.04) - if : ${{ inputs.os == 'linux' && inputs.coverage != 'true' }} + if: ${{ inputs.os == 'linux' && inputs.coverage != 'true' }} shell: ${{ inputs.shell }} {0} run: | # Skip for older Ubuntu versions @@ -107,10 +108,17 @@ runs: if: inputs.os == 'linux' || inputs.os == 'macos' uses: hendrikmuhs/ccache-action@v1.2 with: - create-symlink: true + create-symlink: false key: ${{ inputs.os }}-${{ inputs.builder }}-${{ inputs.cpu }}-${{ inputs.tests }}-${{ inputs.nim_version }} evict-old-files: 7d + - name: Add ccache to path on Linux/Mac + if: inputs.os == 'linux' || inputs.os == 'macos' + shell: ${{ inputs.shell }} {0} + run: | + echo "/usr/lib/ccache:/usr/local/opt/ccache/libexec" >> "$GITHUB_PATH" + echo "/usr/local/opt/ccache/libexec" >> "$GITHUB_PATH" + - name: Install ccache on Windows if: inputs.os == 'windows' uses: hendrikmuhs/ccache-action@v1.2 @@ -123,11 +131,11 @@ runs: shell: ${{ inputs.shell }} {0} run: | CCACHE_DIR=$(dirname $(which ccache))/ccached - mkdir ${CCACHE_DIR} - ln -s $(which ccache) ${CCACHE_DIR}/gcc.exe - ln -s $(which ccache) ${CCACHE_DIR}/g++.exe - ln -s $(which ccache) ${CCACHE_DIR}/cc.exe - ln -s $(which ccache) ${CCACHE_DIR}/c++.exe + mkdir -p ${CCACHE_DIR} + ln -sf $(which ccache) ${CCACHE_DIR}/gcc.exe + ln -sf $(which ccache) ${CCACHE_DIR}/g++.exe + ln -sf $(which ccache) ${CCACHE_DIR}/cc.exe + ln -sf $(which ccache) ${CCACHE_DIR}/c++.exe echo "export PATH=${CCACHE_DIR}:\$PATH" >> $HOME/.bash_profile # prefix path in MSYS2 - name: Derive environment variables @@ -208,7 +216,7 @@ runs: - name: Restore Nim toolchain binaries from cache id: nim-cache uses: actions/cache@v4 - if : ${{ inputs.coverage != 'true' }} + if: ${{ inputs.coverage != 'true' }} with: path: NimBinaries key: ${{ inputs.os }}-${{ inputs.cpu }}-nim-${{ inputs.nim_version }}-cache-${{ env.cache_nonce }}-${{ github.run_id }} @@ -231,4 +239,4 @@ runs: gcc --version make -j${ncpu} CI_CACHE=NimBinaries ${ARCH_OVERRIDE} QUICK_AND_DIRTY_COMPILER=1 update echo - ./env.sh nim --version + ./env.sh nim --version \ No newline at end of file diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml index 55ee294ff..0be0ddb44 100644 --- a/.github/workflows/ci-reusable.yml +++ b/.github/workflows/ci-reusable.yml @@ -43,6 +43,46 @@ jobs: nim_version: ${{ matrix.nim_version }} coverage: false + - name: Check runner resources for parallel integration tests + run: | + echo "Determining runner" + case "${{ matrix.os }}" in + linux) CPU=$(nproc --all) + RAM=$(awk '/MemTotal/ {print int($2 / 1024 / 1024 + 0.5)}' /proc/meminfo) + ;; + macos) CPU=$(sysctl -n hw.ncpu) + RAM=$(sysctl -n hw.memsize | awk '{print $0/1073741824}') + sysctl -n hw.ncpu + ;; + windows) CPU=$NUMBER_OF_PROCESSORS + RAM=$(systeminfo | awk '/Total Physical Memory:/ { gsub(/,/,"."); print int($4 + 0.5) }') + ;; + *) CPU=2 + RAM=8 + echo "Unknown runner" + ;; + esac + echo "CPU=${CPU}" >> $GITHUB_ENV + echo "RAM=${RAM}" >> $GITHUB_ENV + echo "TYPE=${RUNNER_ENVIRONMENT}" >> $GITHUB_ENV + + # Set PARALLEL=1 if the runner has enough resources + if [[ ("${{ matrix.os }}" == "linux" || "${{ matrix.os }}" == "windows") && "${CPU}" -ge 16 ]]; then + echo "PARALLEL=1" >> $GITHUB_ENV + elif [[ "${{ matrix.os }}" == "macos" && "${CPU}" -ge 6 ]]; then + echo "PARALLEL=1" >> $GITHUB_ENV + else + echo "PARALLEL=0" >> $GITHUB_ENV + fi + + - name: Show runner information + run: | + echo "OS: ${{ matrix.os }}" + echo "CPU: ${{ env.CPU }}" + echo "RAM: ${{ env.RAM }} GB" + echo "TYPE: ${{ env.TYPE }}" + echo "PARALLEL: ${{ env.PARALLEL }}" + ## Part 1 Tests ## - name: Unit tests if: matrix.tests == 'unittest' || matrix.tests == 'all' @@ -53,13 +93,20 @@ jobs: with: node-version: 20 - - name: Start Ethereum node with Codex contracts + - name: Install Ethereum node dependencies if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'tools' || matrix.tests == 'all' working-directory: vendor/codex-contracts-eth env: MSYS2_PATH_TYPE: inherit run: | npm install + + - name: Run Ethereum node with Codex contracts + if: matrix.tests == 'contract' || (matrix.tests == 'integration' && env.PARALLEL != 1) || matrix.tests == 'tools' || matrix.tests == 'all' + working-directory: vendor/codex-contracts-eth + env: + MSYS2_PATH_TYPE: inherit + run: | npm start & ## Part 2 Tests ## @@ -70,13 +117,13 @@ jobs: ## Part 3 Tests ## - name: Integration tests if: matrix.tests == 'integration' || matrix.tests == 'all' - run: make -j${ncpu} testIntegration + run: make -j${ncpu} PARALLEL=${{ env.PARALLEL }} DEBUG=${{ runner.debug }} testIntegration - name: Upload integration tests log files uses: actions/upload-artifact@v4 if: (matrix.tests == 'integration' || matrix.tests == 'all') && always() with: - name: ${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-integration-tests-logs + name: ${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.tests }}-tests-logs path: tests/integration/logs/ retention-days: 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c045031c6..41537a05d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,10 @@ on: env: cache_nonce: 0 # Allows for easily busting actions/cache caches nim_version: v2.2.4 + builder_integration_from_vars: true + builder_integration_linux: runner-node-01-linux-03-eu-hel1 + builder_integration_macos: macos-14-xlarge + builder_integration_windows: windows-latest-amd64-32vcpu concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} @@ -17,27 +21,35 @@ concurrency: jobs: matrix: + name: Compute matrix runs-on: ubuntu-latest outputs: matrix: ${{ steps.matrix.outputs.matrix }} cache_nonce: ${{ env.cache_nonce }} steps: + - name: Compute builders + if: ${{ env.builder_integration_from_vars == 'true' }} + run: | + if [[ -n "${{ vars.builder_integration_linux }}" ]]; then echo "builder_integration_linux=${{ vars.builder_integration_linux }}"; fi >> $GITHUB_ENV + if [[ -n "${{ vars.builder_integration_macos }}" ]]; then echo "builder_integration_macos=${{ vars.builder_integration_macos }}"; fi >> $GITHUB_ENV + if [[ -n "${{ vars.builder_integration_windows }}" ]]; then echo "builder_integration_windows=${{ vars.builder_integration_windows }}"; fi >> $GITHUB_ENV + - name: Compute matrix id: matrix uses: fabiocaccamo/create-matrix-action@v5 with: matrix: | - os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {${{ env.builder_integration_linux }}}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} os {macos}, cpu {arm64}, builder {macos-14}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} os {macos}, cpu {arm64}, builder {macos-14}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {macos}, cpu {arm64}, builder {macos-14}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {macos}, cpu {arm64}, builder {${{ env.builder_integration_macos }}}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} os {macos}, cpu {arm64}, builder {macos-14}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {msys2} os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {msys2} - os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2} + os {windows}, cpu {amd64}, builder {${{ env.builder_integration_windows }}}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2} os {windows}, cpu {amd64}, builder {windows-latest}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {msys2} build: diff --git a/.gitmodules b/.gitmodules index 5cc2bfab6..f45d463b2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -231,3 +231,16 @@ url = https://github.com/vacp2p/nim-ngtcp2.git ignore = untracked branch = master +[submodule "vendor/nim-groth16"] + path = vendor/nim-groth16 + url = https://github.com/codex-storage/nim-groth16.git + ignore = untracked + branch = master +[submodule "vendor/nim-goldilocks-hash"] + path = vendor/nim-goldilocks-hash + url = https://github.com/codex-storage/nim-goldilocks-hash.git + ignore = untracked + branch = master +[submodule "vendor/circom-witnessgen"] + path = vendor/circom-witnessgen + url = https://github.com/codex-storage/circom-witnessgen.git diff --git a/Makefile b/Makefile index f39a3394a..11361fae9 100644 --- a/Makefile +++ b/Makefile @@ -140,10 +140,24 @@ testContracts: | build deps echo -e $(BUILD_MSG) "build/$@" && \ $(ENV_SCRIPT) nim testContracts $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims +TEST_PARAMS := +ifdef DEBUG + TEST_PARAMS := $(TEST_PARAMS) -d:DebugTestHarness=$(DEBUG) + TEST_PARAMS := $(TEST_PARAMS) -d:NoCodexLogFilters=$(DEBUG) + TEST_PARAMS := $(TEST_PARAMS) -d:ShowContinuousStatusUpdates=$(DEBUG) + TEST_PARAMS := $(TEST_PARAMS) -d:DebugHardhat=$(DEBUG) +endif +ifdef TEST_TIMEOUT + TEST_PARAMS := $(TEST_PARAMS) -d:TestTimeout=$(TEST_TIMEOUT) +endif +ifdef PARALLEL + TEST_PARAMS := $(TEST_PARAMS) -d:EnableParallelTests=$(PARALLEL) +endif + # Builds and runs the integration tests testIntegration: | build deps echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim testIntegration $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims + $(ENV_SCRIPT) nim testIntegration $(TEST_PARAMS) $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims # Builds and runs all tests (except for Taiko L2 tests) testAll: | build deps diff --git a/build.nims b/build.nims index 886603210..69596e011 100644 --- a/build.nims +++ b/build.nims @@ -1,8 +1,13 @@ mode = ScriptMode.Verbose import std/os except commandLineParams +import std/strutils ### Helper functions +proc truthy(val: string): bool = + const truthySwitches = @["yes", "1", "on", "true"] + return val in truthySwitches + proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") = if not dirExists "build": mkDir "build" @@ -45,11 +50,18 @@ task testContracts, "Build & run Codex Contract tests": task testIntegration, "Run integration tests": buildBinary "codex", params = - "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true" - test "testIntegration" + "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_disabled_topics=JSONRPC-HTTP-CLIENT,websock,libp2p,discv5 -d:codex_enable_proof_failures=true" + var sinks = @["textlines[nocolors,file]"] + for i in 2 ..< paramCount(): + if "DebugTestHarness" in paramStr(i) and truthy paramStr(i).split('=')[1]: + sinks.add "textlines[stdout]" + break + var testParams = + "-d:chronicles_log_level=TRACE -d:chronicles_sinks=\"" & sinks.join(",") & "\"" + test "testIntegration", params = testParams # use params to enable logging from the integration test executable # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & - # "-d:chronicles_enabled_topics:integration:TRACE" + # "-d:chronicles_enabled_topics:integration:TRACE" task build, "build codex binary": codexTask() diff --git a/codex.nim b/codex.nim index 7749bdee2..b534a0b34 100644 --- a/codex.nim +++ b/codex.nim @@ -10,7 +10,6 @@ import pkg/chronos import pkg/questionable import pkg/confutils -import pkg/confutils/defs import pkg/confutils/std/net import pkg/confutils/toml/defs as confTomlDefs import pkg/confutils/toml/std/net as confTomlNet diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index 36d00cf0b..307041f4b 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -68,6 +68,12 @@ const DefaultMaxPeersPerRequest* = 10 DefaultTaskQueueSize = 100 DefaultConcurrentTasks = 10 + # DefaultMaxRetries = 3 + # DefaultConcurrentDiscRequests = 10 + # DefaultConcurrentAdvertRequests = 10 + # DefaultDiscoveryTimeout = 1.minutes + # DefaultMaxQueriedBlocksCache = 1000 + # DefaultMinPeersPerBlock = 3 type TaskHandler* = proc(task: BlockExcPeerCtx): Future[void] {.gcsafe.} diff --git a/codex/codex.nim b/codex/codex.nim index 3ee48d68d..ccef82e14 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -208,22 +208,19 @@ proc new*( .withTcpTransport({ServerFlags.ReuseAddr}) .build() - var - cache: CacheStore = nil - taskpool: Taskpool - - try: - if config.numThreads == ThreadCount(0): - taskpool = Taskpool.new(numThreads = min(countProcessors(), 16)) + let numThreads = + if int(config.numThreads) == 0: + countProcessors() else: - taskpool = Taskpool.new(numThreads = int(config.numThreads)) - info "Threadpool started", numThreads = taskpool.numThreads - except CatchableError as exc: - raiseAssert("Failure in taskpool initialization:" & exc.msg) + int(config.numThreads) + + var tp = + try: + Taskpool.new(numThreads) + except CatchableError as exc: + raiseAssert("Failure in tp initialization:" & exc.msg) - if config.cacheSize > 0'nb: - cache = CacheStore.new(cacheSize = config.cacheSize) - ## Is unused? + info "Threadpool started", numThreads = tp.numThreads let discoveryDir = config.dataDir / CodexDhtNamespace @@ -299,9 +296,8 @@ proc new*( store = NetworkStore.new(engine, repoStore) prover = if config.prover: - let backend = - config.initializeBackend().expect("Unable to create prover backend.") - some Prover.new(store, backend, config.numProofSamples) + let prover = config.initializeProver(tp).expect("Unable to create prover.") + some prover else: none Prover @@ -311,7 +307,7 @@ proc new*( engine = engine, discovery = discovery, prover = prover, - taskPool = taskpool, + taskPool = tp, ) restServer = RestServerRef diff --git a/codex/conf.nim b/codex/conf.nim index af55861f5..44e6d3f41 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -18,6 +18,7 @@ import std/terminal # Is not used in tests import std/options import std/strutils import std/typetraits +import std/cpuinfo import pkg/chronos import pkg/chronicles/helpers @@ -54,9 +55,7 @@ export DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockInterval, DefaultNumBlocksPerInterval, DefaultRequestCacheSize -type ThreadCount* = distinct Natural - -proc `==`*(a, b: ThreadCount): bool {.borrow.} +type ThreadCount* = range[0 .. 256] proc defaultDataDir*(): string = let dataDir = @@ -76,7 +75,6 @@ const DefaultDataDir* = defaultDataDir() DefaultCircuitDir* = defaultDataDir() / "circuits" - DefaultThreadCount* = ThreadCount(0) type StartUpCmd* {.pure.} = enum @@ -87,6 +85,13 @@ type noCmd prover + ProverBackendCmd* {.pure.} = enum + nimgroth16 + circomcompat + + Curves* {.pure.} = enum + bn128 = "bn128" + LogKind* {.pure.} = enum Auto = "auto" Colors = "colors" @@ -193,7 +198,8 @@ type numThreads* {. desc: "Number of worker threads (\"0\" = use as many threads as there are CPU cores available)", - defaultValue: DefaultThreadCount, + defaultValueDesc: "0", + defaultValue: ThreadCount(0), name: "num-threads" .}: ThreadCount @@ -380,6 +386,22 @@ type name: "circuit-dir" .}: OutDir + proverBackend* {. + desc: + "The backend to use for the prover. " & + "Must be one of: nimgroth16, circomcompat", + defaultValue: ProverBackendCmd.nimgroth16, + defaultValueDesc: "nimgroth16", + name: "prover-backend" + .}: ProverBackendCmd + + curve* {. + desc: "The curve to use for the storage circuit", + defaultValue: Curves.bn128, + defaultValueDesc: $Curves.bn128, + name: "curve" + .}: Curves + circomR1cs* {. desc: "The r1cs file for the storage circuit", defaultValue: $DefaultCircuitDir / "proof_main.r1cs", @@ -387,8 +409,17 @@ type name: "circom-r1cs" .}: InputFile + circomGraph* {. + desc: + "The graph file for the storage circuit (only used with nimgroth16 backend)", + defaultValue: $DefaultCircuitDir / "proof_main.bin", + defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.bin", + name: "circom-graph" + .}: InputFile + circomWasm* {. - desc: "The wasm file for the storage circuit", + desc: + "The wasm file for the storage circuit (only used with circomcompat backend)", defaultValue: $DefaultCircuitDir / "proof_main.wasm", defaultValueDesc: $DefaultDataDir & "/circuits/proof_main.wasm", name: "circom-wasm" @@ -401,11 +432,11 @@ type name: "circom-zkey" .}: InputFile - # TODO: should probably be hidden and behind a feature flag circomNoZkey* {. desc: "Ignore the zkey file - use only for testing!", defaultValue: false, - name: "circom-no-zkey" + name: "circom-no-zkey", + hidden .}: bool numProofSamples* {. @@ -496,7 +527,7 @@ const proc parseCmdArg*( T: typedesc[MultiAddress], input: string -): MultiAddress {.upraises: [ValueError].} = +): MultiAddress {.raises: [ValueError].} = var ma: MultiAddress try: let res = MultiAddress.init(input) @@ -510,12 +541,8 @@ proc parseCmdArg*( quit QuitFailure ma -proc parseCmdArg*(T: type ThreadCount, input: string): T {.upraises: [ValueError].} = - let count = parseInt(input) - if count != 0 and count < 2: - warn "Invalid number of threads", input = input - quit QuitFailure - ThreadCount(count) +proc parseCmdArg*(T: type ThreadCount, val: string): T {.raises: [ValueError].} = + ThreadCount(val.parseUInt()) proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T = var res: SignedPeerRecord @@ -577,7 +604,7 @@ proc parseCmdArg*(T: type Duration, val: string): T = proc readValue*( r: var TomlReader, val: var EthAddress -) {.upraises: [SerializationError, IOError].} = +) {.raises: [SerializationError, IOError].} = val = EthAddress.init(r.readValue(string)).get() proc readValue*(r: var TomlReader, val: var SignedPeerRecord) = @@ -605,7 +632,7 @@ proc readValue*(r: var TomlReader, val: var MultiAddress) = proc readValue*( r: var TomlReader, val: var NBytes -) {.upraises: [SerializationError, IOError].} = +) {.raises: [SerializationError, IOError].} = var value = 0'i64 var str = r.readValue(string) let count = parseSize(str, value, alwaysBin = true) @@ -616,7 +643,7 @@ proc readValue*( proc readValue*( r: var TomlReader, val: var ThreadCount -) {.upraises: [SerializationError, IOError].} = +) {.raises: [SerializationError, IOError].} = var str = r.readValue(string) try: val = parseCmdArg(ThreadCount, str) @@ -625,7 +652,7 @@ proc readValue*( proc readValue*( r: var TomlReader, val: var Duration -) {.upraises: [SerializationError, IOError].} = +) {.raises: [SerializationError, IOError].} = var str = r.readValue(string) var dur: Duration let count = parseDuration(str, dur) @@ -692,7 +719,7 @@ proc stripAnsi*(v: string): string = res -proc updateLogLevel*(logLevel: string) {.upraises: [ValueError].} = +proc updateLogLevel*(logLevel: string) {.raises: [ValueError].} = # Updates log levels (without clearing old ones) let directives = logLevel.split(";") try: diff --git a/codex/erasure/erasure.nim b/codex/erasure/erasure.nim index e3d618ea7..472c27547 100644 --- a/codex/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -358,7 +358,7 @@ proc asyncEncode*( proc encodeData( self: Erasure, manifest: Manifest, params: EncodingParams -): Future[?!Manifest] {.async.} = +): Future[?!Manifest] {.async: (raises: [CancelledError]).} = ## Encode blocks pointed to by the protected manifest ## ## `manifest` - the manifest to encode @@ -457,7 +457,7 @@ proc encode*( blocks: Natural, parity: Natural, strategy = SteppedStrategy, -): Future[?!Manifest] {.async.} = +): Future[?!Manifest] {.async: (raises: [CancelledError]).} = ## Encode a manifest into one that is erasure protected. ## ## `manifest` - the original manifest to be encoded @@ -548,7 +548,9 @@ proc asyncDecode*( success() -proc decode*(self: Erasure, encoded: Manifest): Future[?!Manifest] {.async.} = +proc decode*( + self: Erasure, encoded: Manifest +): Future[?!Manifest] {.async: (raises: [CancelledError]).} = ## Decode a protected manifest into it's original ## manifest ## diff --git a/codex/logutils.nim b/codex/logutils.nim index e9604aba3..0d10b0fbb 100644 --- a/codex/logutils.nim +++ b/codex/logutils.nim @@ -92,6 +92,7 @@ import std/sugar import std/typetraits import pkg/chronicles except toJson, `%` +from pkg/chronos import TransportAddress from pkg/libp2p import Cid, MultiAddress, `$` import pkg/questionable import pkg/questionable/results @@ -255,3 +256,5 @@ formatIt(LogFormat.textLines, array[32, byte]): it.short0xHexLog formatIt(LogFormat.json, array[32, byte]): it.to0xHex +formatIt(TransportAddress): + $it diff --git a/codex/nat.nim b/codex/nat.nim index d022dad6c..f11f16ea5 100644 --- a/codex/nat.nim +++ b/codex/nat.nim @@ -423,10 +423,12 @@ proc nattedAddress*( it.remapAddr(ip = newIP, port = tcp) else: # NAT mapping failed - use original address - echo "Failed to get external IP, using original address", it + # TODO: `trace` breaks in the mapIt template + # trace "Failed to get external IP, using original address", it discoveryAddrs.add(getMultiAddrWithIPAndUDPPort(ipPart.get, udpPort)) it else: # Invalid multiaddress format - return as is it + (newAddrs, discoveryAddrs) diff --git a/codex/node.nim b/codex/node.nim index b742df2cc..2250b4313 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -46,6 +46,7 @@ import ./errors import ./logutils import ./utils/asynciter import ./utils/trackedfutures +import ./utils/poseidon2digest export logutils @@ -63,17 +64,17 @@ type ] CodexNode* = object - switch: Switch - networkId: PeerId - networkStore: NetworkStore - engine: BlockExcEngine - prover: ?Prover - discovery: Discovery - contracts*: Contracts - clock*: Clock - storage*: Contracts - taskpool: Taskpool - trackedFutures: TrackedFutures + switch: Switch # the libp2p network switch + networkId: PeerId # the peer id of the node + networkStore: NetworkStore # the network store + engine: BlockExcEngine # the block exchange engine + prover: ?Prover # the prover + discovery: Discovery # the discovery service + contracts*: Contracts # the contracts + clock*: Clock # the clock + storage*: Contracts # the storage + taskpool: Taskpool # the taskpool + trackedFutures: TrackedFutures # the tracked futures CodexNodeRef* = ref CodexNode @@ -96,18 +97,12 @@ func discovery*(self: CodexNodeRef): Discovery = proc storeManifest*( self: CodexNodeRef, manifest: Manifest -): Future[?!bt.Block] {.async.} = - without encodedVerifiable =? manifest.encode(), err: - trace "Unable to encode manifest" - return failure(err) - - without blk =? bt.Block.new(data = encodedVerifiable, codec = ManifestCodec), error: - trace "Unable to create block from manifest" - return failure(error) +): Future[?!bt.Block] {.async: (raises: [CancelledError]).} = + let + encodedVerifiable = ?manifest.encode() + blk = ?bt.Block.new(data = encodedVerifiable, codec = ManifestCodec) - if err =? (await self.networkStore.putBlock(blk)).errorOption: - trace "Unable to store manifest block", cid = blk.cid, err = err.msg - return failure(err) + ?await self.networkStore.putBlock(blk) success blk @@ -338,7 +333,9 @@ proc retrieve*( await self.streamEntireDataset(manifest, cid) -proc deleteSingleBlock(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} = +proc deleteSingleBlock( + self: CodexNodeRef, cid: Cid +): Future[?!void] {.async: (raises: [CancelledError]).} = if err =? (await self.networkStore.delBlock(cid)).errorOption: error "Error deleting block", cid, err = err.msg return failure(err) @@ -346,7 +343,9 @@ proc deleteSingleBlock(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} = trace "Deleted block", cid return success() -proc deleteEntireDataset(self: CodexNodeRef, cid: Cid): Future[?!void] {.async.} = +proc deleteEntireDataset( + self: CodexNodeRef, cid: Cid +): Future[?!void] {.async: (raises: [CancelledError]).} = # Deletion is a strictly local operation var store = self.networkStore.localStore @@ -403,7 +402,7 @@ proc store*( filename: ?string = string.none, mimetype: ?string = string.none, blockSize = DefaultBlockSize, -): Future[?!Cid] {.async.} = +): Future[?!Cid] {.async: (raises: [CancelledError]).} = ## Save stream contents as dataset with given blockSize ## to nodes's BlockStore, and return Cid of its manifest ## @@ -478,7 +477,9 @@ proc store*( return manifestBlk.cid.success -proc iterateManifests*(self: CodexNodeRef, onManifest: OnManifest) {.async.} = +proc iterateManifests*( + self: CodexNodeRef, onManifest: OnManifest +) {.async: (raises: [CancelledError]).} = without cidsIter =? await self.networkStore.listBlocks(BlockType.Manifest): warn "Failed to listBlocks" return @@ -505,7 +506,7 @@ proc setupRequest( pricePerBytePerSecond: UInt256, collateralPerByte: UInt256, expiry: uint64, -): Future[?!StorageRequest] {.async.} = +): Future[?!StorageRequest] {.async: (raises: [CancelledError]).} = ## Setup slots for a given dataset ## @@ -527,32 +528,20 @@ proc setupRequest( trace "Setting up slots" - without manifest =? await self.fetchManifest(cid), error: - trace "Unable to fetch manifest for cid" - return failure error - - # Erasure code the dataset according to provided parameters - let erasure = Erasure.new( - self.networkStore.localStore, leoEncoderProvider, leoDecoderProvider, self.taskpool - ) - - without encoded =? (await erasure.encode(manifest, ecK, ecM)), error: - trace "Unable to erasure code dataset" - return failure(error) - - without builder =? Poseidon2Builder.new(self.networkStore.localStore, encoded), err: - trace "Unable to create slot builder" - return failure(err) + let + manifest = ?await self.fetchManifest(cid) - without verifiable =? (await builder.buildManifest()), err: - trace "Unable to build verifiable manifest" - return failure(err) + # Erasure code the dataset according to provided parameters + erasure = Erasure.new( + self.networkStore.localStore, leoEncoderProvider, leoDecoderProvider, + self.taskpool, + ) - without manifestBlk =? await self.storeManifest(verifiable), err: - trace "Unable to store verifiable manifest" - return failure(err) + encoded = ?await erasure.encode(manifest, ecK, ecM) + builder = ?Poseidon2Builder.new(self.networkStore.localStore, encoded) + verifiable = ?await builder.buildManifest() + manifestBlk = ?await self.storeManifest(verifiable) - let verifyRoot = if builder.verifyRoot.isNone: return failure("No slots root") @@ -586,7 +575,7 @@ proc requestStorage*( pricePerBytePerSecond: UInt256, collateralPerByte: UInt256, expiry: uint64, -): Future[?!PurchaseId] {.async.} = +): Future[?!PurchaseId] {.async: (raises: [CancelledError]).} = ## Initiate a request for storage sequence, this might ## be a multistep procedure. ## @@ -617,7 +606,17 @@ proc requestStorage*( trace "Unable to setup request" return failure err - let purchase = await contracts.purchasing.purchase(request) + # TODO: remove try/except once state machine has checked exceptions + let purchase = + try: + await contracts.purchasing.purchase(request) + except CancelledError as err: + trace "Purchase cancelled", err = err.msg + raise err + except CatchableError as err: + trace "Unable to purchase storage", err = err.msg + return failure(err) + success purchase.id proc onStore( @@ -731,38 +730,28 @@ proc onProve( if prover =? self.prover: trace "Prover enabled" - without cid =? Cid.init(cidStr).mapFailure, err: - error "Unable to parse Cid", cid, err = err.msg - return failure(err) - - without manifest =? await self.fetchManifest(cid), err: - error "Unable to fetch manifest for cid", err = err.msg - return failure(err) + let + cid = ?Cid.init(cidStr).mapFailure + manifest = ?await self.fetchManifest(cid) + builder = + ?Poseidon2Builder.new(self.networkStore, manifest, manifest.verifiableStrategy) + sampler = ?Poseidon2Sampler.new(slotIdx, self.networkStore, builder) when defined(verify_circuit): - without (inputs, proof) =? await prover.prove(slotIdx.int, manifest, challenge), - err: - error "Unable to generate proof", err = err.msg - return failure(err) + let (proof, checked) = + ?await prover.prove(sampler, manifest, challenge, verify = true) - without checked =? await prover.verify(proof, inputs), err: - error "Unable to verify proof", err = err.msg - return failure(err) - - if not checked: + if checked.isSome and not checked.get: error "Proof verification failed" return failure("Proof verification failed") trace "Proof verified successfully" else: - without (_, proof) =? await prover.prove(slotIdx.int, manifest, challenge), err: - error "Unable to generate proof", err = err.msg - return failure(err) + let (proof, _) = ?await prover.prove(sampler, manifest, challenge, verify = false) - let groth16Proof = proof.toGroth16Proof() - trace "Proof generated successfully", groth16Proof + trace "Proof generated successfully", proof - success groth16Proof + success proof else: warn "Prover not enabled" failure "Prover not enabled" diff --git a/codex/purchasing/states/submitted.nim b/codex/purchasing/states/submitted.nim index dd3669e45..96d384a41 100644 --- a/codex/purchasing/states/submitted.nim +++ b/codex/purchasing/states/submitted.nim @@ -30,12 +30,12 @@ method run*( requestId = purchase.requestId proc wait() {.async.} = - let done = newFuture[void]() + let done = newAsyncEvent() proc callback(_: RequestId) = - done.complete() + done.fire() let subscription = await market.subscribeFulfillment(request.id, callback) - await done + await done.wait() await subscription.unsubscribe() proc withTimeout(future: Future[void]) {.async.} = diff --git a/codex/sales/statemachine.nim b/codex/sales/statemachine.nim index ec770ece0..d17325491 100644 --- a/codex/sales/statemachine.nim +++ b/codex/sales/statemachine.nim @@ -12,7 +12,7 @@ export asyncstatemachine type SaleState* = ref object of State - SaleError* = ref object of CodexError + SaleError* = object of CodexError method onCancelled*( state: SaleState, request: StorageRequest diff --git a/codex/sales/states/preparing.nim b/codex/sales/states/preparing.nim index 807bb196c..dba249de5 100644 --- a/codex/sales/states/preparing.nim +++ b/codex/sales/states/preparing.nim @@ -51,7 +51,9 @@ method run*( await agent.subscribe() without request =? data.request: - raiseAssert "no sale request" + error "request could not be retrieved", id = data.requestId + let error = newException(SaleError, "request could not be retrieved") + return some State(SaleErrored(error: error)) let slotId = slotId(data.requestId, data.slotIndex) let state = await market.slotState(slotId) diff --git a/codex/sales/states/unknown.nim b/codex/sales/states/unknown.nim index d182d7442..b714a4b98 100644 --- a/codex/sales/states/unknown.nim +++ b/codex/sales/states/unknown.nim @@ -38,6 +38,11 @@ method run*( await agent.retrieveRequest() await agent.subscribe() + without request =? data.request: + error "request could not be retrieved", id = data.requestId + let error = newException(SaleError, "request could not be retrieved") + return some State(SaleErrored(error: error)) + let slotId = slotId(data.requestId, data.slotIndex) let slotState = await market.slotState(slotId) diff --git a/codex/slots/builder.nim b/codex/slots/builder.nim index 25844db63..1857150c8 100644 --- a/codex/slots/builder.nim +++ b/codex/slots/builder.nim @@ -3,6 +3,6 @@ import ./converters import ../merkletree -export builder, converters +export builder, converters, merkletree type Poseidon2Builder* = SlotsBuilder[Poseidon2Tree, Poseidon2Hash] diff --git a/codex/slots/builder/builder.nim b/codex/slots/builder/builder.nim index a26fc04e0..7bf6baea1 100644 --- a/codex/slots/builder/builder.nim +++ b/codex/slots/builder/builder.nim @@ -34,107 +34,125 @@ export converters, asynciter logScope: topics = "codex slotsbuilder" -type SlotsBuilder*[T, H] = ref object of RootObj +type SlotsBuilder*[SomeTree, SomeHash] = ref object of RootObj store: BlockStore manifest: Manifest # current manifest strategy: IndexingStrategy # indexing strategy cellSize: NBytes # cell size numSlotBlocks: Natural # number of blocks per slot (should yield a power of two number of cells) - slotRoots: seq[H] # roots of the slots + slotRoots: seq[SomeHash] # roots of the slots emptyBlock: seq[byte] # empty block - verifiableTree: ?T # verification tree (dataset tree) - emptyDigestTree: T # empty digest tree for empty blocks + verifiableTree: ?SomeTree # verification tree (dataset tree) + emptyDigestTree: SomeTree # empty digest tree for empty blocks -func verifiable*[T, H](self: SlotsBuilder[T, H]): bool {.inline.} = +func verifiable*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash] +): bool {.inline.} = ## Returns true if the slots are verifiable. ## self.manifest.verifiable -func slotRoots*[T, H](self: SlotsBuilder[T, H]): seq[H] {.inline.} = +func slotRoots*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash] +): seq[SomeHash] {.inline.} = ## Returns the slot roots. ## self.slotRoots -func verifyTree*[T, H](self: SlotsBuilder[T, H]): ?T {.inline.} = +func verifyTree*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash] +): ?SomeTree {.inline.} = ## Returns the slots tree (verification tree). ## self.verifiableTree -func verifyRoot*[T, H](self: SlotsBuilder[T, H]): ?H {.inline.} = +func verifyRoot*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash] +): ?SomeHash {.inline.} = ## Returns the slots root (verification root). ## if tree =? self.verifyTree and root =? tree.root: return some root -func numSlots*[T, H](self: SlotsBuilder[T, H]): Natural = +func numSlots*[SomeTree, SomeHash](self: SlotsBuilder[SomeTree, SomeHash]): Natural = ## Number of slots. ## self.manifest.numSlots -func numSlotBlocks*[T, H](self: SlotsBuilder[T, H]): Natural = +func numSlotBlocks*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash] +): Natural = ## Number of blocks per slot. ## self.numSlotBlocks -func numBlocks*[T, H](self: SlotsBuilder[T, H]): Natural = +func numBlocks*[SomeTree, SomeHash](self: SlotsBuilder[SomeTree, SomeHash]): Natural = ## Number of blocks. ## self.numSlotBlocks * self.manifest.numSlots -func slotBytes*[T, H](self: SlotsBuilder[T, H]): NBytes = +func slotBytes*[SomeTree, SomeHash](self: SlotsBuilder[SomeTree, SomeHash]): NBytes = ## Number of bytes per slot. ## (self.manifest.blockSize.int * self.numSlotBlocks).NBytes -func numBlockCells*[T, H](self: SlotsBuilder[T, H]): Natural = +func numBlockCells*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash] +): Natural = ## Number of cells per block. ## (self.manifest.blockSize div self.cellSize).Natural -func cellSize*[T, H](self: SlotsBuilder[T, H]): NBytes = +func cellSize*[SomeTree, SomeHash](self: SlotsBuilder[SomeTree, SomeHash]): NBytes = ## Cell size. ## self.cellSize -func numSlotCells*[T, H](self: SlotsBuilder[T, H]): Natural = +func numSlotCells*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash] +): Natural = ## Number of cells per slot. ## self.numBlockCells * self.numSlotBlocks -func slotIndiciesIter*[T, H](self: SlotsBuilder[T, H], slot: Natural): ?!Iter[int] = +func slotIndiciesIter*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash], slot: Natural +): ?!Iter[int] = ## Returns the slot indices. ## self.strategy.getIndicies(slot).catch -func slotIndicies*[T, H](self: SlotsBuilder[T, H], slot: Natural): seq[int] = +func slotIndicies*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash], slot: Natural +): seq[int] = ## Returns the slot indices. ## if iter =? self.strategy.getIndicies(slot).catch: return toSeq(iter) -func manifest*[T, H](self: SlotsBuilder[T, H]): Manifest = +func manifest*[SomeTree, SomeHash](self: SlotsBuilder[SomeTree, SomeHash]): Manifest = ## Returns the manifest. ## self.manifest -proc buildBlockTree*[T, H]( - self: SlotsBuilder[T, H], blkIdx: Natural, slotPos: Natural -): Future[?!(seq[byte], T)] {.async: (raises: [CancelledError]).} = +proc buildBlockTree*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash], blkIdx: Natural, slotPos: Natural +): Future[?!(seq[byte], SomeTree)] {.async: (raises: [CancelledError]).} = ## Build the block digest tree and return a tuple with the ## block data and the tree. ## @@ -152,22 +170,17 @@ proc buildBlockTree*[T, H]( trace "Returning empty digest tree for pad block" return success (self.emptyBlock, self.emptyDigestTree) - without blk =? await self.store.getBlock(self.manifest.treeCid, blkIdx), err: - error "Failed to get block CID for tree at index", err = err.msg - return failure(err) + let blk = ?await self.store.getBlock(self.manifest.treeCid, blkIdx) if blk.isEmpty: success (self.emptyBlock, self.emptyDigestTree) else: - without tree =? T.digestTree(blk.data, self.cellSize.int), err: - error "Failed to create digest for block", err = err.msg - return failure(err) - + let tree = ?SomeTree.digestTree(blk.data, self.cellSize.int) success (blk.data, tree) -proc getCellHashes*[T, H]( - self: SlotsBuilder[T, H], slotIndex: Natural -): Future[?!seq[H]] {.async: (raises: [CancelledError, IndexingError]).} = +proc getCellHashes*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash], slotIndex: Natural +): Future[?!seq[SomeHash]] {.async: (raises: [CancelledError]).} = ## Collect all the cells from a block and return ## their hashes. ## @@ -184,7 +197,7 @@ proc getCellHashes*[T, H]( slotIndex = slotIndex let hashes = collect(newSeq): - for i, blkIdx in self.strategy.getIndicies(slotIndex): + for i, blkIdx in ?self.strategy.getIndicies(slotIndex).catch: logScope: blkIdx = blkIdx pos = i @@ -200,25 +213,18 @@ proc getCellHashes*[T, H]( success hashes -proc buildSlotTree*[T, H]( - self: SlotsBuilder[T, H], slotIndex: Natural -): Future[?!T] {.async: (raises: [CancelledError]).} = +proc buildSlotTree*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash], slotIndex: Natural +): Future[?!SomeTree] {.async: (raises: [CancelledError]).} = ## Build the slot tree from the block digest hashes ## and return the tree. - try: - without cellHashes =? (await self.getCellHashes(slotIndex)), err: - error "Failed to select slot blocks", err = err.msg - return failure(err) - - T.init(cellHashes) - except IndexingError as err: - error "Failed to build slot tree", err = err.msg - return failure(err) + let cellHashes = ?await self.getCellHashes(slotIndex) + SomeTree.init(cellHashes) -proc buildSlot*[T, H]( - self: SlotsBuilder[T, H], slotIndex: Natural -): Future[?!H] {.async: (raises: [CancelledError]).} = +proc buildSlot*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash], slotIndex: Natural +): Future[?!SomeHash] {.async: (raises: [CancelledError]).} = ## Build a slot tree and store the proofs in ## the block store. ## @@ -244,18 +250,17 @@ proc buildSlot*[T, H]( error "Failed to get proof for slot tree", err = err.msg return failure(err) - if err =? - (await self.store.putCidAndProof(treeCid, i, cellCid, encodableProof)).errorOption: - error "Failed to store slot tree", err = err.msg - return failure(err) + ?(await self.store.putCidAndProof(treeCid, i, cellCid, encodableProof)) tree.root() -func buildVerifyTree*[T, H](self: SlotsBuilder[T, H], slotRoots: openArray[H]): ?!T = - T.init(@slotRoots) +func buildVerifyTree*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash], slotRoots: openArray[SomeHash] +): ?!SomeTree = + SomeTree.init(@slotRoots) -proc buildSlots*[T, H]( - self: SlotsBuilder[T, H] +proc buildSlots*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash] ): Future[?!void] {.async: (raises: [CancelledError]).} = ## Build all slot trees and store them in the block store. ## @@ -269,10 +274,7 @@ proc buildSlots*[T, H]( if self.slotRoots.len == 0: self.slotRoots = collect(newSeq): for i in 0 ..< self.manifest.numSlots: - without slotRoot =? (await self.buildSlot(i)), err: - error "Failed to build slot", err = err.msg, index = i - return failure(err) - slotRoot + ?(await self.buildSlot(i)) without tree =? self.buildVerifyTree(self.slotRoots) and root =? tree.root, err: error "Failed to build slot roots tree", err = err.msg @@ -286,17 +288,15 @@ proc buildSlots*[T, H]( success() -proc buildManifest*[T, H]( - self: SlotsBuilder[T, H] +proc buildManifest*[SomeTree, SomeHash]( + self: SlotsBuilder[SomeTree, SomeHash] ): Future[?!Manifest] {.async: (raises: [CancelledError]).} = - if err =? (await self.buildSlots()).errorOption: - error "Failed to build slot roots", err = err.msg - return failure(err) + ## Build the manifest from the slots and return it. + ## - without rootCids =? self.slotRoots.toSlotCids(), err: - error "Failed to map slot roots to CIDs", err = err.msg - return failure(err) + ?(await self.buildSlots()) # build all slots first + let rootCids = ?self.slotRoots.toSlotCids() without rootProvingCidRes =? self.verifyRoot .? toVerifyCid() and rootProvingCid =? rootProvingCidRes, err: error "Failed to map slot roots to CIDs", err = err.msg @@ -306,13 +306,13 @@ proc buildManifest*[T, H]( self.manifest, rootProvingCid, rootCids, self.cellSize, self.strategy.strategyType ) -proc new*[T, H]( - _: type SlotsBuilder[T, H], +proc new*[SomeTree, SomeHash]( + _: type SlotsBuilder[SomeTree, SomeHash], store: BlockStore, manifest: Manifest, strategy = SteppedStrategy, cellSize = DefaultCellSize, -): ?!SlotsBuilder[T, H] = +): ?!SlotsBuilder[SomeTree, SomeHash] = if not manifest.protected: trace "Manifest is not protected." return failure("Manifest is not protected.") @@ -352,7 +352,7 @@ proc new*[T, H]( numBlocksTotal = numSlotBlocksTotal * manifest.numSlots # number of blocks per slot emptyBlock = newSeq[byte](manifest.blockSize.int) - emptyDigestTree = ?T.digestTree(emptyBlock, cellSize.int) + emptyDigestTree = ?SomeTree.digestTree(emptyBlock, cellSize.int) strategy = ?strategy.init(0, numBlocksTotal - 1, manifest.numSlots).catch @@ -368,7 +368,7 @@ proc new*[T, H]( trace "Creating slots builder" - var self = SlotsBuilder[T, H]( + var self = SlotsBuilder[SomeTree, SomeHash]( store: store, manifest: manifest, strategy: strategy, diff --git a/codex/slots/proofs.nim b/codex/slots/proofs.nim index 4f7f01b58..a1f56d9a0 100644 --- a/codex/slots/proofs.nim +++ b/codex/slots/proofs.nim @@ -1,5 +1,5 @@ import ./proofs/backends import ./proofs/prover -import ./proofs/backendfactory +import ./proofs/proverfactory -export circomcompat, prover, backendfactory +export backends, prover, proverfactory diff --git a/codex/slots/proofs/backendfactory.nim b/codex/slots/proofs/backendfactory.nim deleted file mode 100644 index 7aba27d85..000000000 --- a/codex/slots/proofs/backendfactory.nim +++ /dev/null @@ -1,82 +0,0 @@ -import os -import strutils -import pkg/chronos -import pkg/chronicles -import pkg/questionable -import pkg/confutils/defs -import pkg/stew/io2 -import pkg/ethers - -import ../../conf -import ./backends -import ./backendutils - -proc initializeFromConfig(config: CodexConf, utils: BackendUtils): ?!AnyBackend = - if not fileAccessible($config.circomR1cs, {AccessFlags.Read}) or - not endsWith($config.circomR1cs, ".r1cs"): - return failure("Circom R1CS file not accessible") - - if not fileAccessible($config.circomWasm, {AccessFlags.Read}) or - not endsWith($config.circomWasm, ".wasm"): - return failure("Circom wasm file not accessible") - - if not fileAccessible($config.circomZkey, {AccessFlags.Read}) or - not endsWith($config.circomZkey, ".zkey"): - return failure("Circom zkey file not accessible") - - trace "Initialized prover backend from cli config" - success( - utils.initializeCircomBackend( - $config.circomR1cs, $config.circomWasm, $config.circomZkey - ) - ) - -proc r1csFilePath(config: CodexConf): string = - config.circuitDir / "proof_main.r1cs" - -proc wasmFilePath(config: CodexConf): string = - config.circuitDir / "proof_main.wasm" - -proc zkeyFilePath(config: CodexConf): string = - config.circuitDir / "proof_main.zkey" - -proc initializeFromCircuitDirFiles( - config: CodexConf, utils: BackendUtils -): ?!AnyBackend {.gcsafe.} = - if fileExists(config.r1csFilePath) and fileExists(config.wasmFilePath) and - fileExists(config.zkeyFilePath): - trace "Initialized prover backend from local files" - return success( - utils.initializeCircomBackend( - config.r1csFilePath, config.wasmFilePath, config.zkeyFilePath - ) - ) - - failure("Circuit files not found") - -proc suggestDownloadTool(config: CodexConf) = - without address =? config.marketplaceAddress: - raise (ref Defect)( - msg: "Proving backend initializing while marketplace address not set." - ) - - let - tokens = ["cirdl", "\"" & $config.circuitDir & "\"", config.ethProvider, $address] - instructions = "'./" & tokens.join(" ") & "'" - - warn "Proving circuit files are not found. Please run the following to download them:", - instructions - -proc initializeBackend*( - config: CodexConf, utils: BackendUtils = BackendUtils() -): ?!AnyBackend = - without backend =? initializeFromConfig(config, utils), cliErr: - info "Could not initialize prover backend from CLI options...", msg = cliErr.msg - without backend =? initializeFromCircuitDirFiles(config, utils), localErr: - info "Could not initialize prover backend from circuit dir files...", - msg = localErr.msg - suggestDownloadTool(config) - return failure("CircuitFilesNotFound") - # Unexpected: value of backend does not survive leaving each scope. (definition does though...) - return success(backend) - return success(backend) diff --git a/codex/slots/proofs/backends.nim b/codex/slots/proofs/backends.nim index 3bd2edb6c..b0b79b370 100644 --- a/codex/slots/proofs/backends.nim +++ b/codex/slots/proofs/backends.nim @@ -1,5 +1,4 @@ import ./backends/circomcompat +import ./backends/nimgroth16 -export circomcompat - -type AnyBackend* = CircomCompat +export circomcompat, nimgroth16 diff --git a/codex/slots/proofs/backends/circomcompat.nim b/codex/slots/proofs/backends/circomcompat.nim index 1d2e3e19a..7e66a264e 100644 --- a/codex/slots/proofs/backends/circomcompat.nim +++ b/codex/slots/proofs/backends/circomcompat.nim @@ -7,6 +7,8 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. +{.deprecated: "use the NimGroth16Backend".} + {.push raises: [].} import std/sugar @@ -24,7 +26,7 @@ import ./converters export circomcompat, converters type - CircomCompat* = object + CircomCompatBackend* = object slotDepth: int # max depth of the slot tree datasetDepth: int # max depth of dataset tree blkDepth: int # depth of the block merkle tree (pow2 for now) @@ -34,13 +36,15 @@ type wasmPath: string # path to the wasm file zkeyPath: string # path to the zkey file backendCfg: ptr CircomBn254Cfg - vkp*: ptr CircomKey + vkp*: ptr CircomCompatKey + + CircomCompatBackendRef* = ref CircomCompatBackend - NormalizedProofInputs*[H] {.borrow: `.`.} = distinct ProofInputs[H] + NormalizedProofInputs*[SomeHash] {.borrow: `.`.} = distinct ProofInputs[SomeHash] -func normalizeInput*[H]( - self: CircomCompat, input: ProofInputs[H] -): NormalizedProofInputs[H] = +func normalizeInput*[SomeHash]( + self: CircomCompatBackendRef, input: ProofInputs[SomeHash] +): NormalizedProofInputs[SomeHash] = ## Parameters in CIRCOM circuits are statically sized and must be properly ## padded before they can be passed onto the circuit. This function takes ## variable length parameters and performs that padding. @@ -53,23 +57,25 @@ func normalizeInput*[H]( for sample in input.samples: var merklePaths = sample.merklePaths merklePaths.setLen(self.slotDepth) - Sample[H](cellData: sample.cellData, merklePaths: merklePaths) + Sample[SomeHash](cellData: sample.cellData, merklePaths: merklePaths) var normSlotProof = input.slotProof normSlotProof.setLen(self.datasetDepth) - NormalizedProofInputs[H] ProofInputs[H]( - entropy: input.entropy, - datasetRoot: input.datasetRoot, - slotIndex: input.slotIndex, - slotRoot: input.slotRoot, - nCellsPerSlot: input.nCellsPerSlot, - nSlotsPerDataSet: input.nSlotsPerDataSet, - slotProof: normSlotProof, - samples: normSamples, + NormalizedProofInputs[SomeHash]( + ProofInputs[SomeHash]( + entropy: input.entropy, + datasetRoot: input.datasetRoot, + slotIndex: input.slotIndex, + slotRoot: input.slotRoot, + nCellsPerSlot: input.nCellsPerSlot, + nSlotsPerDataSet: input.nSlotsPerDataSet, + slotProof: normSlotProof, + samples: normSamples, + ) ) -proc release*(self: CircomCompat) = +proc release*(self: CircomCompatBackendRef) = ## Release the ctx ## @@ -79,7 +85,9 @@ proc release*(self: CircomCompat) = if not isNil(self.vkp): self.vkp.unsafeAddr.release_key() -proc prove[H](self: CircomCompat, input: NormalizedProofInputs[H]): ?!CircomProof = +proc prove[SomeHash]( + self: CircomCompatBackendRef, input: NormalizedProofInputs[SomeHash] +): Future[?!CircomCompatProof] {.async: (raises: [CancelledError]).} = doAssert input.samples.len == self.numSamples, "Number of samples does not match" doAssert input.slotProof.len <= self.datasetDepth, @@ -101,7 +109,7 @@ proc prove[H](self: CircomCompat, input: NormalizedProofInputs[H]): ?!CircomProo ctx.addr.release_circom_compat() if init_circom_compat(self.backendCfg, addr ctx) != ERR_OK or ctx == nil: - raiseAssert("failed to initialize CircomCompat ctx") + raiseAssert("failed to initialize CircomCompatBackend ctx") var entropy = input.entropy.toBytes @@ -172,12 +180,16 @@ proc prove[H](self: CircomCompat, input: NormalizedProofInputs[H]): ?!CircomProo success proof -proc prove*[H](self: CircomCompat, input: ProofInputs[H]): ?!CircomProof = +proc prove*[SomeHash]( + self: CircomCompatBackendRef, input: ProofInputs[SomeHash] +): Future[?!CircomCompatProof] {.async: (raises: [CancelledError], raw: true).} = self.prove(self.normalizeInput(input)) -proc verify*[H]( - self: CircomCompat, proof: CircomProof, inputs: ProofInputs[H] -): ?!bool = +proc verify*[SomeHash]( + self: CircomCompatBackendRef, + proof: CircomCompatProof, + inputs: ProofInputs[SomeHash], +): Future[?!bool] {.async: (raises: [CancelledError]).} = ## Verify a proof using a ctx ## @@ -196,8 +208,8 @@ proc verify*[H]( finally: inputs.releaseCircomInputs() -proc init*( - _: type CircomCompat, +proc new*( + _: type CircomCompatBackendRef, r1csPath: string, wasmPath: string, zkeyPath: string = "", @@ -206,7 +218,7 @@ proc init*( blkDepth = DefaultBlockDepth, cellElms = DefaultCellElms, numSamples = DefaultSamplesNum, -): CircomCompat = +): ?!CircomCompatBackendRef = ## Create a new ctx ## @@ -217,16 +229,16 @@ proc init*( cfg == nil: if cfg != nil: cfg.addr.release_cfg() - raiseAssert("failed to initialize circom compat config") + return failure "failed to initialize circom compat config" var vkpPtr: ptr VerifyingKey = nil if cfg.get_verifying_key(vkpPtr.addr) != ERR_OK or vkpPtr == nil: if vkpPtr != nil: vkpPtr.addr.release_key() - raiseAssert("Failed to get verifying key") + return failure "Failed to get verifying key" - CircomCompat( + success CircomCompatBackendRef( r1csPath: r1csPath, wasmPath: wasmPath, zkeyPath: zkeyPath, diff --git a/codex/slots/proofs/backends/converters.nim b/codex/slots/proofs/backends/converters.nim index ee771477d..65007d84a 100644 --- a/codex/slots/proofs/backends/converters.nim +++ b/codex/slots/proofs/backends/converters.nim @@ -1,5 +1,5 @@ ## Nim-Codex -## Copyright (c) 2024 Status Research & Development GmbH +## Copyright (c) 2025 Status Research & Development GmbH ## Licensed under either of ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) @@ -9,21 +9,27 @@ {.push raises: [].} +import pkg/groth16 import pkg/circomcompat +import pkg/constantine/math/io/io_fields import ../../../contracts import ../../types import ../../../merkletree type - CircomG1* = G1 - CircomG2* = G2 + CircomCompatG1* = circomcompat.G1 + CircomCompatG2* = circomcompat.G2 - CircomProof* = Proof - CircomKey* = VerifyingKey - CircomInputs* = Inputs + CircomCompatProof* = circomcompat.Proof + CircomCompatKey* = circomcompat.VerifyingKey + CircomCompatInputs* = circomcompat.Inputs -proc toCircomInputs*(inputs: ProofInputs[Poseidon2Hash]): CircomInputs = + NimGroth16G1* = groth16.G1 + NimGroth16G2* = groth16.G2 + NimGroth16Proof* = groth16.Proof + +proc toCircomInputs*(inputs: ProofInputs[Poseidon2Hash]): CircomCompatInputs = var slotIndex = inputs.slotIndex.toF.toBytes.toArray32 datasetRoot = inputs.datasetRoot.toBytes.toArray32 @@ -34,21 +40,49 @@ proc toCircomInputs*(inputs: ProofInputs[Poseidon2Hash]): CircomInputs = let inputsPtr = allocShared0(32 * elms.len) copyMem(inputsPtr, addr elms[0], elms.len * 32) - CircomInputs(elms: cast[ptr array[32, byte]](inputsPtr), len: elms.len.uint) + CircomCompatInputs(elms: cast[ptr array[32, byte]](inputsPtr), len: elms.len.uint) -proc releaseCircomInputs*(inputs: var CircomInputs) = +proc releaseCircomInputs*(inputs: var CircomCompatInputs) = if not inputs.elms.isNil: deallocShared(inputs.elms) inputs.elms = nil -func toG1*(g: CircomG1): G1Point = +func toG1*(g: CircomCompatG1): G1Point = G1Point(x: UInt256.fromBytesLE(g.x), y: UInt256.fromBytesLE(g.y)) -func toG2*(g: CircomG2): G2Point = +func toG2*(g: CircomCompatG2): G2Point = G2Point( x: Fp2Element(real: UInt256.fromBytesLE(g.x[0]), imag: UInt256.fromBytesLE(g.x[1])), y: Fp2Element(real: UInt256.fromBytesLE(g.y[0]), imag: UInt256.fromBytesLE(g.y[1])), ) -func toGroth16Proof*(proof: CircomProof): Groth16Proof = +func toGroth16Proof*(proof: CircomCompatProof): Groth16Proof = Groth16Proof(a: proof.a.toG1, b: proof.b.toG2, c: proof.c.toG1) + +func toG1*(g: NimGroth16G1): G1Point = + var + x: array[32, byte] + y: array[32, byte] + + assert x.marshal(g.x, Endianness.littleEndian) + assert y.marshal(g.y, Endianness.littleEndian) + + G1Point(x: UInt256.fromBytesLE(x), y: UInt256.fromBytesLE(y)) + +func toG2*(g: NimGroth16G2): G2Point = + var + x: array[2, array[32, byte]] + y: array[2, array[32, byte]] + + assert x[0].marshal(g.x.coords[0], Endianness.littleEndian) + assert x[1].marshal(g.x.coords[1], Endianness.littleEndian) + assert y[0].marshal(g.y.coords[0], Endianness.littleEndian) + assert y[1].marshal(g.y.coords[1], Endianness.littleEndian) + + G2Point( + x: Fp2Element(real: UInt256.fromBytesLE(x[0]), imag: UInt256.fromBytesLE(x[1])), + y: Fp2Element(real: UInt256.fromBytesLE(y[0]), imag: UInt256.fromBytesLE(y[1])), + ) + +func toGroth16Proof*(proof: NimGroth16Proof): Groth16Proof = + Groth16Proof(a: proof.pi_a.toG1, b: proof.pi_b.toG2, c: proof.pi_c.toG1) diff --git a/codex/slots/proofs/backends/nimgroth16.nim b/codex/slots/proofs/backends/nimgroth16.nim new file mode 100644 index 000000000..5c38fc233 --- /dev/null +++ b/codex/slots/proofs/backends/nimgroth16.nim @@ -0,0 +1,208 @@ +## Nim-Codex +## Copyright (c) 2025 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +{.push raises: [].} + +import std/sugar +import std/isolation +import std/atomics + +import pkg/chronos +import pkg/chronos/threadsync +import pkg/taskpools +import pkg/questionable/results + +import pkg/groth16 +import pkg/nim/circom_witnessgen +import pkg/nim/circom_witnessgen/load +import pkg/nim/circom_witnessgen/witness + +import ../../types +import ../../../stores +import ../../../contracts + +import ./converters + +export converters + +const DefaultCurve* = "bn128" + +type + NimGroth16Backend* = object + curve: string # curve name + slotDepth: int # max depth of the slot tree + datasetDepth: int # max depth of dataset tree + blkDepth: int # depth of the block merkle tree (pow2 for now) + cellElms: int # number of field elements per cell + numSamples: int # number of samples per slot + r1cs: R1CS # path to the r1cs file + zkey: ZKey # path to the zkey file + graph*: Graph # path to the graph file generated with circom-witnesscalc + tp: Taskpool # taskpool for async operations + + NimGroth16BackendRef* = ref NimGroth16Backend + + ProofTask* = object + proof: Isolated[Proof] + self: ptr NimGroth16Backend + inputs: Inputs + signal: ThreadSignalPtr + ok: Atomic[bool] + +proc release*(self: NimGroth16BackendRef) = + ## Release the ctx + ## + + discard + +proc normalizeInput[SomeHash]( + self: NimGroth16BackendRef, input: ProofInputs[SomeHash] +): Inputs = + ## Map inputs to witnessgen inputs + ## + + var normSlotProof = input.slotProof + normSlotProof.setLen(self.datasetDepth) + + { + "slotDepth": @[self.slotDepth.toF], + "datasetDepth": @[self.datasetDepth.toF], + "blkDepth": @[self.blkDepth.toF], + "cellElms": @[self.cellElms.toF], + "numSamples": @[self.numSamples.toF], + "entropy": @[input.entropy], + "dataSetRoot": @[input.datasetRoot], + "slotIndex": @[input.slotIndex.toF], + "slotRoot": @[input.slotRoot], + "nCellsPerSlot": @[input.nCellsPerSlot.toF], + "nSlotsPerDataSet": @[input.nSlotsPerDataSet.toF], + "slotProof": normSlotProof, + "cellData": input.samples.mapIt(it.cellData).concat, + "merklePaths": input.samples.mapIt( + block: + var mekrlePaths = it.merklePaths + mekrlePaths.setLen(self.slotDepth) + mekrlePaths + ).concat, + }.toTable + +proc generateProofTask(task: ptr ProofTask) = + defer: + if task[].signal != nil: + discard task[].signal.fireSync() + + try: + trace "Generating witness" + let + witnessValues = generateWitness(task[].self[].graph, task[].inputs) + witness = Witness( + curve: task[].self[].curve, + r: task[].self[].r1cs.r, + nvars: task[].self[].r1cs.cfg.nWires, + values: witnessValues, + ) + + trace "Generating nim groth16 proof" + var proof = generateProof(task[].self[].zkey, witness, task[].self[].tp) + trace "Proof generated, copying to main thread" + var isolatedProof = isolate(proof) + task[].proof = move isolatedProof + task[].ok.store true + except CatchableError as e: + error "Failed to generate proof", err = e.msg + task[].ok.store false + +proc prove*[SomeHash]( + self: NimGroth16BackendRef, input: ProofInputs[SomeHash] +): Future[?!NimGroth16Proof] {.async: (raises: [CancelledError]).} = + ## Prove a statement using backend. + ## + + var + signalPtr = ?ThreadSignalPtr.new().mapFailure + task = ProofTask( + self: cast[ptr NimGroth16Backend](self), + signal: signalPtr, + inputs: self.normalizeInput(input), + ) + + defer: + if signalPtr != nil: + ?signalPtr.close().mapFailure + signalPtr = nil + + self.tp.spawn generateProofTask(task.addr) + + let taskFut = signalPtr.wait() + if err =? catch(await taskFut.join()).errorOption: + # XXX: we need this because there is no way to cancel a task + # and without waiting for it to finish, we'll be writting to free'd + # memory in the task + warn "Error while generating proof, awaiting task to finish", err = err.msg + ?catch(await noCancel taskFut) + if err of CancelledError: # reraise cancelled error + trace "Task was cancelled" + raise (ref CancelledError) err + + trace "Task failed with error", err = err.msg + return failure err + + defer: + task.proof = default(Isolated[Proof]) + + if not task.ok.load: + trace "Task failed, no proof generated" + return failure("Failed to generate proof") + + var proof = task.proof.extract + trace "Task finished successfully, proof generated" + success proof + +proc verify*( + self: NimGroth16BackendRef, proof: NimGroth16Proof +): Future[?!bool] {.async: (raises: [CancelledError]).} = + let + vKey = self.zkey.extractVKey + verified = ?verifyProof(vKey, proof).catch + + success verified + +proc new*( + _: type NimGroth16BackendRef, + graphPath: string, + r1csPath: string, + zkeyPath: string, + curve = DefaultCurve, + slotDepth = DefaultMaxSlotDepth, + datasetDepth = DefaultMaxDatasetDepth, + blkDepth = DefaultBlockDepth, + cellElms = DefaultCellElms, + numSamples = DefaultSamplesNum, + tp: Taskpool, +): ?!NimGroth16BackendRef = + ## Create a new ctx + ## + + let + graph = ?loadGraph(graphPath).catch + r1cs = ?parseR1CS(r1csPath).catch + zkey = ?parseZKey(zkeyPath).catch + + success NimGroth16BackendRef( + graph: graph, + r1cs: r1cs, + zkey: zkey, + slotDepth: slotDepth, + datasetDepth: datasetDepth, + blkDepth: blkDepth, + cellElms: cellElms, + numSamples: numSamples, + curve: curve, + tp: tp, + ) diff --git a/codex/slots/proofs/backendutils.nim b/codex/slots/proofs/backendutils.nim deleted file mode 100644 index 0e334aced..000000000 --- a/codex/slots/proofs/backendutils.nim +++ /dev/null @@ -1,8 +0,0 @@ -import ./backends - -type BackendUtils* = ref object of RootObj - -method initializeCircomBackend*( - self: BackendUtils, r1csFile: string, wasmFile: string, zKeyFile: string -): AnyBackend {.base, gcsafe.} = - CircomCompat.init(r1csFile, wasmFile, zKeyFile) diff --git a/codex/slots/proofs/prover.nim b/codex/slots/proofs/prover.nim index 1afcd0684..b49ab0218 100644 --- a/codex/slots/proofs/prover.nim +++ b/codex/slots/proofs/prover.nim @@ -12,6 +12,7 @@ import pkg/chronos import pkg/chronicles import pkg/circomcompat import pkg/poseidon2 +import pkg/taskpools import pkg/questionable/results import pkg/libp2p/cid @@ -34,60 +35,71 @@ export backends logScope: topics = "codex prover" -type - AnyProof* = CircomProof - - AnySampler* = Poseidon2Sampler - # add any other generic type here, eg. Poseidon2Sampler | ReinforceConcreteSampler - AnyBuilder* = Poseidon2Builder - # add any other generic type here, eg. Poseidon2Builder | ReinforceConcreteBuilder - - AnyProofInputs* = ProofInputs[Poseidon2Hash] - Prover* = ref object of RootObj - backend: AnyBackend - store: BlockStore - nSamples: int - -proc prove*( - self: Prover, slotIdx: int, manifest: Manifest, challenge: ProofChallenge -): Future[?!(AnyProofInputs, AnyProof)] {.async: (raises: [CancelledError]).} = +type Prover* = ref object + case backendKind: ProverBackendCmd + of ProverBackendCmd.nimgroth16: + groth16Backend*: NimGroth16BackendRef + of ProverBackendCmd.circomcompat: + circomCompatBackend*: CircomCompatBackendRef + nSamples: int + tp: Taskpool + +proc prove*[SomeSampler]( + self: Prover, + sampler: SomeSampler, + manifest: Manifest, + challenge: ProofChallenge, + verify = false, +): Future[?!(Groth16Proof, ?bool)] {.async: (raises: [CancelledError]).} = ## Prove a statement using backend. ## Returns a future that resolves to a proof. logScope: cid = manifest.treeCid - slot = slotIdx challenge = challenge trace "Received proof challenge" - without builder =? AnyBuilder.new(self.store, manifest), err: - error "Unable to create slots builder", err = err.msg - return failure(err) - - without sampler =? AnySampler.new(slotIdx, self.store, builder), err: - error "Unable to create data sampler", err = err.msg - return failure(err) - - without proofInput =? await sampler.getProofInput(challenge, self.nSamples), err: - error "Unable to get proof input for slot", err = err.msg - return failure(err) - - # prove slot - without proof =? self.backend.prove(proofInput), err: - error "Unable to prove slot", err = err.msg - return failure(err) + let + proofInput = ?await sampler.getProofInput(challenge, self.nSamples) + # prove slot + + case self.backendKind + of ProverBackendCmd.nimgroth16: + let + proof = ?await self.groth16Backend.prove(proofInput) + verified = + if verify: + (?await self.groth16Backend.verify(proof)).some + else: + bool.none + return success (proof.toGroth16Proof, verified) + of ProverBackendCmd.circomcompat: + let + proof = ?await self.circomCompatBackend.prove(proofInput) + verified = + if verify: + (?await self.circomCompatBackend.verify(proof, proofInput)).some + else: + bool.none + return success (proof.toGroth16Proof, verified) - success (proofInput, proof) - -proc verify*( - self: Prover, proof: AnyProof, inputs: AnyProofInputs -): Future[?!bool] {.async: (raises: [CancelledError]).} = - ## Prove a statement using backend. - ## Returns a future that resolves to a proof. - self.backend.verify(proof, inputs) +proc new*( + _: type Prover, backend: CircomCompatBackendRef, nSamples: int, tp: Taskpool +): Prover = + Prover( + circomCompatBackend: backend, + backendKind: ProverBackendCmd.circomcompat, + nSamples: nSamples, + tp: tp, + ) proc new*( - _: type Prover, store: BlockStore, backend: AnyBackend, nSamples: int + _: type Prover, backend: NimGroth16BackendRef, nSamples: int, tp: Taskpool ): Prover = - Prover(store: store, backend: backend, nSamples: nSamples) + Prover( + groth16Backend: backend, + backendKind: ProverBackendCmd.nimgroth16, + nSamples: nSamples, + tp: tp, + ) diff --git a/codex/slots/proofs/proverfactory.nim b/codex/slots/proofs/proverfactory.nim new file mode 100644 index 000000000..419b0fa50 --- /dev/null +++ b/codex/slots/proofs/proverfactory.nim @@ -0,0 +1,145 @@ +{.push raises: [].} + +import os +import strutils +import pkg/chronos +import pkg/chronicles +import pkg/questionable +import pkg/confutils/defs +import pkg/stew/io2 +import pkg/ethers +import pkg/taskpools + +import ../../conf +import ./backends +import ./prover + +logScope: + topics = "codex slots proverfactory" + +template graphFilePath(config: CodexConf): string = + config.circuitDir / "proof_main.bin" + +template r1csFilePath(config: CodexConf): string = + config.circuitDir / "proof_main.r1cs" + +template wasmFilePath(config: CodexConf): string = + config.circuitDir / "proof_main.wasm" + +template zkeyFilePath(config: CodexConf): string = + config.circuitDir / "proof_main.zkey" + +proc getGraphFile*(config: CodexConf): ?!string = + if fileAccessible($config.circomGraph, {AccessFlags.Read}) and + endsWith($config.circomGraph, ".bin"): + success $config.circomGraph + elif fileAccessible(config.graphFilePath, {AccessFlags.Read}) and + endsWith(config.graphFilePath, ".bin"): + success config.graphFilePath + else: + failure("Graph file not accessible or not found") + +proc getR1csFile*(config: CodexConf): ?!string = + if fileAccessible($config.circomR1cs, {AccessFlags.Read}) and + endsWith($config.circomR1cs, ".r1cs"): + success $config.circomR1cs + elif fileAccessible(config.r1csFilePath, {AccessFlags.Read}) and + endsWith(config.r1csFilePath, ".r1cs"): + success config.r1csFilePath + else: + failure("R1CS file not accessible or not found") + +proc getWasmFile*(config: CodexConf): ?!string = + if fileAccessible($config.circomWasm, {AccessFlags.Read}) and + endsWith($config.circomWasm, ".wasm"): + success $config.circomWasm + elif fileAccessible(config.wasmFilePath, {AccessFlags.Read}) and + endsWith(config.wasmFilePath, ".wasm"): + success config.wasmFilePath + else: + failure("WASM file not accessible or not found") + +proc getZkeyFile*(config: CodexConf): ?!string = + if fileAccessible($config.circomZkey, {AccessFlags.Read}) and + endsWith($config.circomZkey, ".zkey"): + success $config.circomZkey + elif fileAccessible(config.zkeyFilePath, {AccessFlags.Read}) and + endsWith(config.zkeyFilePath, ".zkey"): + success config.zkeyFilePath + else: + failure("ZKey file not accessible or not found") + +proc suggestDownloadTool(config: CodexConf) = + without address =? config.marketplaceAddress: + raiseAssert("Proving backend initializing while marketplace address not set.") + + let + tokens = ["cirdl", "\"" & $config.circuitDir & "\"", config.ethProvider, $address] + instructions = "'./" & tokens.join(" ") & "'" + + warn "Proving circuit files are not found. Please run the following to download them:", + instructions + +proc initializeNimGroth16Backend( + config: CodexConf, tp: Taskpool +): ?!NimGroth16BackendRef = + trace "Initializing NimGroth16 backend" + + let + graphFile = ?getGraphFile(config) + r1csFile = ?getR1csFile(config) + zkeyFile = ?getZkeyFile(config) + + return NimGroth16BackendRef.new( + $graphFile, + $r1csFile, + $zkeyFile, + $config.curve, + config.maxSlotDepth, + config.maxDatasetDepth, + config.maxBlockDepth, + config.maxCellElms, + config.numProofSamples, + tp, + ) + +proc initializeCircomCompatBackend( + config: CodexConf, tp: Taskpool +): ?!CircomCompatBackendRef = + trace "Initializing CircomCompat backend" + + let + r1csFile = ?getR1csFile(config) + wasmFile = ?getWasmFile(config) + zkeyFile = ?getZkeyFile(config) + + return CircomCompatBackendRef.new( + $r1csFile, + $wasmFile, + $zkeyFile, + config.maxSlotDepth, + config.maxDatasetDepth, + config.maxBlockDepth, + config.maxCellElms, + config.numProofSamples, + ) + +proc initializeProver*(config: CodexConf, tp: Taskpool): ?!Prover = + let prover = + case config.proverBackend + of ProverBackendCmd.nimgroth16: + without backend =? initializeNimGroth16Backend(config, tp), err: + trace "Unable to initialize NimGroth16 backend: ", err = err.msg + suggestDownloadTool(config) + return failure(err) + + Prover.new(backend, config.numProofSamples, tp) + of ProverBackendCmd.circomcompat: + without backend =? initializeCircomCompatBackend(config, tp), err: + trace "Unable to initialize CircomCompat backend: ", err = err.msg + suggestDownloadTool(config) + return failure(err) + + Prover.new(backend, config.numProofSamples, tp) + + success prover diff --git a/codex/slots/sampler/sampler.nim b/codex/slots/sampler/sampler.nim index 6ea41ee33..740f5950e 100644 --- a/codex/slots/sampler/sampler.nim +++ b/codex/slots/sampler/sampler.nim @@ -29,14 +29,14 @@ import ./utils logScope: topics = "codex datasampler" -type DataSampler*[T, H] = ref object of RootObj +type DataSampler*[SomeTree, SomeHash] = ref object of RootObj index: Natural blockStore: BlockStore - builder: SlotsBuilder[T, H] + builder: SlotsBuilder[SomeTree, SomeHash] -func getCell*[T, H]( - self: DataSampler[T, H], blkBytes: seq[byte], blkCellIdx: Natural -): seq[H] = +func getCell*[SomeTree, SomeHash]( + self: DataSampler[SomeTree, SomeHash], blkBytes: seq[byte], blkCellIdx: Natural +): seq[SomeHash] = let cellSize = self.builder.cellSize.uint64 dataStart = cellSize * blkCellIdx.uint64 @@ -44,11 +44,14 @@ func getCell*[T, H]( doAssert (dataEnd - dataStart) == cellSize, "Invalid cell size" - blkBytes[dataStart ..< dataEnd].elements(H).toSeq() + blkBytes[dataStart ..< dataEnd].elements(SomeHash).toSeq() -proc getSample*[T, H]( - self: DataSampler[T, H], cellIdx: int, slotTreeCid: Cid, slotRoot: H -): Future[?!Sample[H]] {.async: (raises: [CancelledError]).} = +proc getSample*[SomeTree, SomeHash]( + self: DataSampler[SomeTree, SomeHash], + cellIdx: int, + slotTreeCid: Cid, + slotRoot: SomeHash, +): Future[?!Sample[SomeHash]] {.async: (raises: [CancelledError]).} = let cellsPerBlock = self.builder.numBlockCells blkCellIdx = cellIdx.toCellInBlk(cellsPerBlock) # block cell index @@ -77,27 +80,22 @@ proc getSample*[T, H]( cellProof = blkTree.getProof(blkCellIdx).valueOr: return failure("Failed to get proof from block tree") - success Sample[H](cellData: cellData, merklePaths: (cellProof.path & slotProof.path)) + success Sample[SomeHash]( + cellData: cellData, merklePaths: (cellProof.path & slotProof.path) + ) -proc getProofInput*[T, H]( - self: DataSampler[T, H], entropy: ProofChallenge, nSamples: Natural -): Future[?!ProofInputs[H]] {.async: (raises: [CancelledError]).} = +proc getProofInput*[SomeTree, SomeHash]( + self: DataSampler[SomeTree, SomeHash], entropy: ProofChallenge, nSamples: Natural +): Future[?!ProofInputs[SomeHash]] {.async: (raises: [CancelledError]).} = ## Generate proofs as input to the proving circuit. ## let - entropy = H.fromBytes(array[31, byte].initCopyFrom(entropy[0 .. 30])) - # truncate to 31 bytes, otherwise it _might_ be greater than mod - - verifyTree = self.builder.verifyTree.toFailure.valueOr: - return failure("Failed to get verify tree") - - slotProof = verifyTree.getProof(self.index).valueOr: - return failure("Failed to get slot proof") - - datasetRoot = verifyTree.root().valueOr: - return failure("Failed to get dataset root") - + # truncate to 31 bytes, otherwise it _might_ be greater than mod + entropy = SomeHash.fromBytes(array[31, byte].initCopyFrom(entropy[0 .. 30])) + verifyTree = ?self.builder.verifyTree.toFailure + slotProof = ?verifyTree.getProof(self.index) + datasetRoot = ?verifyTree.root() slotTreeCid = self.builder.manifest.slotRoots[self.index] slotRoot = self.builder.slotRoots[self.index] cellIdxs = entropy.cellIndices(slotRoot, self.builder.numSlotCells, nSamples) @@ -108,10 +106,9 @@ proc getProofInput*[T, H]( trace "Collecting input for proof" let samples = collect(newSeq): for cellIdx in cellIdxs: - (await self.getSample(cellIdx, slotTreeCid, slotRoot)).valueOr: - return failure("Failed to get sample") + ?(await self.getSample(cellIdx, slotTreeCid, slotRoot)) - success ProofInputs[H]( + success ProofInputs[SomeHash]( entropy: entropy, datasetRoot: datasetRoot, slotProof: slotProof.path, @@ -122,12 +119,12 @@ proc getProofInput*[T, H]( samples: samples, ) -proc new*[T, H]( - _: type DataSampler[T, H], +proc new*[SomeTree, SomeHash]( + _: type DataSampler[SomeTree, SomeHash], index: Natural, blockStore: BlockStore, - builder: SlotsBuilder[T, H], -): ?!DataSampler[T, H] = + builder: SlotsBuilder[SomeTree, SomeHash], +): ?!DataSampler[SomeTree, SomeHash] = if index > builder.slotRoots.high: error "Slot index is out of range" return failure("Slot index is out of range") @@ -135,4 +132,6 @@ proc new*[T, H]( if not builder.verifiable: return failure("Cannot instantiate DataSampler for non-verifiable builder") - success DataSampler[T, H](index: index, blockStore: blockStore, builder: builder) + success DataSampler[SomeTree, SomeHash]( + index: index, blockStore: blockStore, builder: builder + ) diff --git a/codex/slots/types.nim b/codex/slots/types.nim index 0cd243261..aabba0caa 100644 --- a/codex/slots/types.nim +++ b/codex/slots/types.nim @@ -8,23 +8,23 @@ ## those terms. type - Sample*[H] = object - cellData*: seq[H] - merklePaths*: seq[H] + Sample*[SomeHash] = object + cellData*: seq[SomeHash] + merklePaths*: seq[SomeHash] - PublicInputs*[H] = object + PublicInputs*[SomeHash] = object slotIndex*: int - datasetRoot*: H - entropy*: H + datasetRoot*: SomeHash + entropy*: SomeHash - ProofInputs*[H] = object - entropy*: H - datasetRoot*: H + ProofInputs*[SomeHash] = object + entropy*: SomeHash + datasetRoot*: SomeHash slotIndex*: Natural - slotRoot*: H + slotRoot*: SomeHash nCellsPerSlot*: Natural nSlotsPerDataSet*: Natural - slotProof*: seq[H] + slotProof*: seq[SomeHash] # inclusion proof that shows that the slot root (leaf) is part of the dataset (root) - samples*: seq[Sample[H]] + samples*: seq[Sample[SomeHash]] # inclusion proofs which show that the selected cells (leafs) are part of the slot (roots) diff --git a/codex/stores/treehelper.nim b/codex/stores/treehelper.nim index e1f5d48d8..99f8cde1b 100644 --- a/codex/stores/treehelper.nim +++ b/codex/stores/treehelper.nim @@ -25,7 +25,7 @@ import ../merkletree proc putSomeProofs*( store: BlockStore, tree: CodexTree, iter: Iter[int] -): Future[?!void] {.async.} = +): Future[?!void] {.async: (raises: [CancelledError]).} = without treeCid =? tree.rootCid, err: return failure(err) @@ -51,8 +51,10 @@ proc putSomeProofs*( proc putSomeProofs*( store: BlockStore, tree: CodexTree, iter: Iter[Natural] -): Future[?!void] = +): Future[?!void] {.async: (raises: [CancelledError], raw: true).} = store.putSomeProofs(tree, iter.map((i: Natural) => i.ord)) -proc putAllProofs*(store: BlockStore, tree: CodexTree): Future[?!void] = +proc putAllProofs*( + store: BlockStore, tree: CodexTree +): Future[?!void] {.async: (raises: [CancelledError], raw: true).} = store.putSomeProofs(tree, Iter[int].new(0 ..< tree.leavesCount)) diff --git a/codex/utils/arrayutils.nim b/codex/utils/arrayutils.nim index e36a0cb34..c6721f6bb 100644 --- a/codex/utils/arrayutils.nim +++ b/codex/utils/arrayutils.nim @@ -1,5 +1,3 @@ -import std/sequtils - proc createDoubleArray*( outerLen, innerLen: int ): ptr UncheckedArray[ptr UncheckedArray[byte]] = diff --git a/tests/asynctest.nim b/tests/asynctest.nim index 4db8277fd..6f9d2d429 100644 --- a/tests/asynctest.nim +++ b/tests/asynctest.nim @@ -1,3 +1,13 @@ import pkg/asynctest/chronos/unittest2 -export unittest2 +export unittest2 except eventually + +template eventuallySafe*( + expression: untyped, timeout = 5000, pollInterval = 1000 +): bool = + ## More sane defaults, for use with HTTP connections + eventually(expression, timeout, pollInterval) + +template eventually*(expression: untyped, timeout = 5000, pollInterval = 10): bool = + ## Fast defaults, do not use with HTTP connections! + eventually(expression, timeout, pollInterval) diff --git a/tests/circuits/fixtures/proof_main.bin b/tests/circuits/fixtures/proof_main.bin new file mode 100644 index 000000000..6820a11b2 Binary files /dev/null and b/tests/circuits/fixtures/proof_main.bin differ diff --git a/tests/codex/sales/states/testpreparing.nim b/tests/codex/sales/states/testpreparing.nim index 802489a13..747544117 100644 --- a/tests/codex/sales/states/testpreparing.nim +++ b/tests/codex/sales/states/testpreparing.nim @@ -72,6 +72,12 @@ asyncchecksuite "sales state 'preparing'": let next = state.onSlotFilled(request.id, slotIndex) check !next of SaleFilled + test "run switches to errored when the request cannot be retrieved": + agent = newSalesAgent(context, request.id, slotIndex, StorageRequest.none) + let next = !(await state.run(agent)) + check next of SaleErrored + check SaleErrored(next).error.msg == "request could not be retrieved" + proc createAvailability(enabled = true) {.async.} = let a = await reservations.createAvailability( availability.totalSize, diff --git a/tests/codex/sales/states/testunknown.nim b/tests/codex/sales/states/testunknown.nim index 98b23224a..4806122f7 100644 --- a/tests/codex/sales/states/testunknown.nim +++ b/tests/codex/sales/states/testunknown.nim @@ -20,15 +20,22 @@ suite "sales state 'unknown'": let slotId = slotId(request.id, slotIndex) var market: MockMarket + var context: SalesContext var agent: SalesAgent var state: SaleUnknown setup: market = MockMarket.new() - let context = SalesContext(market: market) - agent = newSalesAgent(context, request.id, slotIndex, StorageRequest.none) + context = SalesContext(market: market) + agent = newSalesAgent(context, request.id, slotIndex, request.some) state = SaleUnknown.new() + test "switches to error state when the request cannot be retrieved": + agent = newSalesAgent(context, request.id, slotIndex, StorageRequest.none) + let next = await state.run(agent) + check !next of SaleErrored + check SaleErrored(!next).error.msg == "request could not be retrieved" + test "switches to error state when on chain state cannot be fetched": let next = await state.run(agent) check !next of SaleErrored @@ -37,6 +44,7 @@ suite "sales state 'unknown'": market.slotState[slotId] = SlotState.Free let next = await state.run(agent) check !next of SaleErrored + check SaleErrored(!next).error.msg == "Slot state on chain should not be 'free'" test "switches to filled state when on chain state is 'filled'": market.slotState[slotId] = SlotState.Filled diff --git a/tests/codex/slots/backends/helpers.nim b/tests/codex/slots/backends/helpers.nim index e1b6822a9..fe7f5c96d 100644 --- a/tests/codex/slots/backends/helpers.nim +++ b/tests/codex/slots/backends/helpers.nim @@ -19,13 +19,13 @@ func toJsonDecimal*(big: BigInt[254]): string = let s = big.toDecimal.strip(leading = true, trailing = false, chars = {'0'}) if s.len == 0: "0" else: s -func toJson*(g1: CircomG1): JsonNode = +func toJson*(g1: CircomCompatG1): JsonNode = %*{ "x": Bn254Fr.fromBytes(g1.x).get.toBig.toJsonDecimal, "y": Bn254Fr.fromBytes(g1.y).get.toBig.toJsonDecimal, } -func toJson*(g2: CircomG2): JsonNode = +func toJson*(g2: CircomCompatG2): JsonNode = %*{ "x": [ Bn254Fr.fromBytes(g2.x[0]).get.toBig.toJsonDecimal, @@ -38,8 +38,9 @@ func toJson*(g2: CircomG2): JsonNode = } proc toJson*(vpk: VerifyingKey): JsonNode = - let ic = - toSeq(cast[ptr UncheckedArray[CircomG1]](vpk.ic).toOpenArray(0, vpk.icLen.int - 1)) + let ic = toSeq( + cast[ptr UncheckedArray[CircomCompatG1]](vpk.ic).toOpenArray(0, vpk.icLen.int - 1) + ) echo ic.len %*{ diff --git a/tests/codex/slots/backends/testcircomcompat.nim b/tests/codex/slots/backends/testcircomcompat.nim index b61d4f188..91c04a66c 100644 --- a/tests/codex/slots/backends/testcircomcompat.nim +++ b/tests/codex/slots/backends/testcircomcompat.nim @@ -24,7 +24,7 @@ suite "Test Circom Compat Backend - control inputs": zkey = "tests/circuits/fixtures/proof_main.zkey" var - circom: CircomCompat + circom: CircomCompatBackendRef proofInputs: ProofInputs[Poseidon2Hash] setup: @@ -33,22 +33,20 @@ suite "Test Circom Compat Backend - control inputs": inputJson = !JsonNode.parse(inputData) proofInputs = Poseidon2Hash.jsonToProofInput(inputJson) - circom = CircomCompat.init(r1cs, wasm, zkey) + circom = CircomCompatBackendRef.new(r1cs, wasm, zkey).tryGet teardown: circom.release() # this comes from the rust FFI test "Should verify with correct inputs": - let proof = circom.prove(proofInputs).tryGet - - check circom.verify(proof, proofInputs).tryGet + let proof = (await circom.prove(proofInputs)).tryGet + check (await circom.verify(proof, proofInputs)).tryGet test "Should not verify with incorrect inputs": proofInputs.slotIndex = 1 # change slot index - let proof = circom.prove(proofInputs).tryGet - - check circom.verify(proof, proofInputs).tryGet == false + let proof = (await circom.prove(proofInputs)).tryGet + check (await circom.verify(proof, proofInputs)).tryGet == false suite "Test Circom Compat Backend": let @@ -72,7 +70,7 @@ suite "Test Circom Compat Backend": manifest: Manifest protected: Manifest verifiable: Manifest - circom: CircomCompat + circom: CircomCompatBackendRef proofInputs: ProofInputs[Poseidon2Hash] challenge: array[32, byte] builder: Poseidon2Builder @@ -92,7 +90,7 @@ suite "Test Circom Compat Backend": builder = Poseidon2Builder.new(store, verifiable).tryGet sampler = Poseidon2Sampler.new(slotId, store, builder).tryGet - circom = CircomCompat.init(r1cs, wasm, zkey) + circom = CircomCompatBackendRef.new(r1cs, wasm, zkey).tryGet challenge = 1234567.toF.toBytes.toArray32 proofInputs = (await sampler.getProofInput(challenge, samples)).tryGet @@ -103,13 +101,11 @@ suite "Test Circom Compat Backend": await metaTmp.destroyDb() test "Should verify with correct input": - var proof = circom.prove(proofInputs).tryGet - - check circom.verify(proof, proofInputs).tryGet + var proof = (await circom.prove(proofInputs)).tryGet + check (await circom.verify(proof, proofInputs)).tryGet test "Should not verify with incorrect input": proofInputs.slotIndex = 1 # change slot index - let proof = circom.prove(proofInputs).tryGet - - check circom.verify(proof, proofInputs).tryGet == false + let proof = (await circom.prove(proofInputs)).tryGet + check (await circom.verify(proof, proofInputs)).tryGet == false diff --git a/tests/codex/slots/backends/testnimgroth16.nim b/tests/codex/slots/backends/testnimgroth16.nim new file mode 100644 index 000000000..a7156fcc8 --- /dev/null +++ b/tests/codex/slots/backends/testnimgroth16.nim @@ -0,0 +1,119 @@ +import std/options +import std/isolation + +import ../../../asynctest + +import pkg/chronos +import pkg/poseidon2 +import pkg/serde/json +import pkg/taskpools + +import pkg/codex/slots {.all.} +import pkg/codex/slots/types {.all.} +import pkg/codex/merkletree +import pkg/codex/merkletree/poseidon2 +import pkg/codex/codextypes +import pkg/codex/manifest +import pkg/codex/stores + +import pkg/groth16 +import pkg/nim/circom_witnessgen +import pkg/nim/circom_witnessgen/load +import pkg/nim/circom_witnessgen/witness + +import ./helpers +import ../helpers +import ../../helpers + +suite "Test NimGoth16 Backend - control inputs": + let + graph = "tests/circuits/fixtures/proof_main.bin" + r1cs = "tests/circuits/fixtures/proof_main.r1cs" + zkey = "tests/circuits/fixtures/proof_main.zkey" + + var + nimGroth16: NimGroth16BackendRef + proofInputs: ProofInputs[Poseidon2Hash] + + setup: + let + inputData = readFile("tests/circuits/fixtures/input.json") + inputJson = !JsonNode.parse(inputData) + + proofInputs = Poseidon2Hash.jsonToProofInput(inputJson) + nimGroth16 = NimGroth16BackendRef.new(graph, r1cs, zkey, tp = Taskpool.new()).tryGet + + teardown: + nimGroth16.release() + + test "Should verify with correct inputs": + let proof = (await nimGroth16.prove(proofInputs)).tryGet + check (await nimGroth16.verify(proof)).tryGet + + test "Should not verify with incorrect inputs": + proofInputs.slotIndex = 1 # change slot index + + let proof = (await nimGroth16.prove(proofInputs)).tryGet + check (await nimGroth16.verify(proof)).tryGet == false + +suite "Test NimGoth16 Backend": + let + ecK = 2 + ecM = 2 + slotId = 3 + samples = 5 + numDatasetBlocks = 8 + blockSize = DefaultBlockSize + cellSize = DefaultCellSize + + graph = "tests/circuits/fixtures/proof_main.bin" + r1cs = "tests/circuits/fixtures/proof_main.r1cs" + zkey = "tests/circuits/fixtures/proof_main.zkey" + + repoTmp = TempLevelDb.new() + metaTmp = TempLevelDb.new() + + var + store: BlockStore + manifest: Manifest + protected: Manifest + verifiable: Manifest + nimGroth16: NimGroth16BackendRef + proofInputs: ProofInputs[Poseidon2Hash] + challenge: array[32, byte] + builder: Poseidon2Builder + sampler: Poseidon2Sampler + + setup: + let + repoDs = repoTmp.newDb() + metaDs = metaTmp.newDb() + + store = RepoStore.new(repoDs, metaDs) + + (manifest, protected, verifiable) = await createVerifiableManifest( + store, numDatasetBlocks, ecK, ecM, blockSize, cellSize + ) + + builder = Poseidon2Builder.new(store, verifiable).tryGet + sampler = Poseidon2Sampler.new(slotId, store, builder).tryGet + + nimGroth16 = NimGroth16BackendRef.new(graph, r1cs, zkey, tp = Taskpool.new()).tryGet + challenge = 1234567.toF.toBytes.toArray32 + + proofInputs = (await sampler.getProofInput(challenge, samples)).tryGet + + teardown: + nimGroth16.release() + await repoTmp.destroyDb() + await metaTmp.destroyDb() + + test "Should verify with correct input": + var proof = (await nimGroth16.prove(proofInputs)).tryGet + check (await nimGroth16.verify(proof)).tryGet + + test "Should not verify with incorrect input": + proofInputs.slotIndex = 1 # change slot index + + let proof = (await nimGroth16.prove(proofInputs)).tryGet + check (await nimGroth16.verify(proof)).tryGet == false diff --git a/tests/codex/slots/helpers.nim b/tests/codex/slots/helpers.nim index fced1f1c4..9394fd7c1 100644 --- a/tests/codex/slots/helpers.nim +++ b/tests/codex/slots/helpers.nim @@ -6,6 +6,7 @@ import pkg/libp2p/cid import pkg/codex/codextypes import pkg/codex/stores import pkg/codex/merkletree +import pkg/codex/utils/poseidon2digest import pkg/codex/manifest import pkg/codex/blocktype as bt import pkg/codex/chunker diff --git a/tests/codex/slots/sampler/testutils.nim b/tests/codex/slots/sampler/testutils.nim index f20b5efc4..5460fde76 100644 --- a/tests/codex/slots/sampler/testutils.nim +++ b/tests/codex/slots/sampler/testutils.nim @@ -77,15 +77,13 @@ asyncchecksuite "Test proof sampler utils": ) proc getExpectedIndices(n: int): seq[Natural] = - return collect( - newSeq, + return collect(newSeq): (; for i in 1 .. n: cellIndex( proofInput.entropy, proofInput.slotRoot, proofInput.nCellsPerSlot, i ) - ), - ) + ) check: slotCellIndices(3) == getExpectedIndices(3) diff --git a/tests/codex/slots/testbackendfactory.nim b/tests/codex/slots/testbackendfactory.nim deleted file mode 100644 index a24bc41a5..000000000 --- a/tests/codex/slots/testbackendfactory.nim +++ /dev/null @@ -1,97 +0,0 @@ -import os -import ../../asynctest - -import pkg/chronos -import pkg/confutils/defs -import pkg/codex/conf -import pkg/codex/slots/proofs/backends -import pkg/codex/slots/proofs/backendfactory -import pkg/codex/slots/proofs/backendutils -import pkg/codex/utils/natutils - -import ../helpers -import ../examples - -type BackendUtilsMock = ref object of BackendUtils - argR1csFile: string - argWasmFile: string - argZKeyFile: string - -method initializeCircomBackend*( - self: BackendUtilsMock, r1csFile: string, wasmFile: string, zKeyFile: string -): AnyBackend = - self.argR1csFile = r1csFile - self.argWasmFile = wasmFile - self.argZKeyFile = zKeyFile - # We return a backend with *something* that's not nil that we can check for. - var - key = VerifyingKey(icLen: 123) - vkpPtr: ptr VerifyingKey = key.addr - return CircomCompat(vkp: vkpPtr) - -suite "Test BackendFactory": - let - utilsMock = BackendUtilsMock() - circuitDir = "testecircuitdir" - - setup: - createDir(circuitDir) - - teardown: - removeDir(circuitDir) - - test "Should create backend from cli config": - let - config = CodexConf( - cmd: StartUpCmd.persistence, - nat: NatConfig(hasExtIp: false, nat: NatNone), - metricsAddress: parseIpAddress("127.0.0.1"), - persistenceCmd: PersistenceCmd.prover, - marketplaceAddress: EthAddress.example.some, - circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), - circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"), - circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey"), - ) - backend = config.initializeBackend(utilsMock).tryGet - - check: - backend.vkp != nil - utilsMock.argR1csFile == $config.circomR1cs - utilsMock.argWasmFile == $config.circomWasm - utilsMock.argZKeyFile == $config.circomZkey - - test "Should create backend from local files": - let - config = CodexConf( - cmd: StartUpCmd.persistence, - nat: NatConfig(hasExtIp: false, nat: NatNone), - metricsAddress: parseIpAddress("127.0.0.1"), - persistenceCmd: PersistenceCmd.prover, - marketplaceAddress: EthAddress.example.some, - - # Set the circuitDir such that the tests/circuits/fixtures/ files - # will be picked up as local files: - circuitDir: OutDir("tests/circuits/fixtures"), - ) - backend = config.initializeBackend(utilsMock).tryGet - - check: - backend.vkp != nil - utilsMock.argR1csFile == config.circuitDir / "proof_main.r1cs" - utilsMock.argWasmFile == config.circuitDir / "proof_main.wasm" - utilsMock.argZKeyFile == config.circuitDir / "proof_main.zkey" - - test "Should suggest usage of downloader tool when files not available": - let - config = CodexConf( - cmd: StartUpCmd.persistence, - nat: NatConfig(hasExtIp: false, nat: NatNone), - metricsAddress: parseIpAddress("127.0.0.1"), - persistenceCmd: PersistenceCmd.prover, - marketplaceAddress: EthAddress.example.some, - circuitDir: OutDir(circuitDir), - ) - backendResult = config.initializeBackend(utilsMock) - - check: - backendResult.isErr diff --git a/tests/codex/slots/testbackends.nim b/tests/codex/slots/testbackends.nim index b9994fcdf..f8f1b4508 100644 --- a/tests/codex/slots/testbackends.nim +++ b/tests/codex/slots/testbackends.nim @@ -1,3 +1,4 @@ import ./backends/testcircomcompat +import ./backends/testnimgroth16 {.warning[UnusedImport]: off.} diff --git a/tests/codex/slots/testprover.nim b/tests/codex/slots/testprover.nim index c567db55d..4d13a9a44 100644 --- a/tests/codex/slots/testprover.nim +++ b/tests/codex/slots/testprover.nim @@ -13,17 +13,19 @@ import pkg/confutils/defs import pkg/poseidon2/io import pkg/codex/utils/poseidon2digest import pkg/codex/nat +import pkg/taskpools import pkg/codex/utils/natutils import ./helpers import ../helpers -suite "Test Prover": +suite "Test CircomCompat Prover": let samples = 5 blockSize = DefaultBlockSize cellSize = DefaultCellSize repoTmp = TempLevelDb.new() metaTmp = TempLevelDb.new() + tp = Taskpool.new() challenge = 1234567.toF.toBytes.toArray32 var @@ -34,55 +36,137 @@ suite "Test Prover": let repoDs = repoTmp.newDb() metaDs = metaTmp.newDb() - config = CodexConf( - cmd: StartUpCmd.persistence, - nat: NatConfig(hasExtIp: false, nat: NatNone), - metricsAddress: parseIpAddress("127.0.0.1"), - persistenceCmd: PersistenceCmd.prover, - circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), - circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"), - circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey"), - numProofSamples: samples, + backend = CircomCompatBackendRef.new( + r1csPath = "tests/circuits/fixtures/proof_main.r1cs", + wasmPath = "tests/circuits/fixtures/proof_main.wasm", + zkeyPath = "tests/circuits/fixtures/proof_main.zkey", + ).tryGet + tp = Taskpool.new() + + store = RepoStore.new(repoDs, metaDs) + prover = Prover.new(backend, samples, tp) + + teardown: + await repoTmp.destroyDb() + await metaTmp.destroyDb() + + test "Should sample and prove a slot": + let + (_, _, verifiable) = await createVerifiableManifest( + store, + 8, # number of blocks in the original dataset (before EC) + 5, # ecK + 3, # ecM + blockSize, + cellSize, + ) + + builder = + Poseidon2Builder.new(store, verifiable, verifiable.verifiableStrategy).tryGet + sampler = Poseidon2Sampler.new(1, store, builder).tryGet + (_, checked) = + (await prover.prove(sampler, verifiable, challenge, verify = true)).tryGet + + check: + checked.isSome and checked.get == true + + test "Should generate valid proofs when slots consist of single blocks": + # To get single-block slots, we just need to set the number of blocks in + # the original dataset to be the same as ecK. The total number of blocks + # after generating random data for parity will be ecK + ecM, which will + # match the number of slots. + let + (_, _, verifiable) = await createVerifiableManifest( + store, + 2, # number of blocks in the original dataset (before EC) + 2, # ecK + 1, # ecM + blockSize, + cellSize, ) - backend = config.initializeBackend().tryGet() + + builder = + Poseidon2Builder.new(store, verifiable, verifiable.verifiableStrategy).tryGet + sampler = Poseidon2Sampler.new(1, store, builder).tryGet + (_, checked) = + (await prover.prove(sampler, verifiable, challenge, verify = true)).tryGet + + check: + checked.isSome and checked.get == true + +suite "Test NimGroth16 Prover": + let + samples = 5 + blockSize = DefaultBlockSize + cellSize = DefaultCellSize + repoTmp = TempLevelDb.new() + metaTmp = TempLevelDb.new() + tp = Taskpool.new() + challenge = 1234567.toF.toBytes.toArray32 + + var + store: BlockStore + prover: Prover + + setup: + let + tp = Taskpool.new() + repoDs = repoTmp.newDb() + metaDs = metaTmp.newDb() + backend = NimGroth16BackendRef.new( + r1csPath = "tests/circuits/fixtures/proof_main.r1cs", + graphPath = "tests/circuits/fixtures/proof_main.bin", + zkeyPath = "tests/circuits/fixtures/proof_main.zkey", + tp = tp, + ).tryGet store = RepoStore.new(repoDs, metaDs) - prover = Prover.new(store, backend, config.numProofSamples) + prover = Prover.new(backend, samples, tp) teardown: await repoTmp.destroyDb() await metaTmp.destroyDb() test "Should sample and prove a slot": - let (_, _, verifiable) = await createVerifiableManifest( - store, - 8, # number of blocks in the original dataset (before EC) - 5, # ecK - 3, # ecM - blockSize, - cellSize, - ) + let + (_, _, verifiable) = await createVerifiableManifest( + store, + 8, # number of blocks in the original dataset (before EC) + 5, # ecK + 3, # ecM + blockSize, + cellSize, + ) - let (inputs, proof) = (await prover.prove(1, verifiable, challenge)).tryGet + builder = + Poseidon2Builder.new(store, verifiable, verifiable.verifiableStrategy).tryGet + sampler = Poseidon2Sampler.new(1, store, builder).tryGet + (_, checked) = + (await prover.prove(sampler, verifiable, challenge, verify = true)).tryGet check: - (await prover.verify(proof, inputs)).tryGet == true + checked.isSome and checked.get == true test "Should generate valid proofs when slots consist of single blocks": # To get single-block slots, we just need to set the number of blocks in # the original dataset to be the same as ecK. The total number of blocks # after generating random data for parity will be ecK + ecM, which will # match the number of slots. - let (_, _, verifiable) = await createVerifiableManifest( - store, - 2, # number of blocks in the original dataset (before EC) - 2, # ecK - 1, # ecM - blockSize, - cellSize, - ) + let + (_, _, verifiable) = await createVerifiableManifest( + store, + 2, # number of blocks in the original dataset (before EC) + 2, # ecK + 1, # ecM + blockSize, + cellSize, + ) - let (inputs, proof) = (await prover.prove(1, verifiable, challenge)).tryGet + builder = + Poseidon2Builder.new(store, verifiable, verifiable.verifiableStrategy).tryGet + sampler = Poseidon2Sampler.new(1, store, builder).tryGet + (_, checked) = + (await prover.prove(sampler, verifiable, challenge, verify = true)).tryGet check: - (await prover.verify(proof, inputs)).tryGet == true + checked.isSome and checked.get == true diff --git a/tests/codex/slots/testproverfactory.nim b/tests/codex/slots/testproverfactory.nim new file mode 100644 index 000000000..e3a3f2113 --- /dev/null +++ b/tests/codex/slots/testproverfactory.nim @@ -0,0 +1,111 @@ +import os +import ../../asynctest + +import pkg/chronos +import pkg/taskpools + +import pkg/confutils/defs +import pkg/codex/conf +import pkg/codex/slots/proofs/backends +import pkg/codex/slots/proofs/proverfactory {.all.} +import pkg/codex/utils/natutils + +import ../helpers +import ../examples + +suite "Test BackendFactory": + let circuitDir = "testecircuitdir" + + setup: + createDir(circuitDir) + + teardown: + removeDir(circuitDir) + + test "Should initialize with correct nimGroth16 config files": + let config = CodexConf( + cmd: StartUpCmd.persistence, + nat: NatConfig(hasExtIp: false, nat: NatNone), + metricsAddress: parseIpAddress("127.0.0.1"), + persistenceCmd: PersistenceCmd.prover, + marketplaceAddress: EthAddress.example.some, + proverBackend: ProverBackendCmd.nimgroth16, + circomGraph: InputFile("tests/circuits/fixtures/proof_main.bin"), + circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), + circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey"), + ) + + check: + getGraphFile(config).tryGet == $config.circomGraph + getR1csFile(config).tryGet == $config.circomR1cs + getZkeyFile(config).tryGet == $config.circomZkey + + test "Should initialize with correct circom compat config files": + let config = CodexConf( + cmd: StartUpCmd.persistence, + nat: NatConfig(hasExtIp: false, nat: NatNone), + metricsAddress: parseIpAddress("127.0.0.1"), + persistenceCmd: PersistenceCmd.prover, + marketplaceAddress: EthAddress.example.some, + proverBackend: ProverBackendCmd.circomcompat, + circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"), + circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), + circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey"), + ) + + check: + getWasmFile(config).tryGet == $config.circomWasm + getR1csFile(config).tryGet == $config.circomR1cs + getZkeyFile(config).tryGet == $config.circomZkey + + test "Should initialize circom compat from local directory": + let config = CodexConf( + cmd: StartUpCmd.persistence, + nat: NatConfig(hasExtIp: false, nat: NatNone), + metricsAddress: parseIpAddress("127.0.0.1"), + persistenceCmd: PersistenceCmd.prover, + marketplaceAddress: EthAddress.example.some, + proverBackend: ProverBackendCmd.circomcompat, + # Set the circuitDir such that the tests/circuits/fixtures/ files + # will be picked up as local files: + circuitDir: OutDir("tests/circuits/fixtures"), + ) + + check: + getR1csFile(config).tryGet == config.circuitDir / "proof_main.r1cs" + getWasmFile(config).tryGet == config.circuitDir / "proof_main.wasm" + getZkeyFile(config).tryGet == config.circuitDir / "proof_main.zkey" + + test "Should initialize nim groth16 from local directory": + let config = CodexConf( + cmd: StartUpCmd.persistence, + nat: NatConfig(hasExtIp: false, nat: NatNone), + metricsAddress: parseIpAddress("127.0.0.1"), + persistenceCmd: PersistenceCmd.prover, + marketplaceAddress: EthAddress.example.some, + proverBackend: ProverBackendCmd.nimgroth16, + # Set the circuitDir such that the tests/circuits/fixtures/ files + # will be picked up as local files: + circuitDir: OutDir("tests/circuits/fixtures"), + ) + + check: + getGraphFile(config).tryGet == config.circuitDir / "proof_main.bin" + getR1csFile(config).tryGet == config.circuitDir / "proof_main.r1cs" + getZkeyFile(config).tryGet == config.circuitDir / "proof_main.zkey" + + test "Should suggest usage of downloader tool when files not available": + let + config = CodexConf( + cmd: StartUpCmd.persistence, + nat: NatConfig(hasExtIp: false, nat: NatNone), + metricsAddress: parseIpAddress("127.0.0.1"), + persistenceCmd: PersistenceCmd.prover, + proverBackend: ProverBackendCmd.nimgroth16, + marketplaceAddress: EthAddress.example.some, + circuitDir: OutDir(circuitDir), + ) + proverResult = config.initializeProver(Taskpool.new()) + + check: + proverResult.isErr diff --git a/tests/codex/testslots.nim b/tests/codex/testslots.nim index 059de7c2f..9c1c9204f 100644 --- a/tests/codex/testslots.nim +++ b/tests/codex/testslots.nim @@ -3,6 +3,6 @@ import ./slots/testsampler import ./slots/testconverters import ./slots/testbackends import ./slots/testprover -import ./slots/testbackendfactory +import ./slots/testproverfactory {.warning[UnusedImport]: off.} diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index a13d0c67e..cf4882ab1 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -537,6 +537,8 @@ ethersuite "On-Chain Market": let (_, fromTime) = await ethProvider.blockNumberAndTimestamp(BlockTag.latest) + await ethProvider.advanceTime(1.u256) + await market.reserveSlot(request.id, 1.uint64) await market.reserveSlot(request.id, 2.uint64) await market.fillSlot(request.id, 1.uint64, proof, request.ask.collateralPerSlot) diff --git a/tests/ethertest.nim b/tests/ethertest.nim index 2cab8bf5b..636760a1a 100644 --- a/tests/ethertest.nim +++ b/tests/ethertest.nim @@ -5,6 +5,8 @@ import pkg/chronos import ./asynctest import ./checktest +const HardhatPort {.intdefine.}: int = 8545 + ## Unit testing suite that sets up an Ethereum testing environment. ## Injects a `ethProvider` instance, and a list of `accounts`. ## Calls the `evm_snapshot` and `evm_revert` methods to ensure that any @@ -16,7 +18,7 @@ template ethersuite*(name, body) = var snapshot: JsonNode setup: - ethProvider = JsonRpcProvider.new("ws://localhost:8545") + ethProvider = JsonRpcProvider.new("ws://localhost:" & $HardhatPort) snapshot = await send(ethProvider, "evm_snapshot") accounts = await ethProvider.listAccounts() teardown: diff --git a/tests/helpers.nim b/tests/helpers.nim index b48b787ed..bcac03c30 100644 --- a/tests/helpers.nim +++ b/tests/helpers.nim @@ -4,6 +4,8 @@ import helpers/templeveldb import std/times import std/sequtils, chronos +import ./asynctest + export multisetup, trackers, templeveldb ### taken from libp2p errorhelpers.nim diff --git a/tests/integration/codexconfig.nim b/tests/integration/codexconfig.nim index 138ae274d..8d0cdb339 100644 --- a/tests/integration/codexconfig.nim +++ b/tests/integration/codexconfig.nim @@ -169,7 +169,8 @@ proc withLogFile*(self: CodexConfigs): CodexConfigs {.raises: [CodexConfigError] proc withLogFile*( self: var CodexConfig, logFile: string -) {.raises: [CodexConfigError].} = #: CodexConfigs = +) {.raises: [CodexConfigError].} = + #: CodexConfigs = ## typically called internally from the test suite, sets a log file path to ## be created during the test run, for a specified node in the group # var config = self diff --git a/tests/integration/codexprocess.nim b/tests/integration/codexprocess.nim index 3eca5b04e..67adfae73 100644 --- a/tests/integration/codexprocess.nim +++ b/tests/integration/codexprocess.nim @@ -7,6 +7,7 @@ import pkg/ethers import pkg/libp2p import std/os import std/strutils +import std/times import codex/conf import ./codexclient import ./nodeprocess @@ -15,11 +16,28 @@ export codexclient export chronicles export nodeprocess +{.push raises: [].} + logScope: topics = "integration testing codex process" -type CodexProcess* = ref object of NodeProcess - client: ?CodexClient +type + CodexProcess* = ref object of NodeProcess + client: ?CodexClient + + CodexProcessError* = object of NodeProcessError + +proc raiseCodexProcessError( + msg: string, parent: ref CatchableError +) {.raises: [CodexProcessError].} = + raise newException(CodexProcessError, msg & ": " & parent.msg, parent) + +template convertError(msg, body: typed) = + # Don't use this in an async proc, unless body does not raise CancelledError + try: + body + except CatchableError as parent: + raiseCodexProcessError(msg, parent) method workingDir(node: CodexProcess): string = return currentSourcePath() / ".." / ".." / ".." @@ -33,43 +51,80 @@ method startedOutput(node: CodexProcess): string = method processOptions(node: CodexProcess): set[AsyncProcessOption] = return {AsyncProcessOption.StdErrToStdOut} -method outputLineEndings(node: CodexProcess): string {.raises: [].} = +method outputLineEndings(node: CodexProcess): string = return "\n" -method onOutputLineCaptured(node: CodexProcess, line: string) {.raises: [].} = +method onOutputLineCaptured(node: CodexProcess, line: string) = discard -proc dataDir(node: CodexProcess): string = - let config = CodexConf.load(cmdLine = node.arguments, quitOnFailure = false) - return config.dataDir.string - -proc ethAccount*(node: CodexProcess): Address = - let config = CodexConf.load(cmdLine = node.arguments, quitOnFailure = false) - without ethAccount =? config.ethAccount: +proc config(node: CodexProcess): CodexConf {.raises: [CodexProcessError].} = + # cannot use convertError here as it uses typed parameters which forces type + # resolution, while confutils.load uses untyped parameters and expects type + # resolution not to happen yet. In other words, it won't compile. + try: + return CodexConf.load( + cmdLine = node.arguments, quitOnFailure = false, secondarySources = nil + ) + except ConfigurationError as parent: + raiseCodexProcessError "Failed to load node arguments into CodexConf", parent + +proc dataDir(node: CodexProcess): string {.raises: [CodexProcessError].} = + return node.config.dataDir.string + +proc ethAccount*(node: CodexProcess): Address {.raises: [CodexProcessError].} = + without ethAccount =? node.config.ethAccount: raiseAssert "eth account not set" return Address(ethAccount) -proc apiUrl*(node: CodexProcess): string = - let config = CodexConf.load(cmdLine = node.arguments, quitOnFailure = false) +proc apiUrl*(node: CodexProcess): string {.raises: [CodexProcessError].} = + let config = node.config return "http://" & config.apiBindAddress & ":" & $config.apiPort & "/api/codex/v1" -proc client*(node: CodexProcess): CodexClient = +proc logFile*(node: CodexProcess): ?string {.raises: [CodexProcessError].} = + node.config.logFile + +proc client*(node: CodexProcess): CodexClient {.raises: [CodexProcessError].} = if client =? node.client: return client let client = CodexClient.new(node.apiUrl) node.client = some client return client -method stop*(node: CodexProcess) {.async.} = +proc updateLogFile(node: CodexProcess, newLogFile: string) = + for arg in node.arguments.mitems: + if arg.startsWith("--log-file="): + arg = "--log-file=" & newLogFile + break + +method restart*(node: CodexProcess) {.async.} = + trace "restarting codex" + await node.stop() + if logFile =? node.logFile: + # chronicles truncates the existing log file on start, so changed the log + # file cli param to create a new one + node.updateLogFile( + logFile & "_restartedAt_" & now().format("yyyy-MM-dd'_'HH-mm-ss") & ".log" + ) + await node.start() + await node.waitUntilStarted() + trace "codex process restarted" + +method stop*(node: CodexProcess) {.async: (raises: []).} = logScope: nodeName = node.name + trace "stopping codex client" await procCall NodeProcess(node).stop() - trace "stopping codex client" + if not node.process.isNil: + trace "closing node process' streams" + await node.process.closeWait() + trace "node process' streams closed" + if client =? node.client: await client.close() node.client = none CodexClient -method removeDataDir*(node: CodexProcess) = - removeDir(node.dataDir) +method removeDataDir*(node: CodexProcess) {.raises: [CodexProcessError].} = + convertError("failed to remove codex node data directory"): + removeDir(node.dataDir) diff --git a/tests/integration/hardhatprocess.nim b/tests/integration/hardhatprocess.nim index 40c7942d4..b00b04005 100644 --- a/tests/integration/hardhatprocess.nim +++ b/tests/integration/hardhatprocess.nim @@ -8,27 +8,37 @@ import pkg/stew/io2 import std/os import std/sets import std/sequtils +import std/strformat import std/strutils import pkg/codex/conf import pkg/codex/utils/trackedfutures import ./codexclient import ./nodeprocess +import ./utils export codexclient export chronicles +export nodeprocess + +{.push raises: [].} logScope: topics = "integration testing hardhat process" - nodeName = "hardhat" -type HardhatProcess* = ref object of NodeProcess - logFile: ?IoHandle +type + OnOutputLineCaptured = proc(line: string) {.gcsafe, raises: [].} + HardhatProcess* = ref object of NodeProcess + logFile: ?IoHandle + onOutputLine: OnOutputLineCaptured + + HardhatProcessError* = object of NodeProcessError method workingDir(node: HardhatProcess): string = return currentSourcePath() / ".." / ".." / ".." / "vendor" / "codex-contracts-eth" method executable(node: HardhatProcess): string = - return "node_modules" / ".bin" / "hardhat" + return + "node_modules" / ".bin" / (when defined(windows): "hardhat.cmd" else: "hardhat") method startedOutput(node: HardhatProcess): string = return "Started HTTP and WebSocket JSON-RPC server at" @@ -36,7 +46,7 @@ method startedOutput(node: HardhatProcess): string = method processOptions(node: HardhatProcess): set[AsyncProcessOption] = return {} -method outputLineEndings(node: HardhatProcess): string {.raises: [].} = +method outputLineEndings(node: HardhatProcess): string = return "\n" proc openLogFile(node: HardhatProcess, logFilePath: string): IoHandle = @@ -51,37 +61,66 @@ proc openLogFile(node: HardhatProcess, logFilePath: string): IoHandle = return fileHandle -method start*(node: HardhatProcess) {.async.} = +method start*( + node: HardhatProcess +) {.async: (raises: [CancelledError, NodeProcessError]).} = + logScope: + nodeName = node.name + + var executable = "" + try: + executable = absolutePath(node.workingDir / node.executable) + if not fileExists(executable): + raiseAssert "cannot start hardhat, executable doesn't exist (looking for " & + &"{executable}). Try running `npm install` in {node.workingDir}." + except CatchableError as parent: + raiseAssert "failed build path to hardhat executable: " & parent.msg + let poptions = node.processOptions + {AsyncProcessOption.StdErrToStdOut} - trace "starting node", - args = node.arguments, - executable = node.executable, - workingDir = node.workingDir, - processOptions = poptions + let args = @["node", "--export", "deployment-localhost.json"].concat(node.arguments) + trace "starting node", args, executable, workingDir = node.workingDir try: node.process = await startProcess( - node.executable, + executable, node.workingDir, - @["node", "--export", "deployment-localhost.json"].concat(node.arguments), + args, options = poptions, stdoutHandle = AsyncProcess.Pipe, ) except CancelledError as error: raise error - except CatchableError as e: - error "failed to start hardhat process", error = e.msg + except CatchableError as parent: + raise newException( + HardhatProcessError, "failed to start hardhat process: " & parent.msg, parent + ) + +proc port(node: HardhatProcess): ?int = + var next = false + for arg in node.arguments: + # TODO: move to constructor + if next: + return parseInt(arg).catch.option + if arg.contains "--port": + next = true + + return none int proc startNode*( _: type HardhatProcess, args: seq[string], debug: string | bool = false, name: string, -): Future[HardhatProcess] {.async.} = + onOutputLineCaptured: OnOutputLineCaptured = nil, +): Future[HardhatProcess] {.async: (raises: [CancelledError, NodeProcessError]).} = + logScope: + nodeName = name + var logFilePath = "" var arguments = newSeq[string]() for arg in args: + # TODO: move to constructor if arg.contains "--log-file=": logFilePath = arg.split("=")[1] else: @@ -94,17 +133,25 @@ proc startNode*( arguments: arguments, debug: ($debug != "false"), trackedFutures: TrackedFutures.new(), - name: "hardhat", + name: name, + onOutputLine: onOutputLineCaptured, ) await hardhat.start() + # TODO: move to constructor if logFilePath != "": hardhat.logFile = some hardhat.openLogFile(logFilePath) return hardhat method onOutputLineCaptured(node: HardhatProcess, line: string) = + logScope: + nodeName = node.name + + if not node.onOutputLine.isNil: + node.onOutputLine(line) + without logFile =? node.logFile: return @@ -113,13 +160,49 @@ method onOutputLineCaptured(node: HardhatProcess, line: string) = discard logFile.closeFile() node.logFile = none IoHandle -method stop*(node: HardhatProcess) {.async.} = +proc closeProcessStreams(node: HardhatProcess) {.async: (raises: []).} = + when not defined(windows): + if not node.process.isNil: + trace "closing node process' streams" + await node.process.closeWait() + trace "node process' streams closed" + else: + # Windows hangs when attempting to close hardhat's process streams, so try + # to kill the process externally. + without port =? node.port: + error "Failed to get port from Hardhat args" + return + try: + let cmdResult = await forceKillProcess("node.exe", &"--port {port}") + if cmdResult.status > 0: + error "Failed to forcefully kill windows hardhat process", + port, exitCode = cmdResult.status, stderr = cmdResult.stdError + else: + trace "Successfully killed windows hardhat process by force", + port, exitCode = cmdResult.status, stdout = cmdResult.stdOutput + except ValueError, OSError: + let eMsg = getCurrentExceptionMsg() + error "Failed to forcefully kill windows hardhat process, bad path to command", + error = eMsg + except CancelledError as e: + discard + except AsyncProcessError as e: + error "Failed to forcefully kill windows hardhat process", port, error = e.msg + except AsyncProcessTimeoutError as e: + error "Timeout while forcefully killing windows hardhat process", + port, error = e.msg + +method stop*(node: HardhatProcess) {.async: (raises: []).} = # terminate the process await procCall NodeProcess(node).stop() + await node.closeProcessStreams() + if logFile =? node.logFile: trace "closing hardhat log file" discard logFile.closeFile() + node.process = nil + method removeDataDir*(node: HardhatProcess) = discard diff --git a/tests/integration/marketplacesuite.nim b/tests/integration/marketplacesuite.nim index 1e09963b0..d8f907154 100644 --- a/tests/integration/marketplacesuite.nim +++ b/tests/integration/marketplacesuite.nim @@ -60,7 +60,10 @@ template marketplacesuite*(name: string, body: untyped) = duration: uint64, collateralPerByte: UInt256, minPricePerBytePerSecond: UInt256, - ): Future[void] {.async: (raises: [CancelledError, HttpError, ConfigurationError]).} = + ): Future[void] {. + async: + (raises: [CancelledError, HttpError, ConfigurationError, CodexProcessError]) + .} = let totalCollateral = datasetSize.u256 * collateralPerByte # post availability to each provider for i in 0 ..< providers().len: diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index a95fde589..841f8ef24 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -1,3 +1,4 @@ +import std/httpclient import std/os import std/sequtils import std/strutils @@ -13,6 +14,7 @@ import ./codexprocess import ./hardhatconfig import ./hardhatprocess import ./nodeconfigs +import ./utils import ../asynctest import ../checktest @@ -24,6 +26,8 @@ export hardhatconfig export codexconfig export nodeconfigs +{.push raises: [].} + type RunningNode* = ref object role*: Role @@ -36,43 +40,36 @@ type Hardhat MultiNodeSuiteError = object of CatchableError - -const jsonRpcProviderUrl* = "ws://localhost:8545" - -proc raiseMultiNodeSuiteError(msg: string) = - raise newException(MultiNodeSuiteError, msg) - -proc nextFreePort*(startPort: int): Future[int] {.async.} = - proc client(server: StreamServer, transp: StreamTransport) {.async.} = - await transp.closeWait() - - var port = startPort - while true: - trace "checking if port is free", port + SuiteTimeoutError = object of MultiNodeSuiteError + +const HardhatPort {.intdefine.}: int = 8545 +const CodexApiPort {.intdefine.}: int = 8080 +const CodexDiscPort {.intdefine.}: int = 8090 +const TestId {.strdefine.}: string = "TestId" +const CodexLogToFile {.booldefine.}: bool = false +const CodexLogLevel {.strdefine.}: string = "" +const CodexLogsDir {.strdefine.}: string = "" + +proc raiseMultiNodeSuiteError( + msg: string, parent: ref CatchableError = nil +) {.raises: [MultiNodeSuiteError].} = + raise newException(MultiNodeSuiteError, msg, parent) + +template withLock(lock: AsyncLock, body: untyped) = + if lock.isNil: + lock = newAsyncLock() + + await lock.acquire() + try: + body + finally: try: - let host = initTAddress("127.0.0.1", port) - # We use ReuseAddr here only to be able to reuse the same IP/Port when - # there's a TIME_WAIT socket. It's useful when running the test multiple - # times or if a test ran previously using the same port. - var server = createStreamServer(host, client, {ReuseAddr}) - trace "port is free", port - await server.closeWait() - return port - except TransportOsError: - trace "port is not free", port - inc port - -proc sanitize(pathSegment: string): string = - var sanitized = pathSegment - for invalid in invalidFilenameChars.items: - sanitized = sanitized.replace(invalid, '_').replace(' ', '_') - sanitized - -proc getTempDirName*(starttime: string, role: Role, roleIdx: int): string = - getTempDir() / "Codex" / sanitize($starttime) / sanitize($role & "_" & $roleIdx) - -template multinodesuite*(name: string, body: untyped) = - asyncchecksuite name: + lock.release() + except AsyncLockError as parent: + raiseMultiNodeSuiteError "lock error", parent + +template multinodesuite*(suiteName: string, body: untyped) = + asyncchecksuite suiteName: # Following the problem described here: # https://github.com/NomicFoundation/hardhat/issues/2053 # It may be desirable to use http RPC provider. @@ -85,7 +82,7 @@ template multinodesuite*(name: string, body: untyped) = # If you want to use a different provider url in the nodes, you can # use withEthProvider config modifier in the node config # to set the desired provider url. E.g.: - # NodeConfigs( + # NodeConfigs( # hardhat: # HardhatConfig.none, # clients: @@ -93,6 +90,7 @@ template multinodesuite*(name: string, body: untyped) = # .withEthProvider("ws://localhost:8545") # .some, # ... + var jsonRpcProviderUrl = "ws://localhost:" & $HardhatPort var running {.inject, used.}: seq[RunningNode] var bootstrapNodes: seq[string] let starttime = now().format("yyyy-MM-dd'_'HH:mm:ss") @@ -101,6 +99,10 @@ template multinodesuite*(name: string, body: untyped) = var ethProvider {.inject, used.}: JsonRpcProvider var accounts {.inject, used.}: seq[Address] var snapshot: JsonNode + var lastUsedHardhatPort = HardhatPort + var lastUsedCodexApiPort = CodexApiPort + var lastUsedCodexDiscPort = CodexDiscPort + var codexPortLock: AsyncLock template test(tname, startNodeConfigs, tbody) = currentTestName = tname @@ -108,49 +110,50 @@ template multinodesuite*(name: string, body: untyped) = test tname: tbody - proc sanitize(pathSegment: string): string = - var sanitized = pathSegment - for invalid in invalidFilenameChars.items: - sanitized = sanitized.replace(invalid, '_').replace(' ', '_') - sanitized - - proc getLogFile(role: Role, index: ?int): string = - # create log file path, format: - # tests/integration/logs/ //_.log - - var logDir = - currentSourcePath.parentDir() / "logs" / sanitize($starttime & "__" & name) / - sanitize($currentTestName) - createDir(logDir) - - var fn = $role - if idx =? index: - fn &= "_" & $idx - fn &= ".log" - - let fileName = logDir / fn - return fileName + proc updatePort(url: var string, port: int) = + let parts = url.split(':') + url = @[parts[0], parts[1], $port].join(":") proc newHardhatProcess( config: HardhatConfig, role: Role - ): Future[NodeProcess] {.async.} = + ): Future[NodeProcess] {.async: (raises: [MultiNodeSuiteError, CancelledError]).} = var args: seq[string] = @[] if config.logFile: - let updatedLogFile = getLogFile(role, none int) - args.add "--log-file=" & updatedLogFile + try: + let updatedLogFile = getLogFile( + CodexLogsDir, starttime, suiteName, currentTestName, $role, none int + ) + args.add "--log-file=" & updatedLogFile + except IOError as e: + raiseMultiNodeSuiteError( + "failed to start hardhat because logfile path could not be obtained: " & + e.msg, + e, + ) + except OSError as e: + raiseMultiNodeSuiteError( + "failed to start hardhat because logfile path could not be obtained: " & + e.msg, + e, + ) + + let port = await nextFreePort(lastUsedHardhatPort) + jsonRpcProviderUrl.updatePort(port) + args.add("--port") + args.add($port) + lastUsedHardhatPort = port - let node = await HardhatProcess.startNode(args, config.debugEnabled, "hardhat") try: + let node = await HardhatProcess.startNode(args, config.debugEnabled, "hardhat") await node.waitUntilStarted() + trace "hardhat node started" + return node except NodeProcessError as e: raiseMultiNodeSuiteError "hardhat node not started: " & e.msg - trace "hardhat node started" - return node - proc newCodexProcess( roleIdx: int, conf: CodexConfig, role: Role - ): Future[NodeProcess] {.async.} = + ): Future[NodeProcess] {.async: (raises: [MultiNodeSuiteError, CancelledError]).} = let nodeIdx = running.len var config = conf @@ -158,34 +161,60 @@ template multinodesuite*(name: string, body: untyped) = raiseMultiNodeSuiteError "Cannot start node at nodeIdx " & $nodeIdx & ", not enough eth accounts." - let datadir = getTempDirName(starttime, role, roleIdx) + let datadir = getDataDir(TestId, currentTestName, $starttime, $role, some roleIdx) try: - if config.logFile.isSome: - let updatedLogFile = getLogFile(role, some roleIdx) - config.withLogFile(updatedLogFile) + if config.logFile.isSome or CodexLogToFile: + try: + let updatedLogFile = getLogFile( + CodexLogsDir, starttime, suiteName, currentTestName, $role, some roleIdx + ) + config.withLogFile(updatedLogFile) + except IOError as e: + raiseMultiNodeSuiteError( + "failed to start " & $role & + " because logfile path could not be obtained: " & e.msg, + e, + ) + except OSError as e: + raiseMultiNodeSuiteError( + "failed to start " & $role & + " because logfile path could not be obtained: " & e.msg, + e, + ) + + when CodexLogLevel != "": + config.addCliOption("--log-level", CodexLogLevel) + + var apiPort, discPort: int + withLock(codexPortLock): + apiPort = await nextFreePort(lastUsedCodexApiPort + nodeIdx) + discPort = await nextFreePort(lastUsedCodexDiscPort + nodeIdx) + config.addCliOption("--api-port", $apiPort) + config.addCliOption("--disc-port", $discPort) + lastUsedCodexApiPort = apiPort + lastUsedCodexDiscPort = discPort for bootstrapNode in bootstrapNodes: config.addCliOption("--bootstrap-node", bootstrapNode) - config.addCliOption("--api-port", $await nextFreePort(8080 + nodeIdx)) + config.addCliOption("--data-dir", datadir) config.addCliOption("--nat", "none") config.addCliOption("--listen-addrs", "/ip4/127.0.0.1/tcp/0") - config.addCliOption("--disc-port", $await nextFreePort(8090 + nodeIdx)) except CodexConfigError as e: raiseMultiNodeSuiteError "invalid cli option, error: " & e.msg - let node = await CodexProcess.startNode( - config.cliArgs, config.debugEnabled, $role & $roleIdx - ) - try: + let node = await CodexProcess.startNode( + config.cliArgs, config.debugEnabled, $role & $roleIdx + ) await node.waitUntilStarted() trace "node started", nodeName = $role & $roleIdx + return node + except CodexConfigError as e: + raiseMultiNodeSuiteError "failed to get cli args from config: " & e.msg, e except NodeProcessError as e: - raiseMultiNodeSuiteError "node not started, error: " & e.msg - - return node + raiseMultiNodeSuiteError "node not started, error: " & e.msg, e proc hardhat(): HardhatProcess = for r in running: @@ -211,7 +240,9 @@ template multinodesuite*(name: string, body: untyped) = if r.role == Role.Validator: CodexProcess(r.node) - proc startHardhatNode(config: HardhatConfig): Future[NodeProcess] {.async.} = + proc startHardhatNode( + config: HardhatConfig + ): Future[NodeProcess] {.async: (raises: [MultiNodeSuiteError, CancelledError]).} = return await newHardhatProcess(config, Role.Hardhat) proc startClientNode(conf: CodexConfig): Future[NodeProcess] {.async.} = @@ -223,44 +254,64 @@ template multinodesuite*(name: string, body: untyped) = ) return await newCodexProcess(clientIdx, config, Role.Client) - proc startProviderNode(conf: CodexConfig): Future[NodeProcess] {.async.} = - let providerIdx = providers().len - var config = conf - config.addCliOption(StartUpCmd.persistence, "--eth-provider", jsonRpcProviderUrl) - config.addCliOption( - StartUpCmd.persistence, "--eth-account", $accounts[running.len] - ) - config.addCliOption( - PersistenceCmd.prover, "--circom-r1cs", - "vendor/codex-contracts-eth/verifier/networks/hardhat/proof_main.r1cs", - ) - config.addCliOption( - PersistenceCmd.prover, "--circom-wasm", - "vendor/codex-contracts-eth/verifier/networks/hardhat/proof_main.wasm", - ) - config.addCliOption( - PersistenceCmd.prover, "--circom-zkey", - "vendor/codex-contracts-eth/verifier/networks/hardhat/proof_main.zkey", - ) - - return await newCodexProcess(providerIdx, config, Role.Provider) - - proc startValidatorNode(conf: CodexConfig): Future[NodeProcess] {.async.} = - let validatorIdx = validators().len - var config = conf - config.addCliOption(StartUpCmd.persistence, "--eth-provider", jsonRpcProviderUrl) - config.addCliOption( - StartUpCmd.persistence, "--eth-account", $accounts[running.len] - ) - config.addCliOption(StartUpCmd.persistence, "--validator") - - return await newCodexProcess(validatorIdx, config, Role.Validator) + proc startProviderNode( + conf: CodexConfig + ): Future[NodeProcess] {.async: (raises: [MultiNodeSuiteError, CancelledError]).} = + try: + let providerIdx = providers().len + var config = conf + config.addCliOption( + StartUpCmd.persistence, "--eth-provider", jsonRpcProviderUrl + ) + config.addCliOption( + StartUpCmd.persistence, "--eth-account", $accounts[running.len] + ) + config.addCliOption( + PersistenceCmd.prover, "--circom-r1cs", + "tests/circuits/fixtures/proof_main.r1cs", + ) + config.addCliOption( + PersistenceCmd.prover, "--circom-graph", + "tests/circuits/fixtures/proof_main.bin", + ) + config.addCliOption( + PersistenceCmd.prover, "--circom-zkey", + "tests/circuits/fixtures/proof_main.zkey", + ) + + return await newCodexProcess(providerIdx, config, Role.Provider) + except CodexConfigError as exc: + raiseMultiNodeSuiteError "Failed to start codex node, error adding cli options: " & + exc.msg, exc + + proc startValidatorNode( + conf: CodexConfig + ): Future[NodeProcess] {.async: (raises: [MultiNodeSuiteError, CancelledError]).} = + try: + let validatorIdx = validators().len + var config = conf + config.addCliOption( + StartUpCmd.persistence, "--eth-provider", jsonRpcProviderUrl + ) + config.addCliOption( + StartUpCmd.persistence, "--eth-account", $accounts[running.len] + ) + config.addCliOption(StartUpCmd.persistence, "--validator") + + return await newCodexProcess(validatorIdx, config, Role.Validator) + except CodexConfigError as e: + raiseMultiNodeSuiteError "Failed to start validator node, error adding cli options: " & + e.msg, e - proc teardownImpl() {.async.} = + proc teardownImpl() {.async: (raises: []).} = + trace "Tearing down test", suite = suiteName, test = currentTestName for nodes in @[validators(), clients(), providers()]: for node in nodes: await node.stop() # also stops rest client - node.removeDataDir() + try: + node.removeDataDir() + except CodexProcessError as e: + error "Failed to remove data dir during teardown", error = e.msg # if hardhat was started in the test, kill the node # otherwise revert the snapshot taken in the test setup @@ -268,15 +319,28 @@ template multinodesuite*(name: string, body: untyped) = if not hardhat.isNil: await hardhat.stop() else: - discard await send(ethProvider, "evm_revert", @[snapshot]) + try: + discard await noCancel send(ethProvider, "evm_revert", @[snapshot]) + except ProviderError as e: + error "Failed to revert hardhat state during teardown", error = e.msg - await ethProvider.close() + # TODO: JsonRpcProvider.close should NOT raise any exceptions + try: + await ethProvider.close() + except CatchableError: + discard running = @[] template failAndTeardownOnError(message: string, tryBody: untyped) = try: tryBody + except CancelledError as e: + await teardownImpl() + when declared(teardownAllIMPL): + teardownAllIMPL() + fail() + quit(1) except CatchableError as er: fatal message, error = er.msg echo "[FATAL] ", message, ": ", er.msg @@ -288,19 +352,34 @@ template multinodesuite*(name: string, body: untyped) = proc updateBootstrapNodes( node: CodexProcess - ): Future[void] {.async: (raises: [CatchableError]).} = - without ninfo =? await node.client.info(): - # raise CatchableError instead of Defect (with .get or !) so we - # can gracefully shutdown and prevent zombies - raiseMultiNodeSuiteError "Failed to get node info" - bootstrapNodes.add ninfo["spr"].getStr() + ): Future[void] {.async: (raises: [MultiNodeSuiteError]).} = + try: + without ninfo =? await node.client.info(): + # raise CatchableError instead of Defect (with .get or !) so we + # can gracefully shutdown and prevent zombies + raiseMultiNodeSuiteError "Failed to get node info" + bootstrapNodes.add ninfo["spr"].getStr() + except CatchableError as e: + raiseMultiNodeSuiteError "Failed to get node info: " & e.msg, e + + setupAll: + # When this file is run with `-d:chronicles_sinks=textlines[file]`, we + # need to set the log file path at runtime, otherwise chronicles didn't seem to + # create a log file even when using an absolute path + when defaultChroniclesStream.outputs is (FileOutput,) and CodexLogsDir.len > 0: + let logFile = + CodexLogsDir / sanitize(getAppFilename().extractFilename & ".chronicles.log") + let success = defaultChroniclesStream.outputs[0].open(logFile, fmAppend) + doAssert success, "Failed to open log file: " & logFile setup: + trace "Setting up test", suite = suiteName, test = currentTestName, nodeConfigs + if var conf =? nodeConfigs.hardhat: try: - let node = await startHardhatNode(conf) + let node = await noCancel startHardhatNode(conf) running.add RunningNode(role: Role.Hardhat, node: node) - except CatchableError as e: + except CatchableError as e: # CancelledError not raised due to noCancel echo "failed to start hardhat node" fail() quit(1) @@ -309,12 +388,16 @@ template multinodesuite*(name: string, body: untyped) = # Workaround for https://github.com/NomicFoundation/hardhat/issues/2053 # Do not use websockets, but use http and polling to stop subscriptions # from being removed after 5 minutes - ethProvider = JsonRpcProvider.new(jsonRpcProviderUrl) + ethProvider = JsonRpcProvider.new( + jsonRpcProviderUrl, pollingInterval = chronos.milliseconds(1000) + ) # if hardhat was NOT started by the test, take a snapshot so it can be # reverted in the test teardown if nodeConfigs.hardhat.isNone: snapshot = await send(ethProvider, "evm_snapshot") accounts = await ethProvider.listAccounts() + except CancelledError as e: + raise e except CatchableError as e: echo "Hardhat not running. Run hardhat manually " & "before executing tests, or include a " & "HardhatConfig in the test setup." @@ -344,7 +427,10 @@ template multinodesuite*(name: string, body: untyped) = # ensure that we have a recent block with a fresh timestamp discard await send(ethProvider, "evm_mine") + trace "Starting test", suite = suiteName, test = currentTestName + teardown: await teardownImpl() + trace "Test completed", suite = suiteName, test = currentTestName body diff --git a/tests/integration/nodeprocess.nim b/tests/integration/nodeprocess.nim index d50dacbe2..827a4b0f6 100644 --- a/tests/integration/nodeprocess.nim +++ b/tests/integration/nodeprocess.nim @@ -5,6 +5,7 @@ import pkg/chronicles import pkg/chronos/asyncproc import pkg/libp2p import std/os +import std/strformat import std/strutils import codex/conf import codex/utils/exceptions @@ -14,6 +15,8 @@ import ./codexclient export codexclient export chronicles +{.push raises: [].} + logScope: topics = "integration testing node process" @@ -39,24 +42,19 @@ method startedOutput(node: NodeProcess): string {.base, gcsafe.} = method processOptions(node: NodeProcess): set[AsyncProcessOption] {.base, gcsafe.} = raiseAssert "not implemented" -method outputLineEndings(node: NodeProcess): string {.base, gcsafe, raises: [].} = +method outputLineEndings(node: NodeProcess): string {.base, gcsafe.} = raiseAssert "not implemented" -method onOutputLineCaptured( - node: NodeProcess, line: string -) {.base, gcsafe, raises: [].} = +method onOutputLineCaptured(node: NodeProcess, line: string) {.base, gcsafe.} = raiseAssert "not implemented" -method start*(node: NodeProcess) {.base, async.} = +method start*(node: NodeProcess) {.base, async: (raises: [CancelledError]).} = logScope: nodeName = node.name let poptions = node.processOptions + {AsyncProcessOption.StdErrToStdOut} trace "starting node", - args = node.arguments, - executable = node.executable, - workingDir = node.workingDir, - processOptions = poptions + args = node.arguments, executable = node.executable, workingDir = node.workingDir try: if node.debug: @@ -81,11 +79,13 @@ proc captureOutput( trace "waiting for output", output - let stream = node.process.stdoutStream - try: while node.process.running.option == some true: - while (let line = await stream.readLine(0, node.outputLineEndings); line != ""): + while ( + let line = await node.process.stdoutStream.readLine(0, node.outputLineEndings) + line != "" + ) + : if node.debug: # would be nice if chronicles could parse and display with colors echo line @@ -95,8 +95,8 @@ proc captureOutput( node.onOutputLineCaptured(line) - await sleepAsync(1.millis) - await sleepAsync(1.millis) + await sleepAsync(1.nanos) + await sleepAsync(1.nanos) except CancelledError: discard # do not propagate as captureOutput was asyncSpawned except AsyncStreamError as e: @@ -104,7 +104,7 @@ proc captureOutput( proc startNode*[T: NodeProcess]( _: type T, args: seq[string], debug: string | bool = false, name: string -): Future[T] {.async.} = +): Future[T] {.async: (raises: [CancelledError]).} = ## Starts a Codex Node with the specified arguments. ## Set debug to 'true' to see output of the node. let node = T( @@ -116,34 +116,36 @@ proc startNode*[T: NodeProcess]( await node.start() return node -method stop*(node: NodeProcess) {.base, async.} = +method stop*( + node: NodeProcess, expectedErrCode: int = -1 +) {.base, async: (raises: []).} = logScope: nodeName = node.name await node.trackedFutures.cancelTracked() - if node.process != nil: + if not node.process.isNil: + let processId = node.process.processId + trace "terminating node process...", processId try: - trace "terminating node process..." - if errCode =? node.process.terminate().errorOption: - error "failed to terminate process", errCode = $errCode - - trace "waiting for node process to exit" - let exitCode = await node.process.waitForExit(3.seconds) - if exitCode > 0: - error "failed to exit process, check for zombies", exitCode - - trace "closing node process' streams" - await node.process.closeWait() - except CancelledError as error: - raise error - except CatchableError as e: - error "error stopping node process", error = e.msg - finally: - node.process = nil - - trace "node stopped" - -proc waitUntilOutput*(node: NodeProcess, output: string) {.async.} = + let exitCode = await noCancel node.process.terminateAndWaitForExit(2.seconds) + if exitCode > 0 and exitCode != 143 and # 143 = SIGTERM (initiated above) + exitCode != expectedErrCode: + warn "process exited with a non-zero exit code", exitCode + trace "node process terminated", exitCode + except CatchableError: + try: + let forcedExitCode = await noCancel node.process.killAndWaitForExit(3.seconds) + trace "node process forcibly killed with exit code: ", exitCode = forcedExitCode + except CatchableError as e: + warn "failed to kill node process in time, it will be killed when the parent process exits", + error = e.msg + writeStackTrace() + + trace "node stopped" + +proc waitUntilOutput*( + node: NodeProcess, output: string +) {.async: (raises: [CancelledError, AsyncTimeoutError]).} = logScope: nodeName = node.name @@ -153,9 +155,21 @@ proc waitUntilOutput*(node: NodeProcess, output: string) {.async.} = let fut = node.captureOutput(output, started) node.trackedFutures.track(fut) asyncSpawn fut - await started.wait(60.seconds) # allow enough time for proof generation + try: + await started.wait(60.seconds) # allow enough time for proof generation + except AsyncTimeoutError as e: + raise e + except CancelledError as e: + raise e + except CatchableError as e: # unsure where this originates from + error "unexpected error occurred waiting for node output", error = e.msg + +proc waitUntilStarted*( + node: NodeProcess +) {.async: (raises: [CancelledError, NodeProcessError]).} = + logScope: + nodeName = node.name -proc waitUntilStarted*(node: NodeProcess) {.async.} = try: await node.waitUntilOutput(node.startedOutput) trace "node started" @@ -168,10 +182,10 @@ proc waitUntilStarted*(node: NodeProcess) {.async.} = raise newException(NodeProcessError, "node did not output '" & node.startedOutput & "'") -proc restart*(node: NodeProcess) {.async.} = +method restart*(node: NodeProcess) {.base, async.} = await node.stop() await node.start() await node.waitUntilStarted() -method removeDataDir*(node: NodeProcess) {.base.} = +method removeDataDir*(node: NodeProcess) {.base, raises: [NodeProcessError].} = raiseAssert "[removeDataDir] not implemented" diff --git a/tests/integration/scripts/winkillprocess.sh b/tests/integration/scripts/winkillprocess.sh new file mode 100644 index 000000000..b5e58ab49 --- /dev/null +++ b/tests/integration/scripts/winkillprocess.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +# List all processes with a specific name +list() { + local name=$1 + echo "Listing all processes named '$name'..." + powershell.exe -Command "Get-CimInstance Win32_Process -Filter \"name = '$name'\" | Select-Object ProcessId, Name, CommandLine | Format-Table -AutoSize" +} + +# Search for processes with a specific name and command line pattern +search() { + local name=$1 + local pattern=$2 + echo "Searching for '$name' processes with command line matching '$pattern'..." + powershell.exe -Command " + \$processes = Get-CimInstance Win32_Process -Filter \"name = '$name'\" | Where-Object { \$_.CommandLine -match '$pattern' }; + if (\$processes) { + \$processes | Select-Object ProcessId, Name, CommandLine | Format-Table -AutoSize; + } else { + Write-Host \"No matching '$name' processes found\"; + } + " +} + +# Kill all processes with a specific name +killall() { + local name=$1 + echo "Finding and killing all '$name' processes..." + powershell.exe -Command " + \$processes = Get-CimInstance Win32_Process -Filter \"name = '$name'\"; + if (\$processes) { + foreach (\$process in \$processes) { + Stop-Process -Id \$process.ProcessId -Force; + Write-Host \"Killed process \$(\$process.ProcessId)\"; + } + } else { + Write-Host \"No '$name' processes found\"; + } + " +} + +# Kill processes with a specific name and command line pattern +kill() { + local name=$1 + local pattern=$2 + echo "Finding and killing '$name' processes with command line matching '$pattern'..." + powershell.exe -Command " + \$processes = Get-CimInstance Win32_Process -Filter \"name = '$name'\" | Where-Object { \$_.CommandLine -match '$pattern' }; + if (\$processes) { + foreach (\$process in \$processes) { + Stop-Process -Id \$process.ProcessId -Force; + Write-Host \"Killed process \$(\$process.ProcessId)\"; + } + } else { + Write-Host \"No matching '$name' processes found\"; + } + " +} + +# Check if being run directly or sourced +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + # If run directly (not sourced), provide command line interface + case "$1" in + list) + if [ -z "$2" ]; then + echo "Usage: $0 list PROCESS_NAME" + exit 1 + fi + list "$2" + ;; + search) + if [ -z "$2" ] || [ -z "$3" ]; then + echo "Usage: $0 search PROCESS_NAME COMMANDLINE_PATTERN" + exit 1 + fi + search "$2" "$3" + ;; + killall) + if [ -z "$2" ]; then + echo "Usage: $0 killall PROCESS_NAME" + exit 1 + fi + killall "$2" + ;; + kill) + if [ -z "$2" ] || [ -z "$3" ]; then + echo "Usage: $0 kill PROCESS_NAME COMMANDLINE_PATTERN" + exit 1 + fi + kill "$2" "$3" + ;; + *) + echo "Usage: $0 {list PROCESS_NAME|search PROCESS_NAME COMMANDLINE_PATTERN|killall PROCESS_NAME|kill PROCESS_NAME COMMANDLINE_PATTERN}" + exit 1 + ;; + esac +fi diff --git a/tests/integration/testcli.nim b/tests/integration/testcli.nim index d9f2d0817..2c6422bab 100644 --- a/tests/integration/testcli.nim +++ b/tests/integration/testcli.nim @@ -1,31 +1,91 @@ import std/tempfiles +import std/times import codex/conf import codex/utils/fileutils import ../asynctest import ../checktest import ./codexprocess import ./nodeprocess +import ./utils import ../examples +const HardhatPort {.intdefine.}: int = 8545 +const CodexApiPort {.intdefine.}: int = 8080 +const CodexDiscPort {.intdefine.}: int = 8090 +const CodexLogToFile {.booldefine.}: bool = false +const CodexLogLevel {.strdefine.}: string = "" +const CodexLogsDir {.strdefine.}: string = "" + asyncchecksuite "Command line interface": + let startTime = now().format("yyyy-MM-dd'_'HH:mm:ss") let key = "4242424242424242424242424242424242424242424242424242424242424242" - proc startCodex(args: seq[string]): Future[CodexProcess] {.async.} = - return await CodexProcess.startNode(args, false, "cli-test-node") + var currentTestName = "" + var testCount = 0 + var nodeCount = 0 + + template test(tname, tbody) = + inc testCount + currentTestName = tname + test tname: + tbody + + proc addLogFile(args: seq[string]): seq[string] = + var args = args + when CodexLogToFile: + args.add( + "--log-file=" & + getLogFile( + CodexLogsDir, + startTime, + "Command line interface", + currentTestName, + "Client", + some nodeCount mod testCount, + ) + ) + when CodexLogLevel != "": + args.add "--log-level=" & CodexLogLevel + + return args + + proc startCodex(arguments: seq[string]): Future[CodexProcess] {.async.} = + inc nodeCount + let args = arguments.addLogFile + return await CodexProcess.startNode( + args.concat( + @[ + "--api-port=" & $(await nextFreePort(CodexApiPort + nodeCount)), + "--disc-port=" & $(await nextFreePort(CodexDiscPort + nodeCount)), + ] + ), + debug = false, + "cli-test-node", + ) test "complains when persistence is enabled without ethereum account": let node = await startCodex(@["persistence"]) + + defer: + await node.stop() + await node.waitUntilOutput("Persistence enabled, but no Ethereum account was set") - await node.stop() + await node.stop(expectedErrCode = 1) test "complains when ethereum private key file has wrong permissions": let unsafeKeyFile = genTempPath("", "") discard unsafeKeyFile.writeFile(key, 0o666) - let node = await startCodex(@["persistence", "--eth-private-key=" & unsafeKeyFile]) + let node = await startCodex( + @[ + "persistence", + "--eth-provider=" & "ws://localhost:" & $HardhatPort, + "--eth-private-key=" & unsafeKeyFile, + ] + ) await node.waitUntilOutput( "Ethereum private key file does not have safe file permissions" ) - await node.stop() + await node.stop(expectedErrCode = 1) discard removeFile(unsafeKeyFile) let @@ -36,25 +96,38 @@ asyncchecksuite "Command line interface": test "suggests downloading of circuit files when persistence is enabled without accessible r1cs file": let node = await startCodex(@["persistence", "prover", marketplaceArg]) await node.waitUntilOutput(expectedDownloadInstruction) - await node.stop() + await node.stop(expectedErrCode = 1) test "suggests downloading of circuit files when persistence is enabled without accessible wasm file": let node = await startCodex( @[ - "persistence", "prover", marketplaceArg, + "persistence", + "--eth-provider=" & "ws://localhost:" & $HardhatPort, + "prover", + marketplaceArg, "--circom-r1cs=tests/circuits/fixtures/proof_main.r1cs", ] ) + + defer: + await node.stop() + await node.waitUntilOutput(expectedDownloadInstruction) - await node.stop() + await node.stop(expectedErrCode = 1) test "suggests downloading of circuit files when persistence is enabled without accessible zkey file": let node = await startCodex( @[ - "persistence", "prover", marketplaceArg, + "persistence", + "--eth-provider=" & "ws://localhost:" & $HardhatPort, + "prover", + marketplaceArg, "--circom-r1cs=tests/circuits/fixtures/proof_main.r1cs", - "--circom-wasm=tests/circuits/fixtures/proof_main.wasm", ] ) + + defer: + await node.stop() + await node.waitUntilOutput(expectedDownloadInstruction) - await node.stop() + await node.stop(expectedErrCode = 1) diff --git a/tests/integration/testecbug.nim b/tests/integration/testecbug.nim index 6b86fd29a..855da37af 100644 --- a/tests/integration/testecbug.nim +++ b/tests/integration/testecbug.nim @@ -7,12 +7,12 @@ import ./hardhatconfig marketplacesuite "Bug #821 - node crashes during erasure coding": test "should be able to create storage request and download dataset", NodeConfigs( - clients: CodexConfigs - .init(nodes = 1) - # .debug() # uncomment to enable console log output.debug() - .withLogFile() - # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("node", "erasure", "marketplace").some, + clients: CodexConfigs.init(nodes = 1) + # .debug() # uncomment to enable console log output.debug() + # .withLogFile() + # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("node", "erasure", "marketplace") + .some, providers: CodexConfigs.init(nodes = 0).some, ): let diff --git a/tests/integration/testmanager.nim b/tests/integration/testmanager.nim new file mode 100644 index 000000000..145ce40e6 --- /dev/null +++ b/tests/integration/testmanager.nim @@ -0,0 +1,796 @@ +import std/os +import std/strformat +import std/terminal +from std/times import fromUnix, format, now +from std/unicode import toUpper +import pkg/chronos +import pkg/chronos/asyncproc +import pkg/codex/conf +import pkg/codex/logutils +import pkg/codex/utils/trackedfutures +import pkg/questionable +import pkg/questionable/results +import ./hardhatprocess +import ./utils +import ../examples + +type + Hardhat = ref object + process: HardhatProcess + port: int + + TestManagerConfig* = object # Echoes stdout from Hardhat process + debugHardhat*: bool + # Shows all log topics at TRACE log level by disabling codex node output log + # topic filters, eg libp2p, websock, JSON RPC + noCodexLogFilters*: bool + # Shows test status updates at regular time intervals. Useful for running + # locally while attended. Set to false for unattended runs, eg CI. + showContinuousStatusUpdates*: bool + logsDir*: string + testTimeout*: Duration # individual test timeout + + TestManager* = ref object + config: TestManagerConfig + testConfigs: seq[IntegrationTestConfig] + tests: seq[IntegrationTest] + hardhats: seq[Hardhat] + lastHardhatPort: int + lastCodexApiPort: int + lastCodexDiscPort: int + timeStart: ?Moment + timeEnd: ?Moment + codexPortLock: AsyncLock + hardhatPortLock: AsyncLock + hardhatProcessLock: AsyncLock + trackedFutures: TrackedFutures + + IntegrationTestConfig* = object + startHardhat: bool + testFile: string + name: string + + IntegrationTestStatus = enum ## The status of a test when it is done. + New # Test not yet run + Running # Test currently running + Ok # Test file launched, and exited with 0. Indicates all tests completed and passed. + Failed + # Test file launched, but exited with a non-zero exit code. Indicates either the test file did not compile, or one or more of the tests in the file failed + Timeout # Test file launched, but the tests did not complete before the timeout. + Error + # Test file did not launch correctly. Indicates an error occurred running the tests (usually an error in the harness). + + IntegrationTest = ref object + manager: TestManager + config: IntegrationTestConfig + process: AsyncProcessRef + timeStart: ?Moment + timeEnd: ?Moment + output: ?!TestOutput + testId: string # when used in datadir path, prevents data dir clashes + status: IntegrationTestStatus + command: string + logsDir: string + + TestOutput = ref object + stdOut*: seq[string] + stdErr*: seq[string] + exitCode*: ?int + + TestManagerError* = object of CatchableError + + Border {.pure.} = enum + Left + Right + + Align {.pure.} = enum + Left + Right + + MarkerPosition {.pure.} = enum + Start + Finish + +{.push raises: [].} + +logScope: + topics = "testing integration testmanager" + +proc printOutputMarker( + test: IntegrationTest, position: MarkerPosition, msg: string +) {.gcsafe, raises: [].} + +proc raiseTestManagerError( + msg: string, parent: ref CatchableError = nil +) {.raises: [TestManagerError].} = + raise newException(TestManagerError, msg, parent) + +template echoStyled(args: varargs[untyped]) = + try: + styledEcho args + except CatchableError as parent: + # no need to re-raise this, as it'll eventually have to be logged only + error "failed to print to terminal", error = parent.msg + +template ignoreCancelled(body) = + try: + body + except CancelledError: + discard + +func logFile*(_: type TestManager, dir: string): string = + dir / "testmanager.chronicles.log" + +func logFile(manager: TestManager): string = + TestManager.logFile(manager.config.logsDir) + +func logFile(test: IntegrationTest, fileName: string): string = + let testName = sanitize(test.config.name) + test.logsDir / &"{testName}.{fileName}" + +func isErrorLike(output: ?!TestOutput): bool = + # Three lines is an arbitrary number, however it takes into account the + # "LevelDB already build" line and blank line that is output to stdout. This + # typically means that the exitCode == 1 (test failed) and if stdout is short, + # we're dealing with an error + o =? output and o.stdOut.len < 3 + +proc new*( + _: type TestManager, + config: TestManagerConfig, + testConfigs: seq[IntegrationTestConfig], +): TestManager = + TestManager( + config: config, + testConfigs: testConfigs, + lastHardhatPort: 8545, + lastCodexApiPort: 8000, + lastCodexDiscPort: 18000, # keep separated by 10000 to minimise overlap + trackedFutures: TrackedFutures.new(), + ) + +func init*( + _: type IntegrationTestConfig, testFile: string, startHardhat: bool, name = "" +): IntegrationTestConfig = + IntegrationTestConfig( + testFile: testFile, + name: if name == "": testFile.extractFilename else: name, + startHardhat: startHardhat, + ) + +template withLock*(lock: AsyncLock, body: untyped) = + if lock.isNil: + lock = newAsyncLock() + + await lock.acquire() + try: + body + finally: + try: + lock.release() + except AsyncLockError as parent: + raiseTestManagerError "lock error", parent + +proc duration(manager: TestManager): Duration = + let now = Moment.now() + (manager.timeEnd |? now) - (manager.timeStart |? now) + +proc allTestsPassed*(manager: TestManager): ?!bool = + for test in manager.tests: + if test.status in {IntegrationTestStatus.New, IntegrationTestStatus.Running}: + return failure "Integration tests not complete" + + if test.status != IntegrationTestStatus.Ok: + return success false + + return success true + +proc duration(test: IntegrationTest): Duration = + let now = Moment.now() + (test.timeEnd |? now) - (test.timeStart |? now) + +proc startHardhat( + test: IntegrationTest +): Future[Hardhat] {.async: (raises: [CancelledError, TestManagerError]).} = + var args: seq[string] = @[] + var port: int + + let hardhat = Hardhat.new() + + withLock(test.manager.hardhatPortLock): + port = await nextFreePort(test.manager.lastHardhatPort + 1) + test.manager.lastHardhatPort = port + + args.add("--port") + args.add($port) + if test.manager.config.debugHardhat: + args.add("--log-file=" & test.logsDir / "hardhat.log") + + trace "starting hardhat process on port ", port + try: + withLock(test.manager.hardhatProcessLock): + let node = await HardhatProcess.startNode( + args, false, "hardhat for '" & test.config.name & "'" + ) + hardhat.process = node + hardhat.port = port + await node.waitUntilStarted() + return hardhat + except CancelledError as e: + raise e + except CatchableError as e: + raiseTestManagerError "hardhat node failed to start: " & e.msg, e + +proc printResult(test: IntegrationTest, colour: ForegroundColor) = + echoStyled styleBright, + colour, + &"[{toUpper $test.status}] ", + resetStyle, + test.config.name, + resetStyle, + styleDim, + &" ({test.duration})" + +proc printOutputMarker(test: IntegrationTest, position: MarkerPosition, msg: string) = + if position == MarkerPosition.Start: + echo "" + + echoStyled styleBright, + bgWhite, fgBlack, &"----- {toUpper $position} {test.config.name} {msg} -----" + + if position == MarkerPosition.Finish: + echo "" + +proc colorise(output: string): string = + proc setColour(text: string, colour: ForegroundColor): string = + &"{ansiForegroundColorCode(colour, true)}{text}{ansiResetCode}" + + let replacements = @[("[OK]", fgGreen), ("[FAILED]", fgRed), ("[Suite]", fgBlue)] + result = output + for (text, colour) in replacements: + result = result.replace(text, text.setColour(colour)) + +proc printResult(test: IntegrationTest, printStdOut, printStdErr: bool) = + case test.status + of IntegrationTestStatus.New: + test.printResult(fgBlue) + of IntegrationTestStatus.Running: + test.printResult(fgCyan) + of IntegrationTestStatus.Error: + if error =? test.output.errorOption: + test.printResult(fgRed) + test.printOutputMarker(MarkerPosition.Start, "test harness errors") + echo "Error during test execution: ", error.msg + echo "Stacktrace: ", error.getStackTrace() + test.printOutputMarker(MarkerPosition.Finish, "test harness errors") + if output =? test.output: + if printStdErr: + test.printOutputMarker(MarkerPosition.Start, "test file errors (stderr)") + echo output.stdErr.join("\n") + test.printOutputMarker(MarkerPosition.Finish, "test file errors (stderr)") + of IntegrationTestStatus.Failed: + if output =? test.output: + if printStdErr: + test.printOutputMarker(MarkerPosition.Start, "test file errors (stderr)") + echo output.stdErr.join("\n") + test.printOutputMarker(MarkerPosition.Finish, "test file errors (stderr)") + if printStdOut: + test.printOutputMarker(MarkerPosition.Start, "codex node output (stdout)") + echo output.stdOut.join("\n").colorise + test.printOutputMarker(MarkerPosition.Finish, "codex node output (stdout)") + test.printResult(fgRed) + of IntegrationTestStatus.Timeout: + if printStdOut and output =? test.output: + test.printOutputMarker(MarkerPosition.Start, "codex node output (stdout)") + echo output.stdOut.join("\n").colorise + test.printOutputMarker(MarkerPosition.Finish, "codex node output (stdout)") + test.printResult(fgYellow) + of IntegrationTestStatus.Ok: + if printStdOut and output =? test.output: + test.printOutputMarker(MarkerPosition.Start, "codex node output (stdout)") + echo output.stdOut.join("\n").colorise + test.printOutputMarker(MarkerPosition.Finish, "codex node output (stdout)") + test.printResult(fgGreen) + +proc printSummary(test: IntegrationTest) = + test.printResult(printStdOut = false, printStdErr = false) + +proc printStart(test: IntegrationTest) = + echoStyled styleBright, + fgMagenta, &"[Integration test started] ", resetStyle, test.config.name + +proc buildCommand( + test: IntegrationTest, hardhatPort: ?int +): Future[string] {.async: (raises: [CancelledError, TestManagerError]).} = + var hhPort = string.none + if test.config.startHardhat: + without port =? hardhatPort: + raiseTestManagerError "hardhatPort required when 'config.startHardhat' is true" + hhPort = some "-d:HardhatPort=" & $port + + var testFile: string + try: + testFile = absolutePath( + test.config.testFile, root = currentSourcePath().parentDir().parentDir() + ) + except ValueError as parent: + raiseTestManagerError "bad file name, testFile: " & test.config.testFile, parent + + withLock(test.manager.codexPortLock): + # Increase the port by 1000 to allow each test to run 1000 codex nodes + # (clients, SPs, validators) giving a good chance the port will be free. We + # cannot rely on `nextFreePort` in multinodes entirely as there could be a + # concurrency issue where the port is determined free in mulitiple tests and + # then there is a clash during the run. Windows, in particular, does not + # like giving up ports. + let apiPort = await nextFreePort(test.manager.lastCodexApiPort + 1000) + test.manager.lastCodexApiPort = apiPort + let discPort = await nextFreePort(test.manager.lastCodexDiscPort + 1000) + test.manager.lastCodexDiscPort = discPort + + let codexLogLevel = + if test.manager.config.noCodexLogFilters: + "TRACE" + else: + "TRACE;disabled:libp2p,websock,JSONRPC-HTTP-CLIENT,JSONRPC-WS-CLIENT,discv5" + + withLock(test.manager.hardhatPortLock): + try: + return + #!fmt: off + "nim c " & + &"-d:CodexApiPort={apiPort} " & + &"-d:CodexDiscPort={discPort} " & + &"-d:CodexLogsDir={test.logsDir} " & + &"-d:CodexLogLevel=\"{codexLogLevel}\" " & + &"-d:CodexLogToFile=true " & + (hhPort |? "") & " " & + &"-d:TestId={test.testId} " & + # Log multinodes chronicles logs settings (log to file with no + # colours, and loglevel = TRACE). + "-d:chronicles_log_level=TRACE " & + "-d:chronicles_sinks=textlines[nocolors,file] " & + "-d:nimUnittestOutputLevel:VERBOSE " & + "--verbosity:0 " & + "--hints:off " & + "-d:release " & + "-r " & + &"{testFile}" + #!fmt: on + except ValueError as parent: + raiseTestManagerError "bad command --\n" & ", apiPort: " & $apiPort & + ", discPort: " & $discPort & ", testFile: " & testFile & ", error: " & + parent.msg, parent + +proc setup( + test: IntegrationTest +): Future[?Hardhat] {.async: (raises: [CancelledError, TestManagerError]).} = + var hardhat = Hardhat.none + var hardhatPort = int.none + + if test.config.startHardhat: + let hh = await test.startHardhat() + hardhat = some hh + hardhatPort = some hh.port + test.manager.hardhats.add hh + + test.command = await test.buildCommand(hardhatPort) + + return hardhat + +proc teardownHardhat(test: IntegrationTest, hardhat: Hardhat) {.async: (raises: []).} = + try: + trace "Stopping hardhat", name = test.config.name + await noCancel hardhat.process.stop() + trace "Hardhat stopped", name = test.config.name + except CatchableError as e: # CancelledError not raised due to noCancel + warn "Failed to stop hardhat node, continuing", + error = e.msg, test = test.config.name + + test.manager.hardhats.keepItIf(it != hardhat) + +proc closeProcessStreams(test: IntegrationTest) {.async: (raises: []).} = + logScope: + name = test.config.name + + when not defined(windows): + if not test.process.isNil: + trace "Closing test process' streams" + await test.process.closeWait() + trace "Test process' streams closed" + else: + # Windows hangs when attempting to close the test's process streams, so try + # to kill the process externally. + try: + let cmdResult = await forceKillProcess("nim.exe", &"-d:TestId {test.testId}") + if cmdResult.status > 0: + error "Failed to forcefully kill windows test process", + testId = test.testId, exitCode = cmdResult.status, stderr = cmdResult.stdError + else: + trace "Successfully killed windows test process by force", + testId = test.testId, + exitCode = cmdResult.status, + stdout = cmdResult.stdOutput + except ValueError, OSError: + let eMsg = getCurrentExceptionMsg() + error "Failed to forcefully kill windows test process, bad path to command", + error = eMsg + except CancelledError as e: + discard + except AsyncProcessError as e: + error "Failed to forcefully kill windows test process", + testId = test.testId, error = e.msg + except AsyncProcessTimeoutError as e: + error "Timeout while forcefully killing windows test process", + testId = test.testId, error = e.msg + +proc teardownTest(test: IntegrationTest) {.async: (raises: []).} = + logScope: + test = test.config.name + + trace "Tearing down test" + + test.timeEnd = some Moment.now() + + if not test.process.isNil: + var output = test.output.expect("should have output value") + if test.process.running |? false: + trace "Test process still running, terminating..." + try: + output.exitCode = + some (await noCancel test.process.terminateAndWaitForExit(1.seconds)) + trace "Test process terminated", exitCode = output.exitCode + except AsyncProcessError, AsyncProcessTimeoutError: + let e = getCurrentException() + warn "Test process failed to terminate, check for zombies", error = e.msg + + await test.closeProcessStreams() + test.process = nil + +proc teardown(test: IntegrationTest, hardhat: ?Hardhat) {.async: (raises: []).} = + if test.config.startHardhat and hardhat =? hardhat and not hardhat.process.isNil: + await test.teardownHardhat(hardhat) + + await test.teardownTest() + +proc untilTimeout( + fut: InternalRaisesFuture, timeout: Duration +): Future[void] {.async: (raises: [CancelledError, AsyncTimeoutError]).} = + ## Returns a Future that completes when either fut finishes or timeout elapses, + ## or if they finish at the same time. If timeout elapses, an AsyncTimeoutError + ## is raised. If fut fails, its error is raised. + + let timer = sleepAsync(timeout) + defer: + # Called even when exception raised, including CancelledError. `race` does + # not cancel its futures when it's cancelled, so cancel here, which is ok + # even if they're already completed. + await fut.cancelAndWait() + await timer.cancelAndWait() + + try: + discard await race(fut, timer) + except ValueError as e: + raiseAssert "should not happen" + + if fut.finished(): # or fut and timer both finished simultaneously + if fut.failed(): + await fut # raise fut error + return # unreachable, for readability + else: # timeout + raise newException(AsyncTimeoutError, "Timed out") + +proc captureOutput( + process: AsyncProcessRef, stream: AsyncStreamReader, filePath: string +): Future[seq[string]] {.async: (raises: [CancelledError]).} = + var output: seq[string] = @[] + try: + while process.running.option == some true: + while (let line = await stream.readLine(0, "\n"); line != ""): + try: + output.add line + filePath.appendFile(line & "\n".stripAnsi) + await sleepAsync(1.nanos) + except IOError as e: + warn "Failed to write test stdout and/or stderr to file", error = e.msg + await sleepAsync(1.nanos) + return output + except CancelledError as e: + raise e + except AsyncStreamError as e: + error "Error reading output stream", error = e.msg + +proc captureProcessOutput( + test: IntegrationTest +): Future[(seq[string], seq[string])] {.async: (raises: [CancelledError]).} = + logScope: + name = test.config.name + + trace "Reading stdout and stderr streams from test process" + + let futStdOut = + test.process.captureOutput(test.process.stdoutStream, test.logFile("stdout.log")) + let futStdErr = + test.process.captureOutput(test.process.stderrStream, test.logFile("stderr.log")) + await allFutures(futStdOut, futStdErr) + return (await futStdOut, await futStdErr) + +proc start(test: IntegrationTest) {.async: (raises: []).} = + logScope: + name = test.config.name + duration = test.duration + + trace "Running test" + + test.logsDir = test.manager.config.logsDir / sanitize(test.config.name) + try: + createDir(test.logsDir) + except CatchableError as e: + test.timeStart = some Moment.now() + test.timeEnd = some Moment.now() + test.status = IntegrationTestStatus.Error + test.output = TestOutput.failure(e) + error "failed to create test log dir", logDir = test.logsDir, error = e.msg + return + + test.timeStart = some Moment.now() + test.status = IntegrationTestStatus.Running + + var hardhat = none Hardhat + + ignoreCancelled: + try: + hardhat = await test.setup() + except TestManagerError as e: + test.timeEnd = some Moment.now() + test.status = IntegrationTestStatus.Error + test.output = TestOutput.failure(e) + error "Failed to start hardhat and build command", error = e.msg + return + + trace "Starting parallel integration test", + command = test.command, timeout = test.manager.config.testTimeout + test.printStart() + + try: + test.process = await startProcess( + command = test.command, + options = {AsyncProcessOption.EvalCommand}, + stdoutHandle = AsyncProcess.Pipe, + stderrHandle = AsyncProcess.Pipe, + ) + except AsyncProcessError as e: + test.timeEnd = some Moment.now() + error "Failed to start test process", error = e.msg + test.output = TestOutput.failure(e) + test.status = IntegrationTestStatus.Error + return + + var futCaptureOutput: Future[(seq[string], seq[string])].Raising([CancelledError]) + defer: + # called at the end of successful runs but also when `start` is cancelled + # (from `untilTimeout`) due to a timeout. This defer runs first before + # `untilTimeout` exceptions are handled in `run` + await test.teardown(hardhat) # doesn't raise CancelledError, so noCancel not needed + await futCaptureOutput.cancelAndWait() + + var output = TestOutput.new() + test.output = success(output) + futCaptureOutput = test.captureProcessOutput() + + output.exitCode = + try: + some (await test.process.waitForExit(test.manager.config.testTimeout)) + except AsyncProcessTimeoutError as e: + test.timeEnd = some Moment.now() + test.status = IntegrationTestStatus.Timeout + error "Test process failed to exit before timeout", + timeout = test.manager.config.testTimeout + return + except AsyncProcessError as e: + test.timeEnd = some Moment.now() + test.status = IntegrationTestStatus.Error + test.output = TestOutput.failure(e) + error "Test failed to complete", error = e.msg + return + + let (stdOut, stdErr) = await futCaptureOutput + output.stdOut = stdOut + output.stdErr = stdErr + + test.status = + if output.exitCode == some QuitSuccess: + IntegrationTestStatus.Ok + elif output.exitCode == some QuitFailure: + IntegrationTestStatus.Failed + else: + IntegrationTestStatus.Error + +proc continuallyShowUpdates(manager: TestManager) {.async: (raises: []).} = + ignoreCancelled: + while true: + let sleepDuration = if manager.duration < 5.minutes: 30.seconds else: 1.minutes + + if manager.tests.len > 0: + echo "" + echoStyled styleBright, + bgWhite, fgBlack, &"Integration tests status after {manager.duration}" + + for test in manager.tests: + test.printSummary() + + if manager.tests.len > 0: + echo "" + + await sleepAsync(sleepDuration) + +proc run(test: IntegrationTest) {.async: (raises: []).} = + ignoreCancelled: + let futStart = test.start() + # await futStart + + try: + await futStart.untilTimeout(test.manager.config.testTimeout) + except AsyncTimeoutError: + test.timeEnd = some Moment.now() + test.status = IntegrationTestStatus.Timeout + # futStart will be cancelled by untilTimeout and that will run the + # teardown procedure (in defer) + + test.printResult( + printStdOut = test.status != IntegrationTestStatus.Ok, + printStdErr = + test.status == IntegrationTestStatus.Error or + (test.status == IntegrationTestStatus.Failed and test.output.isErrorLike), + ) + logScope: + name = test.config.name + duration = test.duration + + doAssert test.timeEnd.isSome, "Integration test end time not set!" + doAssert (test.output.isOk and output =? test.output and output != nil) or + test.output.isErr, "Integration test output not set!" + + case test.status + of IntegrationTestStatus.New: + raiseAssert "Test has completed, but is in the New state" + of IntegrationTestStatus.Running: + raiseAssert "Test has completed, but is in the Running state" + of IntegrationTestStatus.Error: + error "Test errored", + exitCode = test.output.option .? exitCode, + error = test.output.errorOption .? msg, + stack = test.output.errorOption .? getStackTrace() + of IntegrationTestStatus.Failed: + error "Test failed", exitCode = test.output.option .? exitCode + of IntegrationTestStatus.Timeout: + error "Test timed out" + of IntegrationTestStatus.Ok: + notice "Test passed" + +proc runTests(manager: TestManager) {.async: (raises: []).} = + var testFutures: seq[Future[void]] + + manager.timeStart = some Moment.now() + + echoStyled styleBright, + bgWhite, fgBlack, "\n[Integration Test Manager] Starting parallel integration tests" + notice "[Integration Test Manager] Starting parallel integration tests", + config = manager.config + + for config in manager.testConfigs: + var test = + IntegrationTest(manager: manager, config: config, testId: $uint16.example) + manager.tests.add test + + let futRun = test.run() + testFutures.add futRun + + try: + await allFutures testFutures + manager.timeEnd = some Moment.now() + except CancelledError as e: + discard + finally: + for fut in testFutures: + await fut.cancelAndWait() + +proc withBorder( + msg: string, align = Align.Left, width = 67, borders = {Border.Left, Border.Right} +): string = + if borders.contains(Border.Left): + result &= "| " + if align == Align.Left: + result &= msg.alignLeft(width) + elif align == Align.Right: + result &= msg.align(width) + if borders.contains(Border.Right): + result &= " |" + +proc printResult(manager: TestManager) = + var successes = 0 + var totalDurationSerial: Duration + echo "" + echoStyled styleBright, styleUnderscore, bgWhite, fgBlack, &"INTEGRATION TESTS RESULT" + + for test in manager.tests: + totalDurationSerial += test.duration + if test.status == IntegrationTestStatus.Ok: + inc successes + # because debug output can really make things hard to read, show a nice + # summary of test results + test.printSummary() + + # estimated time saved as serial execution with a single hardhat instance + # incurs less overhead + let relativeTimeSaved = + ((totalDurationSerial - manager.duration).nanos * 100) div + (totalDurationSerial.nanos) + let passingStyle = if successes < manager.tests.len: fgRed else: fgGreen + + echo "\n▢=====================================================================▢" + echoStyled "| ", + styleBright, + styleUnderscore, + "INTEGRATION TEST SUMMARY", + resetStyle, + "".withBorder(Align.Right, 43, {Border.Right}) + echo "".withBorder() + echoStyled styleBright, + "| TOTAL TIME : ", + resetStyle, + ($manager.duration).withBorder(Align.Right, 49, {Border.Right}) + echoStyled styleBright, + "| TIME SAVED (EST): ", + resetStyle, + (&"{relativeTimeSaved}%").withBorder(Align.Right, 49, {Border.Right}) + echoStyled "| ", + styleBright, + passingStyle, + "PASSING : ", + resetStyle, + passingStyle, + (&"{successes} / {manager.tests.len}").align(49), + resetStyle, + " |" + echo "▢=====================================================================▢" + notice "INTEGRATION TEST SUMMARY", + totalTime = manager.duration, + timeSavedEst = &"{relativeTimeSaved}%", + passing = &"{successes} / {manager.tests.len}" + +proc start*(manager: TestManager) {.async: (raises: [CancelledError]).} = + if manager.config.showContinuousStatusUpdates: + let fut = manager.continuallyShowUpdates() + manager.trackedFutures.track fut + asyncSpawn fut + + let futRunTests = manager.runTests() + manager.trackedFutures.track futRunTests + + await futRunTests + + manager.printResult() + +proc stop*(manager: TestManager) {.async: (raises: []).} = + await manager.trackedFutures.cancelTracked() + + for test in manager.tests: + if not test.process.isNil: + try: + if test.process.running |? false: + discard await noCancel test.process.terminateAndWaitForExit(100.millis) + trace "Terminated running test process", name = test.config.name + except AsyncProcessError, AsyncProcessTimeoutError: + warn "Test process failed to terminate, ignoring...", name = test.config.name + finally: + trace "Closing test process' streams", name = test.config.name + await noCancel test.process.closeWait() + + for hardhat in manager.hardhats: + try: + if not hardhat.process.isNil: + await noCancel hardhat.process.stop() + trace "Terminated running hardhat process" + except CatchableError as e: + trace "failed to stop hardhat node", error = e.msg diff --git a/tests/integration/testmarketplace.nim b/tests/integration/testmarketplace.nim index d66e76133..407d0c7ee 100644 --- a/tests/integration/testmarketplace.nim +++ b/tests/integration/testmarketplace.nim @@ -50,6 +50,14 @@ marketplacesuite "Marketplace": # client requests storage let cid = (await client.upload(data)).get + + var requestStartedFut = Future[void].Raising([CancelledError]).init() + proc onRequestStarted(eventResult: ?!RequestFulfilled) {.raises: [].} = + requestStartedFut.complete() + + let startedSubscription = + await marketplace.subscribe(RequestFulfilled, onRequestStarted) + let id = await client.requestStorage( cid, duration = 20 * 60.uint64, @@ -61,9 +69,9 @@ marketplacesuite "Marketplace": tolerance = ecTolerance, ) - check eventually( - await client.purchaseStateIs(id, "started"), timeout = 10 * 60 * 1000 - ) + # wait for request to start + await requestStartedFut.wait(timeout = chronos.seconds((10 * 60) + 10)) + let purchase = (await client.getPurchase(id)).get check purchase.error == none string let availabilities = (await host.getAvailabilities()).get @@ -75,6 +83,8 @@ marketplacesuite "Marketplace": check reservations.len == 3 check reservations[0].requestId == purchase.requestId + await startedSubscription.unsubscribe() + test "node slots gets paid out and rest of tokens are returned to client", marketplaceConfig: let size = 0xFFFFFF.uint64 @@ -97,6 +107,14 @@ marketplacesuite "Marketplace": # client requests storage let cid = (await client.upload(data)).get + + var requestStartedFut = Future[void].Raising([CancelledError]).init() + proc onRequestStarted(eventResult: ?!RequestFulfilled) {.raises: [].} = + requestStartedFut.complete() + + let startedSubscription = + await marketplace.subscribe(RequestFulfilled, onRequestStarted) + let id = await client.requestStorage( cid, duration = duration, @@ -108,9 +126,8 @@ marketplacesuite "Marketplace": tolerance = ecTolerance, ) - check eventually( - await client.purchaseStateIs(id, "started"), timeout = 10 * 60 * 1000 - ) + await requestStartedFut.wait(timeout = chronos.seconds((10 * 60) + 10)) + let purchase = (await client.getPurchase(id)).get check purchase.error == none string @@ -124,14 +141,15 @@ marketplacesuite "Marketplace": # Checking that the hosting node received reward for at least the time between let slotSize = slotSize(blocks, ecNodes, ecTolerance) let pricePerSlotPerSecond = minPricePerBytePerSecond * slotSize - check eventually (await token.balanceOf(hostAccount)) - startBalanceHost >= + check eventuallySafe (await token.balanceOf(hostAccount)) - startBalanceHost >= (duration - 5 * 60).u256 * pricePerSlotPerSecond * ecNodes.u256 # Checking that client node receives some funds back that were not used for the host nodes - check eventually( + check eventuallySafe( (await token.balanceOf(clientAccount)) - clientBalanceBeforeFinished > 0, timeout = 10 * 1000, # give client a bit of time to withdraw its funds ) + await startedSubscription.unsubscribe() test "SP are able to process slots after workers were busy with other slots and ignored them", NodeConfigs( @@ -178,7 +196,7 @@ marketplacesuite "Marketplace": let requestId = (await client0.client.requestId(purchaseId)).get # We wait that the 3 slots are filled by the first SP - check eventually( + check eventuallySafe( await client0.client.purchaseStateIs(purchaseId, "started"), timeout = 10 * 60.int * 1000, ) @@ -209,10 +227,11 @@ marketplacesuite "Marketplace": check eventually( await client0.client.purchaseStateIs(purchaseId2, "started"), timeout = 10 * 60.int * 1000, + pollInterval = 100, ) # Double check, verify that our second SP hosts the 3 slots - check eventually ((await provider1.client.getSlots()).get).len == 3 + check (await provider1.client.getSlots()).get.len == 3 marketplacesuite "Marketplace payouts": const minPricePerBytePerSecond = 1.u256 @@ -225,23 +244,25 @@ marketplacesuite "Marketplace payouts": NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally hardhat: HardhatConfig.none, - clients: CodexConfigs.init(nodes = 1) - # .debug() # uncomment to enable console log output.debug() - # .withLogFile() - # # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("node", "erasure") - .some, - providers: CodexConfigs.init(nodes = 1) - # .debug() # uncomment to enable console log output - # .withLogFile() - # # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics( - # "node", "marketplace", "sales", "reservations", "node", "statemachine" - # ) - .some, + clients: CodexConfigs + .init(nodes = 1) + .debug() + .withLogFile() + .withLogTopics( + "codex", "codex slots builder", "codex slots sampler", "marketplace", "sales", + "statemachine", "slotqueue", "reservations", "erasure", "ethers", + ).some, + providers: CodexConfigs + .init(nodes = 1) + .debug() + .withLogFile() + .withLogTopics( + "codex", "codex slots builder", "codex slots sampler", "marketplace", "sales", + "statemachine", "slotqueue", "reservations", "erasure", "ethers", + ).some, ): - let duration = 20.periods - let expiry = 10.periods + let duration = 6.periods + let expiry = 4.periods let data = await RandomChunker.example(blocks = blocks) let client = clients()[0] let provider = providers()[0] @@ -251,15 +272,15 @@ marketplacesuite "Marketplace payouts": let startBalanceClient = await token.balanceOf(client.ethAccount) # provider makes storage available - let datasetSize = datasetSize(blocks, ecNodes, ecTolerance) - let totalAvailabilitySize = (datasetSize div 2).truncate(uint64) + let slotSize = slotSize(blocks, ecNodes, ecTolerance) + echo "slotSize: ", $slotSize.truncate(uint64) discard await providerApi.postAvailability( # make availability size small enough that we can't fill all the slots, # thus causing a cancellation - totalSize = totalAvailabilitySize, + totalSize = slotSize.truncate(uint64), duration = duration.uint64, minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = collateralPerByte * totalAvailabilitySize.u256, + totalCollateral = collateralPerByte * slotSize, ) let cid = (await clientApi.upload(data)).get @@ -269,7 +290,15 @@ marketplacesuite "Marketplace payouts": assert not eventResult.isErr slotIdxFilled = some (!eventResult).slotIndex - let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled) + var requestCancelledFut = Future[void].Raising([CancelledError]).init() + proc onRequestCancelled(eventResult: ?!RequestCancelled) {.raises: [].} = + echo "onRequestCancelled ", $eventResult + trace "onRequestCancelled", eventResult + requestCancelledFut.complete() + + let cancelledSubscription = + await marketplace.subscribe(RequestCancelled, onRequestCancelled) + let filledSubscription = await marketplace.subscribe(SlotFilled, onSlotFilled) # client requests storage but requires multiple slots to host the content let id = await clientApi.requestStorage( @@ -286,16 +315,17 @@ marketplacesuite "Marketplace payouts": check eventually(slotIdxFilled.isSome, timeout = expiry.int * 1000) let slotId = slotId(!(await clientApi.requestId(id)), !slotIdxFilled) + # check eventually( + # await providerApi.saleStateIs(slotId, "SaleCancelled"), timeout = expiry.int * 1000, pollInterval = 100 + # ) + # wait until sale is cancelled await ethProvider.advanceTime(expiry.u256) - check eventually( - await providerApi.saleStateIs(slotId, "SaleCancelled"), pollInterval = 100 - ) + await requestCancelledFut.wait(timeout = chronos.seconds(expiry.int + 10)) await advanceToNextPeriod() - let slotSize = slotSize(blocks, ecNodes, ecTolerance) let pricePerSlotPerSecond = minPricePerBytePerSecond * slotSize check eventually ( @@ -313,7 +343,8 @@ marketplacesuite "Marketplace payouts": timeout = 10 * 1000, # give client a bit of time to withdraw its funds ) - await subscription.unsubscribe() + await filledSubscription.unsubscribe() + await cancelledSubscription.unsubscribe() test "the collateral is returned after a sale is ignored", NodeConfigs( @@ -380,6 +411,7 @@ marketplacesuite "Marketplace payouts": check eventually( await client0.client.purchaseStateIs(purchaseId, "started"), timeout = 10 * 60.int * 1000, + pollInterval = 100, ) # Here we will check that for each provider, the total remaining collateral @@ -398,4 +430,5 @@ marketplacesuite "Marketplace payouts": availability.totalRemainingCollateral == availableSlots * slotSize * minPricePerBytePerSecond, timeout = 30 * 1000, + pollInterval = 100, ) diff --git a/tests/integration/testproofs.nim b/tests/integration/testproofs.nim index c49b7b6f6..26fe59212 100644 --- a/tests/integration/testproofs.nim +++ b/tests/integration/testproofs.nim @@ -51,6 +51,13 @@ marketplacesuite "Hosts submit regular proofs": let cid = (await client0.upload(data)).get + var requestStartedFut = Future[void].Raising([CancelledError]).init() + proc onRequestStarted(eventResult: ?!RequestFulfilled) {.raises: [].} = + requestStartedFut.complete() + + let startedSubscription = + await marketplace.subscribe(RequestFulfilled, onRequestStarted) + let purchaseId = await client0.requestStorage( cid, expiry = expiry, @@ -64,19 +71,19 @@ marketplacesuite "Hosts submit regular proofs": let slotSize = slotSize(blocks, ecNodes, ecTolerance) - check eventually( - await client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000 - ) + await requestStartedFut.wait(timeout = chronos.seconds(10.periods.int + 10)) var proofWasSubmitted = false proc onProofSubmitted(event: ?!ProofSubmitted) = proofWasSubmitted = event.isOk - let subscription = await marketplace.subscribe(ProofSubmitted, onProofSubmitted) + let proofSubmittedSubscription = + await marketplace.subscribe(ProofSubmitted, onProofSubmitted) check eventually(proofWasSubmitted, timeout = (duration - expiry).int * 1000) - await subscription.unsubscribe() + await proofSubmittedSubscription.unsubscribe() + await startedSubscription.unsubscribe() marketplacesuite "Simulate invalid proofs": # TODO: these are very loose tests in that they are not testing EXACTLY how @@ -113,7 +120,7 @@ marketplacesuite "Simulate invalid proofs": .some, ): let client0 = clients()[0].client - let expiry = 10.periods + let expiry = 15.periods let duration = expiry + 10.periods let data = await RandomChunker.example(blocks = blocks) @@ -140,7 +147,7 @@ marketplacesuite "Simulate invalid proofs": ) let requestId = (await client0.requestId(purchaseId)).get - check eventually( + check eventuallySafe( await client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000 ) diff --git a/tests/integration/testpurchasing.nim b/tests/integration/testpurchasing.nim index ba8dd1907..227df02ea 100644 --- a/tests/integration/testpurchasing.nim +++ b/tests/integration/testpurchasing.nim @@ -95,13 +95,17 @@ twonodessuite "Purchasing": ) ).get check eventually( - await client1.purchaseStateIs(id, "submitted"), timeout = 3 * 60 * 1000 + await client1.purchaseStateIs(id, "submitted"), + timeout = 3 * 60 * 1000, + pollInterval = 100, ) await node1.restart() check eventually( - await client1.purchaseStateIs(id, "submitted"), timeout = 3 * 60 * 1000 + await client1.purchaseStateIs(id, "submitted"), + timeout = 3 * 60 * 1000, + pollInterval = 100, ) let request = (await client1.getPurchase(id)).get.request.get check request.ask.duration == (10 * 60).uint64 diff --git a/tests/integration/testsales.nim b/tests/integration/testsales.nim index ef9999908..66287bdb7 100644 --- a/tests/integration/testsales.nim +++ b/tests/integration/testsales.nim @@ -8,6 +8,7 @@ import ../contracts/time import ./codexconfig import ./codexclient import ./nodeconfigs +import ./marketplacesuite proc findItem[T](items: seq[T], item: T): ?!T = for tmp in items: @@ -16,7 +17,7 @@ proc findItem[T](items: seq[T], item: T): ?!T = return failure("Not found") -multinodesuite "Sales": +marketplacesuite "Sales": let salesConfig = NodeConfigs( clients: CodexConfigs.init(nodes = 1).some, providers: CodexConfigs.init(nodes = 1) @@ -128,22 +129,27 @@ multinodesuite "Sales": # Lets create storage request that will utilize some of the availability's space let cid = (await client.upload(data)).get - let id = ( - await client.requestStorage( - cid, - duration = 20 * 60.uint64, - pricePerBytePerSecond = minPricePerBytePerSecond, - proofProbability = 3.u256, - expiry = (10 * 60).uint64, - collateralPerByte = collateralPerByte, - nodes = 3, - tolerance = 1, - ) - ).get - check eventually( - await client.purchaseStateIs(id, "started"), timeout = 10 * 60 * 1000 + var requestStartedFut = Future[void].Raising([CancelledError]).init() + proc onRequestStarted(eventResult: ?!RequestFulfilled) {.raises: [].} = + requestStartedFut.complete() + + let startedSubscription = + await marketplace.subscribe(RequestFulfilled, onRequestStarted) + + let id = await client.requestStorage( + cid, + duration = 20 * 60.uint64, + pricePerBytePerSecond = minPricePerBytePerSecond, + proofProbability = 3.u256, + expiry = (10 * 60).uint64, + collateralPerByte = collateralPerByte, + nodes = 3, + tolerance = 1, ) + + await requestStartedFut.wait(timeout = chronos.seconds((10 * 60) + 10)) + let updatedAvailability = ((await host.getAvailabilities()).get).findItem(availability).get check updatedAvailability.totalSize != updatedAvailability.freeSize @@ -165,6 +171,7 @@ multinodesuite "Sales": ((await host.getAvailabilities()).get).findItem(availability).get check newUpdatedAvailability.totalSize == originalSize + 20000 check newUpdatedAvailability.freeSize - updatedAvailability.freeSize == 20000 + await startedSubscription.unsubscribe() test "updating availability fails with until negative", salesConfig: let availability = ( @@ -202,6 +209,13 @@ multinodesuite "Sales": ) ).get + var requestStartedFut = Future[void].Raising([CancelledError]).init() + proc onRequestStarted(eventResult: ?!RequestFulfilled) {.raises: [].} = + requestStartedFut.complete() + + let startedSubscription = + await marketplace.subscribe(RequestFulfilled, onRequestStarted) + # client requests storage let cid = (await client.upload(data)).get let id = ( @@ -217,9 +231,8 @@ multinodesuite "Sales": ) ).get - check eventually( - await client.purchaseStateIs(id, "started"), timeout = 10 * 60 * 1000 - ) + await requestStartedFut.wait(timeout = chronos.seconds((10 * 60) + 10)) + let purchase = (await client.getPurchase(id)).get check purchase.error == none string @@ -234,3 +247,5 @@ multinodesuite "Sales": response.status == 422 (await response.body) == "Until parameter must be greater or equal to the longest currently hosted slot" + + await startedSubscription.unsubscribe() diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 0d1a50e80..222f1ba79 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -61,12 +61,12 @@ marketplacesuite "Validation": NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally hardhat: HardhatConfig.none, - clients: CodexConfigs - .init(nodes = 1) - # .debug() # uncomment to enable console log output - .withLogFile() - # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("purchases", "onchain").some, + clients: CodexConfigs.init(nodes = 1) + # .debug() # uncomment to enable console log output + # .withLogFile() + # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("purchases", "onchain") + .some, providers: CodexConfigs .init(nodes = 1) .withSimulateProofFailures(idx = 0, failEveryNProofs = 1) @@ -80,9 +80,9 @@ marketplacesuite "Validation": .withValidationGroupIndex(idx = 0, groupIndex = 0) .withValidationGroupIndex(idx = 1, groupIndex = 1) # .debug() # uncomment to enable console log output - .withLogFile() + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("validator") + # .withLogTopics("validator") # each topic as a separate string argument .some, ): @@ -142,12 +142,12 @@ marketplacesuite "Validation": NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally hardhat: HardhatConfig.none, - clients: CodexConfigs - .init(nodes = 1) - # .debug() # uncomment to enable console log output - .withLogFile() - # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("purchases", "onchain").some, + clients: CodexConfigs.init(nodes = 1) + # .debug() # uncomment to enable console log output + # .withLogFile() + # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("purchases", "onchain") + .some, providers: CodexConfigs .init(nodes = 1) .withSimulateProofFailures(idx = 0, failEveryNProofs = 1) diff --git a/tests/integration/utils.nim b/tests/integration/utils.nim new file mode 100644 index 000000000..3e522a042 --- /dev/null +++ b/tests/integration/utils.nim @@ -0,0 +1,92 @@ +import std/os +import std/strformat +import pkg/chronos +import pkg/chronos/asyncproc +import pkg/codex/logutils + +{.push raises: [].} + +proc nextFreePort*(startPort: int): Future[int] {.async: (raises: [CancelledError]).} = + proc client(server: StreamServer, transp: StreamTransport) {.async: (raises: []).} = + await transp.closeWait() + + var port = startPort + while true: + trace "checking if port is free", port + try: + let host = initTAddress("127.0.0.1", port) + # We use ReuseAddr here only to be able to reuse the same IP/Port when + # there's a TIME_WAIT socket. It's useful when running the test multiple + # times or if a test ran previously using the same port. + var server = createStreamServer(host, client, {ReuseAddr}) + trace "port is free", port + await server.closeWait() + return port + except TransportOsError: + trace "port is not free", port + inc port + except TransportAddressError: + raiseAssert "bad address" + +proc sanitize*(pathSegment: string): string = + var sanitized = pathSegment + for invalid in invalidFilenameChars.items: + sanitized = sanitized.replace(invalid, '_').replace(' ', '_') + sanitized + +proc getLogFile*( + logDir, startTime, suiteName, testName, role: string, index = int.none +): string {.raises: [IOError, OSError].} = + let logsDir = + if logDir == "": + currentSourcePath.parentDir() / "logs" / sanitize(startTime & "__" & suiteName) / + sanitize(testName) + else: + logDir / sanitize(suiteName) / sanitize(testName) + + createDir(logsDir) + + var fn = $role + if idx =? index: + fn &= "_" & $idx + fn &= ".log" + + let fileName = logsDir / fn + return fileName + +proc appendFile*(filename: string, content: string) {.raises: [IOError].} = + ## Opens a file named `filename` for writing. Then writes the + ## `content` completely to the file and closes the file afterwards. + ## Raises an IO exception in case of an error. + var f: File + try: + f = open(filename, fmAppend) + f.write(content) + except IOError as e: + raise newException(IOError, "cannot open and write " & filename & ": " & e.msg) + finally: + close(f) + +when defined(windows): + proc forceKillProcess*( + processName, matchingCriteria: string + ): Future[CommandExResponse] {. + async: ( + raises: [ + AsyncProcessError, AsyncProcessTimeoutError, CancelledError, ValueError, + OSError, + ] + ) + .} = + let path = splitFile(currentSourcePath()).dir / "scripts" / "winkillprocess.sh" + let cmd = &"{absolutePath(path)} kill {processName} \"{matchingCriteria}\"" + trace "Forcefully killing windows process", processName, matchingCriteria, cmd + return await execCommandEx(cmd, timeout = 5.seconds) + +proc getDataDir*(testId, testName, startTime, role: string, index = int.none): string = + var suffix = role + if idx =? index: + suffix &= "_" & $idx + + getTempDir() / "Codex" / sanitize(testId) / sanitize(testName) / sanitize(startTime) / + sanitize(suffix) diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index 152d22dde..b22534270 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -1,13 +1,127 @@ -import ./integration/testcli -import ./integration/testrestapi -import ./integration/testrestapivalidation -import ./integration/testupdownload -import ./integration/testsales -import ./integration/testpurchasing -import ./integration/testblockexpiration -import ./integration/testmarketplace -import ./integration/testproofs -import ./integration/testvalidator -import ./integration/testecbug +import std/os +import std/strformat +import std/terminal +from std/times import format, now +import std/terminal +import std/typetraits +import pkg/chronos +import pkg/codex/conf +import pkg/codex/logutils +import ./integration/testmanager +import ./integration/utils {.warning[UnusedImport]: off.} +{.push raises: [].} + +const TestConfigs = + @[ + IntegrationTestConfig.init("./integration/testcli", startHardhat = true), + IntegrationTestConfig.init("./integration/testrestapi", startHardhat = true), + IntegrationTestConfig.init("./integration/testupdownload", startHardhat = true), + IntegrationTestConfig.init("./integration/testsales", startHardhat = true), + IntegrationTestConfig.init("./integration/testpurchasing", startHardhat = true), + IntegrationTestConfig.init("./integration/testblockexpiration", startHardhat = true), + IntegrationTestConfig.init("./integration/testmarketplace", startHardhat = true), + IntegrationTestConfig.init("./integration/testproofs", startHardhat = true), + IntegrationTestConfig.init("./integration/testvalidator", startHardhat = true), + IntegrationTestConfig.init("./integration/testecbug", startHardhat = true), + IntegrationTestConfig.init( + "./integration/testrestapivalidation", startHardhat = true + ), + ] + +# Echoes stdout from Hardhat process +const DebugHardhat {.booldefine.} = false +# When true, shows all TRACE logs in Codex nodes' chronicles logs +const NoCodexLogFilters {.booldefine.} = false +# Shows test status updates at time intervals. Useful for running locally with +# active terminal interaction. Set to false for unattended runs, eg CI. +const ShowContinuousStatusUpdates {.booldefine.} = false +# Timeout duration (in minutes) for EACH integration test file. +const TestTimeout {.intdefine.} = 60 + +const EnableParallelTests {.booldefine.} = true + +proc setupLogging(logFile: string) = + try: + let success = defaultChroniclesStream.outputs[0].open(logFile, fmAppend) + doAssert success, "Failed to open log file: " & logFile + except IOError, OSError: + let error = getCurrentException() + fatal "Failed to open log file", error = error.msg + raiseAssert "Could not open test manager log file: " & error.msg + +proc run(): Future[bool] {.async: (raises: []).} = + let startTime = now().format("yyyy-MM-dd'_'HH-mm-ss") + let logsDir = + currentSourcePath.parentDir() / "integration" / "logs" / + sanitize(startTime & "-IntegrationTests") + try: + createDir(logsDir) + #!fmt: off + styledEcho bgWhite, fgBlack, styleBright, + "\n\n ", + styleUnderscore, + "ℹ️ LOGS AVAILABLE ℹ️\n\n", + resetStyle, bgWhite, fgBlack, styleBright, + """ Logs for this run will be available at:""", + resetStyle, bgWhite, fgBlack, + &"\n\n {logsDir}\n\n", + resetStyle, bgWhite, fgBlack, styleBright, + " NOTE: For CI runs, logs will be attached as artefacts\n" + #!fmt: on + except IOError as e: + raiseAssert "Failed to create log directory and echo log message: " & e.msg + except OSError as e: + raiseAssert "Failed to create log directory and echo log message: " & e.msg + + setupLogging(TestManager.logFile(logsDir)) + + let manager = TestManager.new( + config = TestManagerConfig( + debugHardhat: DebugHardhat, + noCodexLogFilters: NoCodexLogFilters, + showContinuousStatusUpdates: ShowContinuousStatusUpdates, + logsDir: logsDir, + testTimeout: TestTimeout.minutes, + ), + testConfigs = TestConfigs, + ) + try: + trace "starting test manager" + await manager.start() + except TestManagerError as e: + error "Failed to run test manager", error = e.msg + return false + except CancelledError: + return false + finally: + trace "Stopping test manager" + await manager.stop() + trace "Test manager stopped" + + without wasSuccessful =? manager.allTestsPassed, error: + raiseAssert "Failed to get test status: " & error.msg + + return wasSuccessful + +when isMainModule: + when EnableParallelTests: + let wasSuccessful = waitFor run() + if wasSuccessful: + quit(QuitSuccess) + else: + quit(QuitFailure) # indicate with a non-zero exit code that the tests failed + else: + # run tests serially + import ./integration/testcli + import ./integration/testrestapi + import ./integration/testupdownload + import ./integration/testsales + import ./integration/testpurchasing + import ./integration/testblockexpiration + import ./integration/testmarketplace + import ./integration/testproofs + import ./integration/testvalidator + import ./integration/testecbug + import ./integration/testrestapivalidation diff --git a/vendor/circom-witnessgen b/vendor/circom-witnessgen new file mode 160000 index 000000000..1291cf0e6 --- /dev/null +++ b/vendor/circom-witnessgen @@ -0,0 +1 @@ +Subproject commit 1291cf0e62ddd829b7be14cccc301af82e033391 diff --git a/vendor/codex-contracts-eth b/vendor/codex-contracts-eth index aee91f1ac..de392769e 160000 --- a/vendor/codex-contracts-eth +++ b/vendor/codex-contracts-eth @@ -1 +1 @@ -Subproject commit aee91f1ac411258af338af5145e0112e6ab6f5df +Subproject commit de392769ec051e488aa7898aa2cd704823bcec26 diff --git a/vendor/nim-chronos b/vendor/nim-chronos index c04576d82..0646c444f 160000 --- a/vendor/nim-chronos +++ b/vendor/nim-chronos @@ -1 +1 @@ -Subproject commit c04576d829b8a0a1b12baaa8bc92037501b3a4a0 +Subproject commit 0646c444fce7c7ed08ef6f2c9a7abfd172ffe655 diff --git a/vendor/nim-goldilocks-hash b/vendor/nim-goldilocks-hash new file mode 160000 index 000000000..3c5a2bea1 --- /dev/null +++ b/vendor/nim-goldilocks-hash @@ -0,0 +1 @@ +Subproject commit 3c5a2bea154b0712ac9576c17dc4d92c00fe003e diff --git a/vendor/nim-groth16 b/vendor/nim-groth16 new file mode 160000 index 000000000..434170541 --- /dev/null +++ b/vendor/nim-groth16 @@ -0,0 +1 @@ +Subproject commit 434170541e154bcac37a3306d835d2dfe1e81549