From efb877e9bf475874d11825ab3d76781dac854741 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Tue, 15 Jul 2025 17:50:37 +0700 Subject: [PATCH 01/16] save --- core/vm/interpreter.go | 4 ++++ turbo/transactions/tracing.go | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 7fa942585d1..6025136f912 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -240,6 +240,10 @@ func NewEVMInterpreter(evm *EVM, cfg Config) *EVMInterpreter { // considered a revert-and-consume-all-gas operation except for // ErrExecutionReverted which means revert-and-keep-gas-left. func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (ret []byte, err error) { + if in.evm.abort.Load() { + return ret, nil + } + // Don't bother with the execution if there's no code. if len(contract.Code) == 0 { return nil, nil diff --git a/turbo/transactions/tracing.go b/turbo/transactions/tracing.go index c58b0496649..929ce52175d 100644 --- a/turbo/transactions/tracing.go +++ b/turbo/transactions/tracing.go @@ -177,9 +177,11 @@ func AssembleTracer( return tracer, false, cancel, nil case config == nil: - return logger.NewJsonStreamLogger(nil, ctx, stream).Tracer(), true, func() {}, nil + ctx, cancel := context.WithTimeout(ctx, callTimeout) + return logger.NewJsonStreamLogger(nil, ctx, stream).Tracer(), true, cancel, nil default: - return logger.NewJsonStreamLogger(config.LogConfig, ctx, stream).Tracer(), true, func() {}, nil + ctx, cancel := context.WithTimeout(ctx, callTimeout) + return logger.NewJsonStreamLogger(config.LogConfig, ctx, stream).Tracer(), true, cancel, nil } } From fc0ef764e90d35512eed9ec97b727ad9d00de3bd Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Tue, 15 Jul 2025 17:50:37 +0700 Subject: [PATCH 02/16] save --- core/state_processor.go | 19 +++ eth/protocols/eth/handler_test.go | 147 ++++++++++++++++++ execution/stages/mock/mock_sentry.go | 2 +- .../sentry_multi_client.go | 2 +- rpc/jsonrpc/eth_api.go | 2 +- rpc/jsonrpc/receipts/receipts_generator.go | 37 ++++- 6 files changed, 204 insertions(+), 5 deletions(-) create mode 100644 eth/protocols/eth/handler_test.go diff --git a/core/state_processor.go b/core/state_processor.go index 6706ce1f6e0..3ec7831e539 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -126,3 +126,22 @@ func ApplyTransaction(config *chain.Config, blockHashFunc func(n uint64) (common return applyTransaction(config, engine, gp, ibs, stateWriter, header, txn, gasUsed, usedBlobGas, vmenv, cfg) } + +func CreateEVM(config *chain.Config, blockHashFunc func(n uint64) libcommon.Hash, engine consensus.EngineReader, author *libcommon.Address, ibs *state.IntraBlockState, header *types.Header, cfg vm.Config) *vm.EVM { + // Create a new context to be used in the EVM environment + + // Add addresses to access list if applicable + // about the transaction and calling mechanisms. + cfg.SkipAnalysis = SkipAnalysis(config, header.Number.Uint64()) + + blockContext := NewEVMBlockContext(header, blockHashFunc, engine, author, config) + return vm.NewEVM(blockContext, evmtypes.TxContext{}, ibs, config, cfg) +} + +func ApplyTransactionWithEVM(config *chain.Config, engine consensus.EngineReader, gp *GasPool, + ibs *state.IntraBlockState, + stateWriter state.StateWriter, header *types.Header, txn types.Transaction, usedGas, usedBlobGas *uint64, + cfg vm.Config, vmenv *vm.EVM, +) (*types.Receipt, []byte, error) { + return applyTransaction(config, engine, gp, ibs, stateWriter, header, txn, usedGas, usedBlobGas, vmenv, cfg) +} diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go new file mode 100644 index 00000000000..7cf7bbbdfe1 --- /dev/null +++ b/eth/protocols/eth/handler_test.go @@ -0,0 +1,147 @@ +// Copyright 2015 The go-ethereum Authors +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package eth_test + +import ( + "math/big" + "testing" + "time" + + "github.com/holiman/uint256" + "github.com/stretchr/testify/require" + + libcommon "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/crypto" + "github.com/erigontech/erigon-lib/direct" + sentry "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" + "github.com/erigontech/erigon-lib/rlp" + "github.com/erigontech/erigon/core" + "github.com/erigontech/erigon/core/rawdb" + "github.com/erigontech/erigon/core/types" + "github.com/erigontech/erigon/eth/protocols/eth" + "github.com/erigontech/erigon/params" + "github.com/erigontech/erigon/turbo/jsonrpc/receipts" + "github.com/erigontech/erigon/turbo/stages/mock" +) + +var ( + // testKey is a private key to use for funding a tester account. + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + + // testAddr is the Ethereum address of the tester account. + testAddr = crypto.PubkeyToAddress(testKey.PublicKey) +) + +func TestGetBlockReceipts(t *testing.T) { + // Define three accounts to simulate transactions with + acc1Key, _ := crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + acc2Key, _ := crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + acc1Addr := crypto.PubkeyToAddress(acc1Key.PublicKey) + acc2Addr := crypto.PubkeyToAddress(acc2Key.PublicKey) + + signer := types.LatestSignerForChainID(nil) + // Create a chain generator with some simple transactions (blatantly stolen from @fjl/chain_markets_test) + generator := func(i int, block *core.BlockGen) { + switch i { + case 0: + // In block 1, the test bank sends account #1 some ether. + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), acc1Addr, uint256.NewInt(10000), params.TxGas, nil, nil), *signer, testKey) + block.AddTx(tx) + case 1: + // In block 2, the test bank sends some more ether to account #1. + // acc1Addr passes it on to account #2. + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), acc1Addr, uint256.NewInt(1000), params.TxGas, nil, nil), *signer, testKey) + tx2, _ := types.SignTx(types.NewTransaction(block.TxNonce(acc1Addr), acc2Addr, uint256.NewInt(1000), params.TxGas, nil, nil), *signer, acc1Key) + block.AddTx(tx1) + block.AddTx(tx2) + case 2: + // Block 3 is empty but was mined by account #2. + block.SetCoinbase(acc2Addr) + block.SetExtra([]byte("yeehaw")) + case 3: + // Block 4 includes blocks 2 and 3 as uncle headers (with modified extra data). + b2 := block.PrevBlock(1).Header() + b2.Extra = []byte("foo") + block.AddUncle(b2) + b3 := block.PrevBlock(2).Header() + b3.Extra = []byte("foo") + block.AddUncle(b3) + } + } + // Assemble the test environment + m := mockWithGenerator(t, 4, generator) + receiptsGetter := receipts.NewGenerator(m.BlockReader, m.Engine, time.Minute) + // Collect the hashes to request, and the response to expect + var ( + hashes []libcommon.Hash + receipts []rlp.RawValue + ) + tx, err := m.DB.BeginTemporalRo(m.Ctx) + require.NoError(t, err) + defer tx.Rollback() + + for i := uint64(0); i <= rawdb.ReadCurrentHeader(tx).Number.Uint64(); i++ { + block, err := m.BlockReader.BlockByNumber(m.Ctx, tx, i) + require.NoError(t, err) + + hashes = append(hashes, block.Hash()) + // If known, encode and queue for response packet + + r, err := receiptsGetter.GetReceipts(m.Ctx, m.ChainConfig, tx, block) + require.NoError(t, err) + encoded, err := rlp.EncodeToBytes(r) + require.NoError(t, err) + receipts = append(receipts, encoded) + } + + require.NoError(t, err) + b, err := rlp.EncodeToBytes(eth.GetReceiptsPacket66{RequestId: 1, GetReceiptsPacket: hashes}) + require.NoError(t, err) + + m.StreamWg.Wait() + + m.ReceiveWg.Add(1) + // Send the hash request and verify the response + for _, err = range m.Send(&sentry.InboundMessage{Id: eth.ToProto[direct.ETH67][eth.GetReceiptsMsg], Data: b, PeerId: m.PeerId}) { + require.NoError(t, err) + } + + expect, err := rlp.EncodeToBytes(eth.ReceiptsRLPPacket66{RequestId: 1, ReceiptsRLPPacket: receipts}) + require.NoError(t, err) + m.ReceiveWg.Wait() + sent := m.SentMessage(0) + require.Equal(t, eth.ToProto[m.SentryClient.Protocol()][eth.ReceiptsMsg], sent.Id) + require.Equal(t, expect, sent.Data) +} + +// newTestBackend creates a chain with a number of explicitly defined blocks and +// wraps it into a mock backend. +func mockWithGenerator(t *testing.T, blocks int, generator func(int, *core.BlockGen)) *mock.MockSentry { + m := mock.MockWithGenesis(t, &types.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, + }, testKey, false) + if blocks > 0 { + chain, _ := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, blocks, generator) + err := m.InsertChain(chain) + require.NoError(t, err) + } + return m +} diff --git a/execution/stages/mock/mock_sentry.go b/execution/stages/mock/mock_sentry.go index 451050a22fc..8f7d8fdbf96 100644 --- a/execution/stages/mock/mock_sentry.go +++ b/execution/stages/mock/mock_sentry.go @@ -314,7 +314,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK PeerId: gointerfaces.ConvertHashToH512([64]byte{0x12, 0x34, 0x50}), // "12345" BlockSnapshots: allSnapshots, BlockReader: br, - ReceiptsReader: receipts.NewGenerator(br, engine), + ReceiptsReader: receipts.NewGenerator(br, engine, 5*time.Second), HistoryV3: true, cfg: cfg, } diff --git a/p2p/sentry/sentry_multi_client/sentry_multi_client.go b/p2p/sentry/sentry_multi_client/sentry_multi_client.go index 74429fb7520..6e937d0de06 100644 --- a/p2p/sentry/sentry_multi_client/sentry_multi_client.go +++ b/p2p/sentry/sentry_multi_client/sentry_multi_client.go @@ -224,7 +224,7 @@ func NewMultiClient( disableBlockDownload: disableBlockDownload, logger: logger, getReceiptsActiveGoroutineNumber: semaphore.NewWeighted(1), - ethApiWrapper: receipts.NewGenerator(blockReader, engine), + ethApiWrapper: receipts.NewGenerator(blockReader, engine, 30*time.Second), } return cs, nil diff --git a/rpc/jsonrpc/eth_api.go b/rpc/jsonrpc/eth_api.go index 79389136e7e..4df4a3a0ca5 100644 --- a/rpc/jsonrpc/eth_api.go +++ b/rpc/jsonrpc/eth_api.go @@ -170,7 +170,7 @@ func NewBaseApi(f *rpchelper.Filters, stateCache kvcache.Cache, blockReader serv _txNumReader: blockReader.TxnumReader(context.Background()), evmCallTimeout: evmCallTimeout, _engine: engine, - receiptsGenerator: receipts.NewGenerator(blockReader, engine), + receiptsGenerator: receipts.NewGenerator(blockReader, engine, evmCallTimeout), borReceiptGenerator: receipts.NewBorGenerator(blockReader, engine), dirs: dirs, useBridgeReader: bridgeReader != nil && !reflect.ValueOf(bridgeReader).IsNil(), // needed for interface nil caveat diff --git a/rpc/jsonrpc/receipts/receipts_generator.go b/rpc/jsonrpc/receipts/receipts_generator.go index 2c54b5ccb8a..4f9c70356ac 100644 --- a/rpc/jsonrpc/receipts/receipts_generator.go +++ b/rpc/jsonrpc/receipts/receipts_generator.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "sync" + "time" "github.com/erigontech/erigon-db/rawdb" "github.com/erigontech/erigon-db/rawdb/rawtemporaldb" @@ -39,6 +40,7 @@ type Generator struct { receiptsCacheTrace bool receiptCacheTrace bool + evmTimeout time.Duration blockReader services.FullBlockReader txNumReader rawdbv3.TxNumsReader @@ -60,7 +62,7 @@ var ( receiptsCacheTrace = dbg.EnvBool("R_LRU_TRACE", false) ) -func NewGenerator(blockReader services.FullBlockReader, engine consensus.EngineReader) *Generator { +func NewGenerator(blockReader services.FullBlockReader, engine consensus.EngineReader, evmTimeout time.Duration) *Generator { receiptsCache, err := lru.New[common.Hash, types.Receipts](receiptsCacheLimit) //TODO: is handling both of them a good idea though...? if err != nil { panic(err) @@ -81,6 +83,7 @@ func NewGenerator(blockReader services.FullBlockReader, engine consensus.EngineR receiptsCacheTrace: receiptsCacheTrace, receiptCacheTrace: receiptsCacheTrace, receiptCache: receiptCache, + evmTimeout: evmTimeout, blockExecMutex: &loaderMutex[common.Hash]{}, txnExecMutex: &loaderMutex[common.Hash]{}, @@ -216,6 +219,19 @@ func (g *Generator) GetReceipt(ctx context.Context, cfg *chain.Config, tx kv.Tem return nil, err } + //evm := core.CreateEVM(cfg, core.GetHashFn(genEnv.header, genEnv.getHeader), g.engine, nil, genEnv.ibs, genEnv.header, vm.Config{}) + //ctx, cancel := context.WithTimeout(ctx, g.evmTimeout) + //defer cancel() + //go func() { + // <-ctx.Done() + // evm.Cancel() + //}() + // + //receipt, _, err = core.ApplyTransactionWithEVM(cfg, g.engine, genEnv.gp, genEnv.ibs, genEnv.noopWriter, genEnv.header, txn, genEnv.usedGas, genEnv.usedBlobGas, vm.Config{}, evm) + //if err != nil { + // return nil, fmt.Errorf("ReceiptGen.GetReceipt: bn=%d, txnIdx=%d, %w", blockNum, index, err) + //} + status, gasUsed, err := aa.ExecuteAATransaction(aaTxn, paymasterContext, validationGasUsed, genEnv.gp, evm, header, genEnv.ibs) if err != nil { return nil, err @@ -288,9 +304,26 @@ func (g *Generator) GetReceipts(ctx context.Context, cfg *chain.Config, tx kv.Te //genEnv.ibs.SetTrace(true) blockNum := block.NumberU64() + evm := core.CreateEVM(cfg, core.GetHashFn(genEnv.header, genEnv.getHeader), g.engine, nil, genEnv.ibs, genEnv.header, vm.Config{}) + ctx, cancel := context.WithTimeout(ctx, g.evmTimeout) + defer cancel() + go func() { + <-ctx.Done() + evm.Cancel() + }() + for i, txn := range block.Transactions() { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + evm2 := core.CreateEVM(cfg, core.GetHashFn(genEnv.header, genEnv.getHeader), g.engine, nil, genEnv.ibs, genEnv.header, vm.Config{}) + evm2.JumpDestCache = evm.JumpDestCache + evm = evm2 genEnv.ibs.SetTxContext(blockNum, i) - receipt, _, err := core.ApplyTransaction(cfg, core.GetHashFn(genEnv.header, genEnv.getHeader), g.engine, nil, genEnv.gp, genEnv.ibs, genEnv.noopWriter, genEnv.header, txn, genEnv.gasUsed, genEnv.usedBlobGas, vm.Config{}) + receipt, _, err := core.ApplyTransactionWithEVM(cfg, g.engine, genEnv.gp, genEnv.ibs, genEnv.noopWriter, genEnv.header, txn, genEnv.usedGas, genEnv.usedBlobGas, vm.Config{}, evm) if err != nil { return nil, fmt.Errorf("ReceiptGen.GetReceipts: bn=%d, txnIdx=%d, %w", block.NumberU64(), i, err) } From 939b77054b82a6230b2f2ae21cf0a534a9aa8fac Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 15 Jul 2025 18:18:12 +0700 Subject: [PATCH 03/16] save --- core/state_processor.go | 2 +- rpc/jsonrpc/receipts/receipts_generator.go | 20 +++++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/core/state_processor.go b/core/state_processor.go index 3ec7831e539..e30a554f39f 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -127,7 +127,7 @@ func ApplyTransaction(config *chain.Config, blockHashFunc func(n uint64) (common return applyTransaction(config, engine, gp, ibs, stateWriter, header, txn, gasUsed, usedBlobGas, vmenv, cfg) } -func CreateEVM(config *chain.Config, blockHashFunc func(n uint64) libcommon.Hash, engine consensus.EngineReader, author *libcommon.Address, ibs *state.IntraBlockState, header *types.Header, cfg vm.Config) *vm.EVM { +func CreateEVM(config *chain.Config, blockHashFunc func(n uint64) (common.Hash, error), engine consensus.EngineReader, author *common.Address, ibs *state.IntraBlockState, header *types.Header, cfg vm.Config) *vm.EVM { // Create a new context to be used in the EVM environment // Add addresses to access list if applicable diff --git a/rpc/jsonrpc/receipts/receipts_generator.go b/rpc/jsonrpc/receipts/receipts_generator.go index 4f9c70356ac..d734823ba28 100644 --- a/rpc/jsonrpc/receipts/receipts_generator.go +++ b/rpc/jsonrpc/receipts/receipts_generator.go @@ -240,7 +240,15 @@ func (g *Generator) GetReceipt(ctx context.Context, cfg *chain.Config, tx kv.Tem logs := genEnv.ibs.GetLogs(genEnv.ibs.TxnIndex(), txn.Hash(), header.Number.Uint64(), header.Hash()) receipt = aa.CreateAAReceipt(txn.Hash(), status, gasUsed, header.GasUsed, header.Number.Uint64(), uint64(genEnv.ibs.TxnIndex()), logs) } else { - receipt, _, err = core.ApplyTransaction(cfg, core.GetHashFn(genEnv.header, genEnv.getHeader), g.engine, nil, genEnv.gp, genEnv.ibs, genEnv.noopWriter, genEnv.header, txn, genEnv.gasUsed, genEnv.usedBlobGas, vm.Config{}) + evm := core.CreateEVM(cfg, core.GetHashFn(genEnv.header, genEnv.getHeader), g.engine, nil, genEnv.ibs, genEnv.header, vm.Config{}) + ctx, cancel := context.WithTimeout(ctx, g.evmTimeout) + defer cancel() + go func() { + <-ctx.Done() + evm.Cancel() + }() + + receipt, _, err = core.ApplyTransactionWithEVM(cfg, g.engine, genEnv.gp, genEnv.ibs, genEnv.noopWriter, genEnv.header, txn, genEnv.gasUsed, genEnv.usedBlobGas, vm.Config{}, evm) if err != nil { return nil, fmt.Errorf("ReceiptGen.GetReceipt: bn=%d, txnIdx=%d, %w", blockNum, index, err) } @@ -304,6 +312,10 @@ func (g *Generator) GetReceipts(ctx context.Context, cfg *chain.Config, tx kv.Te //genEnv.ibs.SetTrace(true) blockNum := block.NumberU64() + vmCfg := vm.Config{ + JumpDestCache: vm.NewJumpDestCache(16), + } + evm := core.CreateEVM(cfg, core.GetHashFn(genEnv.header, genEnv.getHeader), g.engine, nil, genEnv.ibs, genEnv.header, vm.Config{}) ctx, cancel := context.WithTimeout(ctx, g.evmTimeout) defer cancel() @@ -319,11 +331,9 @@ func (g *Generator) GetReceipts(ctx context.Context, cfg *chain.Config, tx kv.Te default: } - evm2 := core.CreateEVM(cfg, core.GetHashFn(genEnv.header, genEnv.getHeader), g.engine, nil, genEnv.ibs, genEnv.header, vm.Config{}) - evm2.JumpDestCache = evm.JumpDestCache - evm = evm2 + evm = core.CreateEVM(cfg, core.GetHashFn(genEnv.header, genEnv.getHeader), g.engine, nil, genEnv.ibs, genEnv.header, vmCfg) genEnv.ibs.SetTxContext(blockNum, i) - receipt, _, err := core.ApplyTransactionWithEVM(cfg, g.engine, genEnv.gp, genEnv.ibs, genEnv.noopWriter, genEnv.header, txn, genEnv.usedGas, genEnv.usedBlobGas, vm.Config{}, evm) + receipt, _, err := core.ApplyTransactionWithEVM(cfg, g.engine, genEnv.gp, genEnv.ibs, genEnv.noopWriter, genEnv.header, txn, genEnv.gasUsed, genEnv.usedBlobGas, vmCfg, evm) if err != nil { return nil, fmt.Errorf("ReceiptGen.GetReceipts: bn=%d, txnIdx=%d, %w", block.NumberU64(), i, err) } From 70bc0d47ed82e8e215d036eb1e76e56450982cb6 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 15 Jul 2025 18:19:17 +0700 Subject: [PATCH 04/16] save --- rpc/jsonrpc/receipts/receipts_generator.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/rpc/jsonrpc/receipts/receipts_generator.go b/rpc/jsonrpc/receipts/receipts_generator.go index d734823ba28..3b506024fbd 100644 --- a/rpc/jsonrpc/receipts/receipts_generator.go +++ b/rpc/jsonrpc/receipts/receipts_generator.go @@ -219,18 +219,12 @@ func (g *Generator) GetReceipt(ctx context.Context, cfg *chain.Config, tx kv.Tem return nil, err } - //evm := core.CreateEVM(cfg, core.GetHashFn(genEnv.header, genEnv.getHeader), g.engine, nil, genEnv.ibs, genEnv.header, vm.Config{}) - //ctx, cancel := context.WithTimeout(ctx, g.evmTimeout) - //defer cancel() - //go func() { - // <-ctx.Done() - // evm.Cancel() - //}() - // - //receipt, _, err = core.ApplyTransactionWithEVM(cfg, g.engine, genEnv.gp, genEnv.ibs, genEnv.noopWriter, genEnv.header, txn, genEnv.usedGas, genEnv.usedBlobGas, vm.Config{}, evm) - //if err != nil { - // return nil, fmt.Errorf("ReceiptGen.GetReceipt: bn=%d, txnIdx=%d, %w", blockNum, index, err) - //} + ctx, cancel := context.WithTimeout(ctx, g.evmTimeout) + defer cancel() + go func() { + <-ctx.Done() + evm.Cancel() + }() status, gasUsed, err := aa.ExecuteAATransaction(aaTxn, paymasterContext, validationGasUsed, genEnv.gp, evm, header, genEnv.ibs) if err != nil { From 20b8e37fe3c637db68811365ecf30ef7bc166c69 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 15 Jul 2025 18:21:49 +0700 Subject: [PATCH 05/16] save --- core/vm/evm.go | 4 ++++ core/vm/interpreter.go | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index 40136e07eab..bbe5035f1c4 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -179,6 +179,10 @@ func (evm *EVM) Interpreter() Interpreter { } func (evm *EVM) call(typ OpCode, caller ContractRef, addr common.Address, input []byte, gas uint64, value *uint256.Int, bailout bool) (ret []byte, leftOverGas uint64, err error) { + if evm.abort.Load() { + return ret, leftOverGas, nil + } + depth := evm.interpreter.Depth() p, isPrecompile := evm.precompile(addr) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 6025136f912..7fa942585d1 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -240,10 +240,6 @@ func NewEVMInterpreter(evm *EVM, cfg Config) *EVMInterpreter { // considered a revert-and-consume-all-gas operation except for // ErrExecutionReverted which means revert-and-keep-gas-left. func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (ret []byte, err error) { - if in.evm.abort.Load() { - return ret, nil - } - // Don't bother with the execution if there's no code. if len(contract.Code) == 0 { return nil, nil From 6a4dc2077a315c9c9a2b3e643b170533e73b3646 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 15 Jul 2025 18:24:02 +0700 Subject: [PATCH 06/16] save --- eth/protocols/eth/handler_test.go | 147 ------------------------------ 1 file changed, 147 deletions(-) delete mode 100644 eth/protocols/eth/handler_test.go diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go deleted file mode 100644 index 7cf7bbbdfe1..00000000000 --- a/eth/protocols/eth/handler_test.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package eth_test - -import ( - "math/big" - "testing" - "time" - - "github.com/holiman/uint256" - "github.com/stretchr/testify/require" - - libcommon "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon-lib/crypto" - "github.com/erigontech/erigon-lib/direct" - sentry "github.com/erigontech/erigon-lib/gointerfaces/sentryproto" - "github.com/erigontech/erigon-lib/rlp" - "github.com/erigontech/erigon/core" - "github.com/erigontech/erigon/core/rawdb" - "github.com/erigontech/erigon/core/types" - "github.com/erigontech/erigon/eth/protocols/eth" - "github.com/erigontech/erigon/params" - "github.com/erigontech/erigon/turbo/jsonrpc/receipts" - "github.com/erigontech/erigon/turbo/stages/mock" -) - -var ( - // testKey is a private key to use for funding a tester account. - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - - // testAddr is the Ethereum address of the tester account. - testAddr = crypto.PubkeyToAddress(testKey.PublicKey) -) - -func TestGetBlockReceipts(t *testing.T) { - // Define three accounts to simulate transactions with - acc1Key, _ := crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") - acc2Key, _ := crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") - acc1Addr := crypto.PubkeyToAddress(acc1Key.PublicKey) - acc2Addr := crypto.PubkeyToAddress(acc2Key.PublicKey) - - signer := types.LatestSignerForChainID(nil) - // Create a chain generator with some simple transactions (blatantly stolen from @fjl/chain_markets_test) - generator := func(i int, block *core.BlockGen) { - switch i { - case 0: - // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), acc1Addr, uint256.NewInt(10000), params.TxGas, nil, nil), *signer, testKey) - block.AddTx(tx) - case 1: - // In block 2, the test bank sends some more ether to account #1. - // acc1Addr passes it on to account #2. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testAddr), acc1Addr, uint256.NewInt(1000), params.TxGas, nil, nil), *signer, testKey) - tx2, _ := types.SignTx(types.NewTransaction(block.TxNonce(acc1Addr), acc2Addr, uint256.NewInt(1000), params.TxGas, nil, nil), *signer, acc1Key) - block.AddTx(tx1) - block.AddTx(tx2) - case 2: - // Block 3 is empty but was mined by account #2. - block.SetCoinbase(acc2Addr) - block.SetExtra([]byte("yeehaw")) - case 3: - // Block 4 includes blocks 2 and 3 as uncle headers (with modified extra data). - b2 := block.PrevBlock(1).Header() - b2.Extra = []byte("foo") - block.AddUncle(b2) - b3 := block.PrevBlock(2).Header() - b3.Extra = []byte("foo") - block.AddUncle(b3) - } - } - // Assemble the test environment - m := mockWithGenerator(t, 4, generator) - receiptsGetter := receipts.NewGenerator(m.BlockReader, m.Engine, time.Minute) - // Collect the hashes to request, and the response to expect - var ( - hashes []libcommon.Hash - receipts []rlp.RawValue - ) - tx, err := m.DB.BeginTemporalRo(m.Ctx) - require.NoError(t, err) - defer tx.Rollback() - - for i := uint64(0); i <= rawdb.ReadCurrentHeader(tx).Number.Uint64(); i++ { - block, err := m.BlockReader.BlockByNumber(m.Ctx, tx, i) - require.NoError(t, err) - - hashes = append(hashes, block.Hash()) - // If known, encode and queue for response packet - - r, err := receiptsGetter.GetReceipts(m.Ctx, m.ChainConfig, tx, block) - require.NoError(t, err) - encoded, err := rlp.EncodeToBytes(r) - require.NoError(t, err) - receipts = append(receipts, encoded) - } - - require.NoError(t, err) - b, err := rlp.EncodeToBytes(eth.GetReceiptsPacket66{RequestId: 1, GetReceiptsPacket: hashes}) - require.NoError(t, err) - - m.StreamWg.Wait() - - m.ReceiveWg.Add(1) - // Send the hash request and verify the response - for _, err = range m.Send(&sentry.InboundMessage{Id: eth.ToProto[direct.ETH67][eth.GetReceiptsMsg], Data: b, PeerId: m.PeerId}) { - require.NoError(t, err) - } - - expect, err := rlp.EncodeToBytes(eth.ReceiptsRLPPacket66{RequestId: 1, ReceiptsRLPPacket: receipts}) - require.NoError(t, err) - m.ReceiveWg.Wait() - sent := m.SentMessage(0) - require.Equal(t, eth.ToProto[m.SentryClient.Protocol()][eth.ReceiptsMsg], sent.Id) - require.Equal(t, expect, sent.Data) -} - -// newTestBackend creates a chain with a number of explicitly defined blocks and -// wraps it into a mock backend. -func mockWithGenerator(t *testing.T, blocks int, generator func(int, *core.BlockGen)) *mock.MockSentry { - m := mock.MockWithGenesis(t, &types.Genesis{ - Config: params.TestChainConfig, - Alloc: types.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, - }, testKey, false) - if blocks > 0 { - chain, _ := core.GenerateChain(m.ChainConfig, m.Genesis, m.Engine, m.DB, blocks, generator) - err := m.InsertChain(chain) - require.NoError(t, err) - } - return m -} From 0f118edd7969ee5fe7cf4cbfb4b3856002e2248b Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 15 Jul 2025 18:24:42 +0700 Subject: [PATCH 07/16] save --- p2p/sentry/sentry_multi_client/sentry_multi_client.go | 2 +- rpc/jsonrpc/receipts/handler_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/p2p/sentry/sentry_multi_client/sentry_multi_client.go b/p2p/sentry/sentry_multi_client/sentry_multi_client.go index 6e937d0de06..12b4a7c4a36 100644 --- a/p2p/sentry/sentry_multi_client/sentry_multi_client.go +++ b/p2p/sentry/sentry_multi_client/sentry_multi_client.go @@ -224,7 +224,7 @@ func NewMultiClient( disableBlockDownload: disableBlockDownload, logger: logger, getReceiptsActiveGoroutineNumber: semaphore.NewWeighted(1), - ethApiWrapper: receipts.NewGenerator(blockReader, engine, 30*time.Second), + ethApiWrapper: receipts.NewGenerator(blockReader, engine, 5*time.Minute), } return cs, nil diff --git a/rpc/jsonrpc/receipts/handler_test.go b/rpc/jsonrpc/receipts/handler_test.go index 03799795e15..ded9875c76b 100644 --- a/rpc/jsonrpc/receipts/handler_test.go +++ b/rpc/jsonrpc/receipts/handler_test.go @@ -23,6 +23,7 @@ import ( "math" "math/big" "testing" + "time" "github.com/holiman/uint256" "github.com/stretchr/testify/require" @@ -297,7 +298,7 @@ func TestGetBlockReceipts(t *testing.T) { } // Assemble the test environment m := mockWithGenerator(t, 4, generator) - receiptsGetter := receipts.NewGenerator(m.BlockReader, m.Engine) + receiptsGetter := receipts.NewGenerator(m.BlockReader, m.Engine, time.Minute) // Collect the hashes to request, and the response to expect var ( hashes []common.Hash From 2659a9bf320251c87305946e7f4b68e3ccfbeb6a Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 21 Jul 2025 08:53:46 +0700 Subject: [PATCH 08/16] save --- erigon-lib/state/inverted_index_test.go | 129 ++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/erigon-lib/state/inverted_index_test.go b/erigon-lib/state/inverted_index_test.go index ef671c23bd8..706bc242576 100644 --- a/erigon-lib/state/inverted_index_test.go +++ b/erigon-lib/state/inverted_index_test.go @@ -27,6 +27,8 @@ import ( "testing" "time" + "github.com/c2h5oh/datasize" + "github.com/spaolacci/murmur3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -807,3 +809,130 @@ func TestInvIndex_OpenFolder(t *testing.T) { require.NoError(t, err) ii.Close() } + +func TestInvIndexPruningPerf(t *testing.T) { + //t.Skip("for manual benchmarks ") + testDbAndInvertedIndex2 := func(tb testing.TB, aggStep uint64, logger log.Logger) (kv.RwDB, *InvertedIndex) { + tb.Helper() + dirs := datadir.New("/Users/alex/data/remove_me_test") + keysTable := "Keys" + indexTable := "Index" + db := mdbx.New(kv.ChainDB, logger).Path(dirs.Chaindata).WriteMap(true).PageSize(4 * 1024).WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { + return kv.TableCfg{ + keysTable: kv.TableCfgItem{Flags: kv.DupSort}, + indexTable: kv.TableCfgItem{Flags: kv.DupSort}, + kv.TblPruningProgress: kv.TableCfgItem{}, + } + }).MustOpen() + tb.Cleanup(db.Close) + salt := uint32(1) + cfg := iiCfg{salt: new(atomic.Pointer[uint32]), dirs: dirs, filenameBase: "inv", keysTable: keysTable, valuesTable: indexTable, version: IIVersionTypes{DataEF: version.V1_0_standart, AccessorEFI: version.V1_0_standart}} + cfg.salt.Store(&salt) + cfg.Accessors = AccessorHashMap + ii, err := NewInvertedIndex(cfg, aggStep, logger) + require.NoError(tb, err) + ii.DisableFsync() + tb.Cleanup(ii.Close) + return db, ii + } + filledInvIndexOfSize2 := func(tb testing.TB, txs, module uint64, db kv.RwDB, ii *InvertedIndex) uint64 { + tb.Helper() + ctx, require := context.Background(), require.New(tb) + tb.Cleanup(db.Close) + + err := db.Update(ctx, func(tx kv.RwTx) error { + if cnt, _ := tx.Count(ii.keysTable); cnt > 0 { // if db is re-usable + return nil + } + ic := ii.BeginFilesRo() + defer ic.Close() + writer := ic.NewWriter() + defer writer.close() + + // keys are encodings of numbers 1..31 + // each key changes value on every txNum which is multiple of the key + var k [32]byte + for txNum := uint64(1); txNum <= txs; txNum++ { + for keyNum := uint64(1); keyNum <= module; keyNum++ { + if txNum%keyNum == 0 { + binary.BigEndian.PutUint64(k[:], keyNum) + binary.BigEndian.PutUint64(k[:], murmur3.Sum64(k[:])) + err := writer.Add(k[:], txNum) + require.NoError(err) + } + } + } + require.NoError(writer.Flush(ctx, tx)) + return nil + }) + require.NoError(err) + return txs + } + + txCnt := uint64(1_000) * 10_000 + mod := uint64(1) * 31 + db, ii := testDbAndInvertedIndex2(t, 16*1_000, log.New()) + _ = filledInvIndexOfSize2(t, txCnt, mod, db, ii) + defer ii.Close() + + require.NoError(t, db.View(context.Background(), func(tx kv.Tx) error { + collation, err := ii.collate(context.Background(), 0, tx) + require.NoError(t, err) + sf, _ := ii.buildFiles(context.Background(), 0, collation, background.NewProgressSet()) + txFrom, txTo := firstTxNumOfStep(0, ii.aggregationStep), firstTxNumOfStep(1, ii.aggregationStep) + ii.integrateDirtyFiles(sf, txFrom, txTo) + + // after reCalcVisibleFiles must be able to prune step 0. but not more + ii.reCalcVisibleFiles(ii.dirtyFilesEndTxNumMinimax()) + return err + })) + + logEvery := time.NewTicker(30 * time.Second) + defer logEvery.Stop() + + { //first rwtx - does some additional job and slow. do it and skip it + tx, err := db.BeginRw(context.Background()) + require.NoError(t, err) + ic := ii.BeginFilesRo() + ic.Prune(context.Background(), tx, 0, ic.aggStep, ic.aggStep, logEvery, true, nil) + tx.Rollback() + ic.Close() + } + + { + tx, err := db.BeginRw(context.Background()) + require.NoError(t, err) + ic := ii.BeginFilesRo() + start := time.Now() + ic.Prune(context.Background(), tx, 0, ic.aggStep, ic.aggStep, logEvery, true, nil) + a, _, _ := tx.(*mdbx.MdbxTx).SpaceDirty() + fmt.Printf("[dbg] 1 step: took=%s dirt=%s\n", time.Since(start), datasize.ByteSize(a).HR()) + tx.Rollback() + ic.Close() + } + + { + tx, err := db.BeginRw(context.Background()) + require.NoError(t, err) + ic := ii.BeginFilesRo() + start := time.Now() + pruneLimit := uint64(1_000) + ic.Prune(context.Background(), tx, 0, txCnt, pruneLimit, logEvery, true, nil) + a, _, _ := tx.(*mdbx.MdbxTx).SpaceDirty() + fmt.Printf("[dbg] 1K: took=%s dirt=%s\n", time.Since(start), datasize.ByteSize(a).HR()) + tx.Rollback() + ic.Close() + } + { + tx, err := db.BeginRw(context.Background()) + require.NoError(t, err) + ic := ii.BeginFilesRo() + start := time.Now() + pruneLimit := ic.aggStep * 30 + ic.Prune(context.Background(), tx, 0, txCnt, pruneLimit, logEvery, true, nil) + a, _, _ := tx.(*mdbx.MdbxTx).SpaceDirty() + fmt.Printf("[dbg] 30 steps: took=%s dirt=%s\n", time.Since(start), datasize.ByteSize(a).HR()) + tx.Rollback() + ic.Close() + } +} From abc58043cd016ff9e5dd09449dc5796868210db7 Mon Sep 17 00:00:00 2001 From: JkLondon Date: Mon, 21 Jul 2025 13:55:48 +0200 Subject: [PATCH 09/16] commit --- erigon-lib/state/inverted_index_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/inverted_index_test.go b/erigon-lib/state/inverted_index_test.go index 706bc242576..917446759aa 100644 --- a/erigon-lib/state/inverted_index_test.go +++ b/erigon-lib/state/inverted_index_test.go @@ -814,7 +814,7 @@ func TestInvIndexPruningPerf(t *testing.T) { //t.Skip("for manual benchmarks ") testDbAndInvertedIndex2 := func(tb testing.TB, aggStep uint64, logger log.Logger) (kv.RwDB, *InvertedIndex) { tb.Helper() - dirs := datadir.New("/Users/alex/data/remove_me_test") + dirs := datadir.New(t.TempDir()) keysTable := "Keys" indexTable := "Index" db := mdbx.New(kv.ChainDB, logger).Path(dirs.Chaindata).WriteMap(true).PageSize(4 * 1024).WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { From 8685eecb2135a0579985ca7bc9e20bdc5889bf1c Mon Sep 17 00:00:00 2001 From: JkLondon Date: Mon, 21 Jul 2025 14:14:30 +0200 Subject: [PATCH 10/16] commit --- erigon-lib/state/inverted_index_test.go | 34 ++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/erigon-lib/state/inverted_index_test.go b/erigon-lib/state/inverted_index_test.go index 917446759aa..b2012eca109 100644 --- a/erigon-lib/state/inverted_index_test.go +++ b/erigon-lib/state/inverted_index_test.go @@ -812,19 +812,21 @@ func TestInvIndex_OpenFolder(t *testing.T) { func TestInvIndexPruningPerf(t *testing.T) { //t.Skip("for manual benchmarks ") - testDbAndInvertedIndex2 := func(tb testing.TB, aggStep uint64, logger log.Logger) (kv.RwDB, *InvertedIndex) { + dirs := datadir.New(t.TempDir()) + keysTable := "Keys" + indexTable := "Index" + logger := log.New() + db := mdbx.New(kv.ChainDB, logger).Path(dirs.Chaindata).WriteMap(true).PageSize(4 * 1024).WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { + return kv.TableCfg{ + keysTable: kv.TableCfgItem{Flags: kv.DupSort}, + indexTable: kv.TableCfgItem{Flags: kv.DupSort}, + kv.TblPruningProgress: kv.TableCfgItem{}, + } + }).MustOpen() + t.Cleanup(db.Close) + testDbAndInvertedIndex2 := func(tb testing.TB, aggStep uint64, logger log.Logger) *InvertedIndex { tb.Helper() - dirs := datadir.New(t.TempDir()) - keysTable := "Keys" - indexTable := "Index" - db := mdbx.New(kv.ChainDB, logger).Path(dirs.Chaindata).WriteMap(true).PageSize(4 * 1024).WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { - return kv.TableCfg{ - keysTable: kv.TableCfgItem{Flags: kv.DupSort}, - indexTable: kv.TableCfgItem{Flags: kv.DupSort}, - kv.TblPruningProgress: kv.TableCfgItem{}, - } - }).MustOpen() - tb.Cleanup(db.Close) + salt := uint32(1) cfg := iiCfg{salt: new(atomic.Pointer[uint32]), dirs: dirs, filenameBase: "inv", keysTable: keysTable, valuesTable: indexTable, version: IIVersionTypes{DataEF: version.V1_0_standart, AccessorEFI: version.V1_0_standart}} cfg.salt.Store(&salt) @@ -832,13 +834,11 @@ func TestInvIndexPruningPerf(t *testing.T) { ii, err := NewInvertedIndex(cfg, aggStep, logger) require.NoError(tb, err) ii.DisableFsync() - tb.Cleanup(ii.Close) - return db, ii + return ii } filledInvIndexOfSize2 := func(tb testing.TB, txs, module uint64, db kv.RwDB, ii *InvertedIndex) uint64 { tb.Helper() ctx, require := context.Background(), require.New(tb) - tb.Cleanup(db.Close) err := db.Update(ctx, func(tx kv.RwTx) error { if cnt, _ := tx.Count(ii.keysTable); cnt > 0 { // if db is re-usable @@ -871,9 +871,9 @@ func TestInvIndexPruningPerf(t *testing.T) { txCnt := uint64(1_000) * 10_000 mod := uint64(1) * 31 - db, ii := testDbAndInvertedIndex2(t, 16*1_000, log.New()) + ii := testDbAndInvertedIndex2(t, 16*1_000, logger) + t.Cleanup(ii.Close) _ = filledInvIndexOfSize2(t, txCnt, mod, db, ii) - defer ii.Close() require.NoError(t, db.View(context.Background(), func(tx kv.Tx) error { collation, err := ii.collate(context.Background(), 0, tx) From 78125417764d37b031c66749227e16fe016d6d65 Mon Sep 17 00:00:00 2001 From: JkLondon Date: Mon, 21 Jul 2025 14:29:01 +0200 Subject: [PATCH 11/16] commit --- erigon-lib/state/inverted_index_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/inverted_index_test.go b/erigon-lib/state/inverted_index_test.go index b2012eca109..870f795157d 100644 --- a/erigon-lib/state/inverted_index_test.go +++ b/erigon-lib/state/inverted_index_test.go @@ -816,7 +816,7 @@ func TestInvIndexPruningPerf(t *testing.T) { keysTable := "Keys" indexTable := "Index" logger := log.New() - db := mdbx.New(kv.ChainDB, logger).Path(dirs.Chaindata).WriteMap(true).PageSize(4 * 1024).WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { + db := mdbx.New(kv.ChainDB, logger).Path(dirs.Chaindata).WriteMap(false).PageSize(4 * 1024).WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { return kv.TableCfg{ keysTable: kv.TableCfgItem{Flags: kv.DupSort}, indexTable: kv.TableCfgItem{Flags: kv.DupSort}, From 4b9ed6d79d2be07c0d9760856c3510b9ace512a6 Mon Sep 17 00:00:00 2001 From: JkLondon Date: Mon, 21 Jul 2025 14:34:20 +0200 Subject: [PATCH 12/16] commit --- erigon-lib/state/inverted_index_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/inverted_index_test.go b/erigon-lib/state/inverted_index_test.go index 870f795157d..b2012eca109 100644 --- a/erigon-lib/state/inverted_index_test.go +++ b/erigon-lib/state/inverted_index_test.go @@ -816,7 +816,7 @@ func TestInvIndexPruningPerf(t *testing.T) { keysTable := "Keys" indexTable := "Index" logger := log.New() - db := mdbx.New(kv.ChainDB, logger).Path(dirs.Chaindata).WriteMap(false).PageSize(4 * 1024).WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { + db := mdbx.New(kv.ChainDB, logger).Path(dirs.Chaindata).WriteMap(true).PageSize(4 * 1024).WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { return kv.TableCfg{ keysTable: kv.TableCfgItem{Flags: kv.DupSort}, indexTable: kv.TableCfgItem{Flags: kv.DupSort}, From bfb4b7c146f4799c087a1b0098f24ecc358443eb Mon Sep 17 00:00:00 2001 From: JkLondon Date: Mon, 21 Jul 2025 15:58:46 +0200 Subject: [PATCH 13/16] commit --- erigon-lib/state/inverted_index_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/erigon-lib/state/inverted_index_test.go b/erigon-lib/state/inverted_index_test.go index b2012eca109..f09f866e7bb 100644 --- a/erigon-lib/state/inverted_index_test.go +++ b/erigon-lib/state/inverted_index_test.go @@ -21,6 +21,7 @@ import ( "context" "encoding/binary" "fmt" + "github.com/erigontech/erigon-lib/common/race" "math" "os" "sync/atomic" @@ -871,6 +872,10 @@ func TestInvIndexPruningPerf(t *testing.T) { txCnt := uint64(1_000) * 10_000 mod := uint64(1) * 31 + if race.Enabled { + txCnt = uint64(10_000) // lowering PERFOMANCE test params if RACE enabled + mod = uint64(1) * 10 + } ii := testDbAndInvertedIndex2(t, 16*1_000, logger) t.Cleanup(ii.Close) _ = filledInvIndexOfSize2(t, txCnt, mod, db, ii) From d978991169edec985f1c6de3a325d3af2c0c2b9c Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 23 Jul 2025 10:22:05 +0700 Subject: [PATCH 14/16] save --- erigon-lib/state/inverted_index_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/erigon-lib/state/inverted_index_test.go b/erigon-lib/state/inverted_index_test.go index f09f866e7bb..f68753993dc 100644 --- a/erigon-lib/state/inverted_index_test.go +++ b/erigon-lib/state/inverted_index_test.go @@ -21,13 +21,14 @@ import ( "context" "encoding/binary" "fmt" - "github.com/erigontech/erigon-lib/common/race" "math" "os" "sync/atomic" "testing" "time" + "github.com/erigontech/erigon-lib/common/race" + "github.com/c2h5oh/datasize" "github.com/spaolacci/murmur3" "github.com/stretchr/testify/assert" @@ -812,7 +813,7 @@ func TestInvIndex_OpenFolder(t *testing.T) { } func TestInvIndexPruningPerf(t *testing.T) { - //t.Skip("for manual benchmarks ") + t.Skip("for manual benchmarks ") dirs := datadir.New(t.TempDir()) keysTable := "Keys" indexTable := "Index" From 4d2ac4ea2f8bc89dda2a6af37630f1ef4e79077f Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 23 Jul 2025 10:24:19 +0700 Subject: [PATCH 15/16] save --- erigon-lib/state/inverted_index_test.go | 135 ------------------------ 1 file changed, 135 deletions(-) diff --git a/erigon-lib/state/inverted_index_test.go b/erigon-lib/state/inverted_index_test.go index f68753993dc..ef671c23bd8 100644 --- a/erigon-lib/state/inverted_index_test.go +++ b/erigon-lib/state/inverted_index_test.go @@ -27,10 +27,6 @@ import ( "testing" "time" - "github.com/erigontech/erigon-lib/common/race" - - "github.com/c2h5oh/datasize" - "github.com/spaolacci/murmur3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -811,134 +807,3 @@ func TestInvIndex_OpenFolder(t *testing.T) { require.NoError(t, err) ii.Close() } - -func TestInvIndexPruningPerf(t *testing.T) { - t.Skip("for manual benchmarks ") - dirs := datadir.New(t.TempDir()) - keysTable := "Keys" - indexTable := "Index" - logger := log.New() - db := mdbx.New(kv.ChainDB, logger).Path(dirs.Chaindata).WriteMap(true).PageSize(4 * 1024).WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { - return kv.TableCfg{ - keysTable: kv.TableCfgItem{Flags: kv.DupSort}, - indexTable: kv.TableCfgItem{Flags: kv.DupSort}, - kv.TblPruningProgress: kv.TableCfgItem{}, - } - }).MustOpen() - t.Cleanup(db.Close) - testDbAndInvertedIndex2 := func(tb testing.TB, aggStep uint64, logger log.Logger) *InvertedIndex { - tb.Helper() - - salt := uint32(1) - cfg := iiCfg{salt: new(atomic.Pointer[uint32]), dirs: dirs, filenameBase: "inv", keysTable: keysTable, valuesTable: indexTable, version: IIVersionTypes{DataEF: version.V1_0_standart, AccessorEFI: version.V1_0_standart}} - cfg.salt.Store(&salt) - cfg.Accessors = AccessorHashMap - ii, err := NewInvertedIndex(cfg, aggStep, logger) - require.NoError(tb, err) - ii.DisableFsync() - return ii - } - filledInvIndexOfSize2 := func(tb testing.TB, txs, module uint64, db kv.RwDB, ii *InvertedIndex) uint64 { - tb.Helper() - ctx, require := context.Background(), require.New(tb) - - err := db.Update(ctx, func(tx kv.RwTx) error { - if cnt, _ := tx.Count(ii.keysTable); cnt > 0 { // if db is re-usable - return nil - } - ic := ii.BeginFilesRo() - defer ic.Close() - writer := ic.NewWriter() - defer writer.close() - - // keys are encodings of numbers 1..31 - // each key changes value on every txNum which is multiple of the key - var k [32]byte - for txNum := uint64(1); txNum <= txs; txNum++ { - for keyNum := uint64(1); keyNum <= module; keyNum++ { - if txNum%keyNum == 0 { - binary.BigEndian.PutUint64(k[:], keyNum) - binary.BigEndian.PutUint64(k[:], murmur3.Sum64(k[:])) - err := writer.Add(k[:], txNum) - require.NoError(err) - } - } - } - require.NoError(writer.Flush(ctx, tx)) - return nil - }) - require.NoError(err) - return txs - } - - txCnt := uint64(1_000) * 10_000 - mod := uint64(1) * 31 - if race.Enabled { - txCnt = uint64(10_000) // lowering PERFOMANCE test params if RACE enabled - mod = uint64(1) * 10 - } - ii := testDbAndInvertedIndex2(t, 16*1_000, logger) - t.Cleanup(ii.Close) - _ = filledInvIndexOfSize2(t, txCnt, mod, db, ii) - - require.NoError(t, db.View(context.Background(), func(tx kv.Tx) error { - collation, err := ii.collate(context.Background(), 0, tx) - require.NoError(t, err) - sf, _ := ii.buildFiles(context.Background(), 0, collation, background.NewProgressSet()) - txFrom, txTo := firstTxNumOfStep(0, ii.aggregationStep), firstTxNumOfStep(1, ii.aggregationStep) - ii.integrateDirtyFiles(sf, txFrom, txTo) - - // after reCalcVisibleFiles must be able to prune step 0. but not more - ii.reCalcVisibleFiles(ii.dirtyFilesEndTxNumMinimax()) - return err - })) - - logEvery := time.NewTicker(30 * time.Second) - defer logEvery.Stop() - - { //first rwtx - does some additional job and slow. do it and skip it - tx, err := db.BeginRw(context.Background()) - require.NoError(t, err) - ic := ii.BeginFilesRo() - ic.Prune(context.Background(), tx, 0, ic.aggStep, ic.aggStep, logEvery, true, nil) - tx.Rollback() - ic.Close() - } - - { - tx, err := db.BeginRw(context.Background()) - require.NoError(t, err) - ic := ii.BeginFilesRo() - start := time.Now() - ic.Prune(context.Background(), tx, 0, ic.aggStep, ic.aggStep, logEvery, true, nil) - a, _, _ := tx.(*mdbx.MdbxTx).SpaceDirty() - fmt.Printf("[dbg] 1 step: took=%s dirt=%s\n", time.Since(start), datasize.ByteSize(a).HR()) - tx.Rollback() - ic.Close() - } - - { - tx, err := db.BeginRw(context.Background()) - require.NoError(t, err) - ic := ii.BeginFilesRo() - start := time.Now() - pruneLimit := uint64(1_000) - ic.Prune(context.Background(), tx, 0, txCnt, pruneLimit, logEvery, true, nil) - a, _, _ := tx.(*mdbx.MdbxTx).SpaceDirty() - fmt.Printf("[dbg] 1K: took=%s dirt=%s\n", time.Since(start), datasize.ByteSize(a).HR()) - tx.Rollback() - ic.Close() - } - { - tx, err := db.BeginRw(context.Background()) - require.NoError(t, err) - ic := ii.BeginFilesRo() - start := time.Now() - pruneLimit := ic.aggStep * 30 - ic.Prune(context.Background(), tx, 0, txCnt, pruneLimit, logEvery, true, nil) - a, _, _ := tx.(*mdbx.MdbxTx).SpaceDirty() - fmt.Printf("[dbg] 30 steps: took=%s dirt=%s\n", time.Since(start), datasize.ByteSize(a).HR()) - tx.Rollback() - ic.Close() - } -} From ef9ee788c2febf068f4b625b78e9045e0ab495cc Mon Sep 17 00:00:00 2001 From: JkLondon Date: Wed, 23 Jul 2025 09:54:51 +0200 Subject: [PATCH 16/16] commit --- txnprovider/shutter/block_building_integration_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/txnprovider/shutter/block_building_integration_test.go b/txnprovider/shutter/block_building_integration_test.go index 871b24635d4..40529d56ab3 100644 --- a/txnprovider/shutter/block_building_integration_test.go +++ b/txnprovider/shutter/block_building_integration_test.go @@ -21,6 +21,7 @@ import ( "crypto/ecdsa" "errors" "fmt" + "github.com/erigontech/erigon/rpc/rpccfg" "math/big" "path" "runtime" @@ -261,6 +262,7 @@ func initBlockBuildingUniverse(ctx context.Context, t *testing.T) blockBuildingU AuthRpcPort: engineApiPort, JWTSecretPath: path.Join(dataDir, "jwt.hex"), ReturnDataLimit: 100_000, + EvmCallTimeout: rpccfg.DefaultEvmCallTimeout, } nodeKeyConfig := p2p.NodeKeyConfig{}