8337 modify chain data pruner

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -2699,7 +2699,8 @@ private void setMergeConfigOptions() {
 
   /** Set ignorable segments in RocksDB Storage Provider plugin. */
   public void setIgnorableStorageSegments() {
-    if (!unstableChainPruningOptions.getChainDataPruningEnabled()) {
+    if (!unstableChainPruningOptions.getChainDataPruningEnabled()
+        && !unstableChainPruningOptions.getPreMergePruningEnabled()) {
       rocksDBPlugin.addIgnorableSegmentIdentifier(KeyValueSegmentIdentifier.CHAIN_PRUNER_STATE);
     }
   }

besu/src/main/java/org/hyperledger/besu/cli/config/NetworkName.java

@@ -21,59 +21,93 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
+import java.util.OptionalInt;
 import java.util.function.Supplier;
 
 import org.apache.commons.lang3.StringUtils;
 
 /** The enum Network name. */
 public enum NetworkName {
   /** Mainnet network name. */
-  MAINNET("/mainnet.json", BigInteger.valueOf(1), true, NativeRequirement.MAINNET),
+  MAINNET(
+      "/mainnet.json",
+      BigInteger.valueOf(1),
+      OptionalInt.of(15_537_394),
+      true,
+      NativeRequirement.MAINNET),
   /** Sepolia network name. */
-  SEPOLIA("/sepolia.json", BigInteger.valueOf(11155111), true, NativeRequirement.MAINNET),
+  SEPOLIA(
+      "/sepolia.json",
+      BigInteger.valueOf(11155111),
+      OptionalInt.of(1_450_409),
+      true,
+      NativeRequirement.MAINNET),
   /** Holešky network name. */
-  HOLESKY("/holesky.json", BigInteger.valueOf(17000), true, NativeRequirement.MAINNET),
+  HOLESKY(
+      "/holesky.json",
+      BigInteger.valueOf(17000),
+      OptionalInt.empty(),
+      true,
+      NativeRequirement.MAINNET),
   /** Hoodi network name. */
-  HOODI("/hoodi.json", BigInteger.valueOf(560048), true, NativeRequirement.MAINNET),
+  HOODI(
+      "/hoodi.json",
+      BigInteger.valueOf(560048),
+      OptionalInt.empty(),
+      true,
+      NativeRequirement.MAINNET),
   /**
    * EPHEMERY network name. The actual networkId used is calculated based on this default value and
    * the current time. https://ephemery.dev/
    */
-  EPHEMERY("/ephemery.json", BigInteger.valueOf(39438135), true, NativeRequirement.MAINNET),
+  EPHEMERY(
+      "/ephemery.json",
+      BigInteger.valueOf(39438135),
+      OptionalInt.empty(),
+      true,
+      NativeRequirement.MAINNET),
   /** LUKSO mainnet network name. */
-  LUKSO("/lukso.json", BigInteger.valueOf(42)),
+  LUKSO("/lukso.json", BigInteger.valueOf(42), OptionalInt.empty()),
   /** Dev network name. */
-  DEV("/dev.json", BigInteger.valueOf(2018), false),
+  DEV("/dev.json", BigInteger.valueOf(2018), OptionalInt.empty(), false),
   /** Future EIPs network name. */
-  FUTURE_EIPS("/future.json", BigInteger.valueOf(2022), false),
+  FUTURE_EIPS("/future.json", BigInteger.valueOf(2022), OptionalInt.empty(), false),
   /** Experimental EIPs network name. */
-  EXPERIMENTAL_EIPS("/experimental.json", BigInteger.valueOf(2023), false),
+  EXPERIMENTAL_EIPS("/experimental.json", BigInteger.valueOf(2023), OptionalInt.empty(), false),
   /** Classic network name. */
-  CLASSIC("/classic.json", BigInteger.valueOf(1)),
+  CLASSIC("/classic.json", BigInteger.valueOf(1), OptionalInt.empty()),
   /** Mordor network name. */
-  MORDOR("/mordor.json", BigInteger.valueOf(7));
+  MORDOR("/mordor.json", BigInteger.valueOf(7), OptionalInt.empty());
 
   private final String genesisFile;
   private final BigInteger networkId;
+  private final OptionalInt firstPosBlockNumber;
   private final boolean canSnapSync;
   private final String deprecationDate;
   private final Supplier<List<NativeRequirementResult>> nativeRequirements;
 
-  NetworkName(final String genesisFile, final BigInteger networkId) {
-    this(genesisFile, networkId, true);
+  NetworkName(
+      final String genesisFile, final BigInteger networkId, final OptionalInt firstPosBlockNumber) {
+    this(genesisFile, networkId, firstPosBlockNumber, true);
   }
 
-  NetworkName(final String genesisFile, final BigInteger networkId, final boolean canSnapSync) {
-    this(genesisFile, networkId, canSnapSync, Collections::emptyList);
+  NetworkName(
+      final String genesisFile,
+      final BigInteger networkId,
+      final OptionalInt firstPosBlockNumber,
+      final boolean canSnapSync) {
+    this(genesisFile, networkId, firstPosBlockNumber, canSnapSync, Collections::emptyList);
   }
 
   NetworkName(
       final String genesisFile,
       final BigInteger networkId,
+      final OptionalInt firstPosBlockNumber,
       final boolean canSnapSync,
       final Supplier<List<NativeRequirementResult>> nativeRequirements) {
     this.genesisFile = genesisFile;
     this.networkId = networkId;
+    this.firstPosBlockNumber = firstPosBlockNumber;
     this.canSnapSync = canSnapSync;
     // no deprecations planned
     this.deprecationDate = null;
@@ -98,6 +132,15 @@ public BigInteger getNetworkId() {
     return networkId;
   }
 
+  /**
+   * Gets the optional merge block number
+   *
+   * @return the optional merge block number
+   */
+  public OptionalInt getFirstPosBlockNumber() {
+    return firstPosBlockNumber;
+  }
+
   /**
    * Can SNAP sync boolean.
    *

besu/src/main/java/org/hyperledger/besu/cli/options/ChainPruningOptions.java

@@ -25,11 +25,13 @@
 /** The Chain pruning CLI options. */
 public class ChainPruningOptions implements CLIOptions<ChainPrunerConfiguration> {
   private static final String CHAIN_PRUNING_ENABLED_FLAG = "--Xchain-pruning-enabled";
+  private static final String PRE_MERGE_PRUNING_ENABLED_FLAG = "--Xpre-merge-pruning-enabled";
   private static final String CHAIN_PRUNING_BLOCKS_RETAINED_FLAG =
       "--Xchain-pruning-blocks-retained";
   private static final String CHAIN_PRUNING_BLOCKS_RETAINED_LIMIT_FLAG =
       "--Xchain-pruning-blocks-retained-limit";
   private static final String CHAIN_PRUNING_FREQUENCY_FLAG = "--Xchain-pruning-frequency";
+  private static final String PRE_MERGE_PRUNING_QUANTITY_FLAG = "--Xpre-merge-pruning-quantity";
 
   /**
    * The "CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED_LIMIT" field sets the minimum limit for the
@@ -42,20 +44,32 @@ public class ChainPruningOptions implements CLIOptions<ChainPrunerConfiguration>
   /** The constant DEFAULT_CHAIN_DATA_PRUNING_FREQUENCY. */
   public static final int DEFAULT_CHAIN_DATA_PRUNING_FREQUENCY = 256;
 
+  /** The constant DEFAULT_PRE_MERGE_PRUNING_QUANTITY. */
+  public static final int DEFAULT_PRE_MERGE_PRUNING_QUANTITY = 256;
+
   @CommandLine.Option(
       hidden = true,
       names = {CHAIN_PRUNING_ENABLED_FLAG},
       description =
           "Enable the chain pruner to actively prune old chain data (default: ${DEFAULT-VALUE})")
   private final Boolean chainDataPruningEnabled = Boolean.FALSE;
 
+  @CommandLine.Option(
+      hidden = true,
+      names = {PRE_MERGE_PRUNING_ENABLED_FLAG},
+      description =
+          "Enable the chain pruner to actively prune pre-merge blocks, but not headers (default: ${DEFAULT-VALUE})")
+  private final Boolean preMergePruningEnabled = Boolean.FALSE;
+
   @CommandLine.Option(
       hidden = true,
       names = {CHAIN_PRUNING_BLOCKS_RETAINED_FLAG},
       description =
           "The number of recent blocks for which to keep the chain data. Should be >= "
               + CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED_LIMIT
-              + " (default: ${DEFAULT-VALUE})")
+              + " (default: ${DEFAULT-VALUE}). Unused if "
+              + PRE_MERGE_PRUNING_ENABLED_FLAG
+              + " is enabled")
   private final Long chainDataPruningBlocksRetained = CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED_LIMIT;
 
   @CommandLine.Option(
@@ -65,7 +79,9 @@ public class ChainPruningOptions implements CLIOptions<ChainPrunerConfiguration>
           "Allows setting the limit below which no more blocks can be pruned. This prevents setting a value lower than this for "
               + CHAIN_PRUNING_BLOCKS_RETAINED_FLAG
               + ". This flag should be used with caution as reducing the limit may have unintended side effects."
-              + " (default: ${DEFAULT-VALUE})")
+              + " (default: ${DEFAULT-VALUE}). Unused if "
+              + PRE_MERGE_PRUNING_ENABLED_FLAG
+              + " is enabled")
   private final Long chainDataPruningBlocksRetainedLimit =
       CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED_LIMIT;
 
@@ -77,6 +93,14 @@ public class ChainPruningOptions implements CLIOptions<ChainPrunerConfiguration>
   private final PositiveNumber chainDataPruningBlocksFrequency =
       PositiveNumber.fromInt(DEFAULT_CHAIN_DATA_PRUNING_FREQUENCY);
 
+  @CommandLine.Option(
+      hidden = true,
+      names = {PRE_MERGE_PRUNING_QUANTITY_FLAG},
+      description =
+          "The number of pre-merge blocks to prune per pruning operation. Must be non-negative (default: ${DEFAULT-VALUE})")
+  private final PositiveNumber preMergePruningBlocksQuantity =
+      PositiveNumber.fromInt(DEFAULT_PRE_MERGE_PRUNING_QUANTITY);
+
   /** Default Constructor. */
   ChainPruningOptions() {}
 
@@ -98,6 +122,15 @@ public Boolean getChainDataPruningEnabled() {
     return chainDataPruningEnabled;
   }
 
+  /**
+   * Gets pre-merge pruning enabled
+   *
+   * @return the pre-merge pruning enabled
+   */
+  public Boolean getPreMergePruningEnabled() {
+    return preMergePruningEnabled;
+  }
+
   /**
    * Gets chain data pruning blocks retained.
    *
@@ -120,21 +153,27 @@ public Long getChainDataPruningBlocksRetainedLimit() {
   public ChainPrunerConfiguration toDomainObject() {
     return new ChainPrunerConfiguration(
         chainDataPruningEnabled,
+        preMergePruningEnabled,
         chainDataPruningBlocksRetained,
         chainDataPruningBlocksRetainedLimit,
-        chainDataPruningBlocksFrequency.getValue());
+        chainDataPruningBlocksFrequency.getValue(),
+        preMergePruningBlocksQuantity.getValue());
   }
 
   @Override
   public List<String> getCLIOptions() {
     return Arrays.asList(
         CHAIN_PRUNING_ENABLED_FLAG,
         chainDataPruningEnabled.toString(),
+        PRE_MERGE_PRUNING_ENABLED_FLAG,
+        preMergePruningEnabled.toString(),
         CHAIN_PRUNING_BLOCKS_RETAINED_FLAG,
         chainDataPruningBlocksRetained.toString(),
         CHAIN_PRUNING_BLOCKS_RETAINED_LIMIT_FLAG,
         chainDataPruningBlocksRetainedLimit.toString(),
         CHAIN_PRUNING_FREQUENCY_FLAG,
-        chainDataPruningBlocksFrequency.toString());
+        chainDataPruningBlocksFrequency.toString(),
+        PRE_MERGE_PRUNING_QUANTITY_FLAG,
+        preMergePruningBlocksQuantity.toString());
   }
 }

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -16,6 +16,7 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import org.hyperledger.besu.cli.config.NetworkName;
 import org.hyperledger.besu.components.BesuComponent;
 import org.hyperledger.besu.config.CheckpointConfigOptions;
 import org.hyperledger.besu.config.GenesisConfig;
@@ -117,8 +118,10 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.OptionalLong;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
+import java.util.stream.Stream;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -702,14 +705,27 @@ public BesuController build() {
     final boolean fullSyncDisabled = !SyncMode.isFullSync(syncConfig.getSyncMode());
     final SyncState syncState = new SyncState(blockchain, ethPeers, fullSyncDisabled, checkpoint);
 
-    if (chainPrunerConfiguration.getChainPruningEnabled()) {
-      final ChainDataPruner chainDataPruner = createChainPruner(blockchainStorage);
-      blockchain.observeBlockAdded(chainDataPruner);
-      LOG.info(
-          "Chain data pruning enabled with recent blocks retained to be: "
-              + chainPrunerConfiguration.getChainPruningBlocksRetained()
-              + " and frequency to be: "
-              + chainPrunerConfiguration.getChainPruningBlocksFrequency());
+    if (chainPrunerConfiguration.chainPruningEnabled()
+        || chainPrunerConfiguration.preMergePruningEnabled()) {
+      LOG.info("Adding ChainDataPruner to observe block added events");
+      final AtomicLong chainDataPrunerObserverId = new AtomicLong();
+      final ChainDataPruner chainDataPruner =
+          createChainPruner(
+              blockchainStorage, () -> blockchain.removeObserver(chainDataPrunerObserverId.get()));
+      chainDataPrunerObserverId.set(blockchain.observeBlockAdded(chainDataPruner));
+      if (chainPrunerConfiguration.chainPruningEnabled()) {
+        LOG.info(
+            "Chain data pruning enabled with recent blocks retained to be: "
+                + chainPrunerConfiguration.chainPruningBlocksRetained()
+                + " and frequency to be: "
+                + chainPrunerConfiguration.blocksFrequency());
+      } else if (chainPrunerConfiguration.preMergePruningEnabled()) {
+        LOG.info(
+            "Pre-merge block pruning enabled with frequency: "
+                + chainPrunerConfiguration.blocksFrequency()
+                + " and quantity: "
+                + chainPrunerConfiguration.preMergePruningBlocksQuantity());
+      }
     }
 
     final TransactionPool transactionPool =
@@ -1174,16 +1190,30 @@ yield new ForestWorldStateArchive(
     };
   }
 
-  private ChainDataPruner createChainPruner(final BlockchainStorage blockchainStorage) {
+  private ChainDataPruner createChainPruner(
+      final BlockchainStorage blockchainStorage, final Runnable unsubscribeRunnable) {
+    NetworkName network =
+        Stream.of(NetworkName.values())
+            .filter((n) -> n.getNetworkId().equals(networkId))
+            .findAny()
+            .orElseThrow(() -> new RuntimeException("Unrecognised network"));
     return new ChainDataPruner(
         blockchainStorage,
+        unsubscribeRunnable,
         new ChainDataPrunerStorage(
             storageProvider.getStorageBySegmentIdentifier(
                 KeyValueSegmentIdentifier.CHAIN_PRUNER_STATE)),
-        chainPrunerConfiguration.getChainPruningBlocksRetained(),
-        chainPrunerConfiguration.getChainPruningBlocksFrequency(),
+        network.getFirstPosBlockNumber().orElse(0),
+        chainPrunerConfiguration.chainPruningEnabled()
+            ? ChainDataPruner.Mode.CHAIN_PRUNING
+            : (chainPrunerConfiguration.preMergePruningEnabled()
+                ? ChainDataPruner.Mode.PRE_MERGE_PRUNING
+                : null),
+        chainPrunerConfiguration.chainPruningBlocksRetained(),
+        chainPrunerConfiguration.blocksFrequency(),
+        chainPrunerConfiguration.preMergePruningBlocksQuantity(),
         MonitoredExecutors.newBoundedThreadPool(
-            ChainDataPruner.class.getSimpleName(),
+            EthScheduler.class.getSimpleName() + "-ChainDataPruner",
             1,
             1,
             ChainDataPruner.MAX_PRUNING_THREAD_QUEUE_SIZE,