feat(*): optimize old rewards withdrawal

actuator/src/main/java/org/tron/core/utils/ProposalUtil.java

@@ -743,13 +743,9 @@ public static void validator(DynamicPropertiesStore dynamicPropertiesStore,
         }
         if (!dynamicPropertiesStore.useNewRewardAlgorithm()) {
           throw new ContractValidateException(
-              "[ALLOW_NEW_REWARD] proposal must be approved "
+              "[ALLOW_NEW_REWARD] or [ALLOW_TVM_VOTE] proposal must be approved "
                   + "before [ALLOW_OLD_REWARD_OPT] can be proposed");
         }
-        if (dynamicPropertiesStore.useNewRewardAlgorithmFromStart()) {
-          throw new ContractValidateException(
-              "no need old reward opt, ALLOW_NEW_REWARD from start cycle 1");
-        }
         break;
       }
       default:

chainbase/src/main/java/org/tron/core/capsule/utils/MerkleTree.java

@@ -5,10 +5,12 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import lombok.Getter;
+import net.jcip.annotations.NotThreadSafe;
 import org.tron.common.parameter.CommonParameter;
 import org.tron.common.utils.Sha256Hash;
 
 @Getter
+@NotThreadSafe
 public class MerkleTree {
 
   private static volatile MerkleTree instance;

chainbase/src/main/java/org/tron/core/db/common/iterator/DBIterator.java

@@ -16,12 +16,6 @@ public interface DBIterator extends Iterator<Entry<byte[], byte[]>>, AutoCloseab
 
   void seekToLast();
 
-  default UnmodifiableIterator<Entry<byte[], byte[]>> prefixQueryAfterThat
-      (byte[] key, byte[] afterThat) {
-    this.seek(afterThat == null ? key : afterThat);
-    return Iterators.filter(this, entry -> Bytes.indexOf(entry.getKey(), key) == 0);
-  }
-
   /**
    * An iterator is either positioned at a key/value pair, or
    * not valid.  This method returns true iff the iterator is valid.

chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java

@@ -108,14 +108,12 @@ public boolean valid() {
 
   @Override
   public byte[] getKey() {
-    checkState();
     checkValid();
     return dbIterator.key();
   }
 
   @Override
   public byte[] getValue() {
-    checkState();
     checkValid();
     return dbIterator.value();
   }

chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java

@@ -93,14 +93,12 @@ public boolean valid() {
 
   @Override
   public byte[] getKey() {
-    checkState();
     checkValid();
     return dbIterator.peekNext().getKey();
   }
 
   @Override
   public byte[] getValue() {
-    checkState();
     checkValid();
     return dbIterator.peekNext().getValue();
   }

chainbase/src/main/java/org/tron/core/service/RewardViCalService.java

@@ -3,7 +3,6 @@
 import static org.tron.core.store.DelegationStore.DECIMAL_OF_VI_REWARD;
 import static org.tron.core.store.DelegationStore.REMARK;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Streams;
 import com.google.common.primitives.Bytes;
 import com.google.protobuf.ByteString;
@@ -18,7 +17,6 @@
 import java.util.stream.Collectors;
 import java.util.stream.LongStream;
 import javax.annotation.PreDestroy;
-import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.bouncycastle.util.encoders.Hex;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -27,9 +25,9 @@
 import org.tron.common.es.ExecutorServiceManager;
 import org.tron.common.parameter.CommonParameter;
 import org.tron.common.utils.ByteArray;
+import org.tron.common.utils.MerkleRoot;
 import org.tron.common.utils.Pair;
 import org.tron.common.utils.Sha256Hash;
-import org.tron.core.capsule.utils.MerkleTree;
 import org.tron.core.db.common.iterator.DBIterator;
 import org.tron.core.db2.common.DB;
 import org.tron.core.store.DelegationStore;
@@ -55,10 +53,11 @@ public class RewardViCalService {
 
   private volatile long lastBlockNumber = -1;
 
-  @VisibleForTesting
-  @Setter
-  private Sha256Hash rewardViRoot = Sha256Hash.wrap(
-      ByteString.fromHex("9debcb9924055500aaae98cdee10501c5c39d4daa75800a996f4bdda73dbccd8"));
+  private static final String MAIN_NET_ROOT_HEX =
+      "9debcb9924055500aaae98cdee10501c5c39d4daa75800a996f4bdda73dbccd8";
+
+  private final Sha256Hash rewardViRoot = CommonParameter.getInstance().getStorage().getDbRoot(
+      "reward-vi", Sha256Hash.wrap(ByteString.fromHex(MAIN_NET_ROOT_HEX)));
 
   private final CountDownLatch lock = new CountDownLatch(1);
 
@@ -99,25 +98,30 @@ private boolean isDone() {
   }
 
   private void maybeRun() {
-    if (enableNewRewardAlgorithm()) {
-      if (this.newRewardCalStartCycle > 1) {
-        if (isDone()) {
-          this.clearUp(true);
-          logger.info("rewardViCalService is already done");
-        } else {
-          if (lastBlockNumber ==  Long.MAX_VALUE // start rewardViCalService immediately
-              || this.getLatestBlockHeaderNumber() > lastBlockNumber) {
-            // checkpoint is flushed to db, so we can start rewardViCalService
-            startRewardCal();
-            clearUp(true);
+    try {
+      if (enableNewRewardAlgorithm()) {
+        if (this.newRewardCalStartCycle > 1) {
+          if (isDone()) {
+            this.clearUp(true);
+            logger.info("rewardViCalService is already done");
           } else {
-            logger.info("startRewardCal will run after checkpoint is flushed to db");
+            if (lastBlockNumber ==  Long.MAX_VALUE // start rewardViCalService immediately
+                || this.getLatestBlockHeaderNumber() > lastBlockNumber) {
+              // checkpoint is flushed to db, so we can start rewardViCalService
+              startRewardCal();
+              clearUp(true);
+            } else {
+              logger.info("startRewardCal will run after checkpoint is flushed to db");
+            }
           }
+        } else {
+          clearUp(false);
+          logger.info("rewardViCalService is no need to run");
         }
-      } else {
-        clearUp(false);
-        logger.info("rewardViCalService is no need to run");
       }
+    } catch (Exception e) {
+      logger.error(" Find fatal error, program will be exited soon.", e);
+      System.exit(1);
     }
   }
 
@@ -137,7 +141,7 @@ private void destroy() {
 
   public long getNewRewardAlgorithmReward(long beginCycle, long endCycle,
                                           List<Pair<byte[], Long>> votes) {
-    if (!rewardViStore.has(IS_DONE_KEY)) {
+    if (!isDone()) {
       logger.warn("rewardViCalService is not done, wait for it");
       try {
         lock.await();
@@ -174,10 +178,13 @@ private void calcMerkleRoot() {
         .map(this::getHash)
         .collect(Collectors.toCollection(ArrayList::new));
 
-    Sha256Hash rewardViRootLocal = MerkleTree.getInstance().createTree(ids).getRoot().getHash();
+    Sha256Hash rewardViRootLocal = MerkleRoot.root(ids);
     if (!Objects.equals(rewardViRoot, rewardViRootLocal)) {
-      logger.warn("merkle root mismatch, expect: {}, actual: {}",
-          rewardViRoot, rewardViRootLocal);
+      logger.warn("Merkle root mismatch, expect: {}, actual: {}."
+              + " If you are quite sure that there is no miscalculation (not on the main network)"
+              + ", please configure 'storage.merkleRoot.reward-vi = {}'"
+              + "(for a specific network such as Nile, etc.) in config.conf to fix the hints",
+          rewardViRoot, rewardViRootLocal, rewardViRootLocal);
     }
     logger.info("calcMerkleRoot: {}", rewardViRootLocal);
   }

chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java

@@ -2525,10 +2525,6 @@ public boolean useNewRewardAlgorithm() {
     return getNewRewardAlgorithmEffectiveCycle() != Long.MAX_VALUE;
   }
 
-  public boolean useNewRewardAlgorithmFromStart() {
-    return getNewRewardAlgorithmEffectiveCycle() == 1;
-  }
-
   public void saveNewRewardAlgorithmEffectiveCycle() {
     if (getNewRewardAlgorithmEffectiveCycle() == Long.MAX_VALUE) {
       long currentCycle = getCurrentCycleNumber();
@@ -2839,19 +2835,8 @@ public boolean supportMaxDelegateLockPeriod() {
             getUnfreezeDelayDays() > 0;
   }
 
-  /**
-   *  @require NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE != Long.MAX_VALUE
-   *  @require NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE > 1
-   */
   public void saveAllowOldRewardOpt(long allowOldRewardOpt) {
-    if (useNewRewardAlgorithm()) {
-      if (useNewRewardAlgorithmFromStart()) {
-        throw new IllegalStateException("no need old reward opt, ALLOW_NEW_REWARD from start");
-      }
-      this.put(ALLOW_OLD_REWARD_OPT, new BytesCapsule(ByteArray.fromLong(allowOldRewardOpt)));
-    } else {
-      throw new IllegalStateException("not support old reward opt, ALLOW_NEW_REWARD not set");
-    }
+    this.put(ALLOW_OLD_REWARD_OPT, new BytesCapsule(ByteArray.fromLong(allowOldRewardOpt)));
   }
 
   public boolean allowOldRewardOpt() {

common/src/main/java/org/tron/common/prometheus/MetricKeys.java

@@ -62,8 +62,6 @@ public static class Histogram {
     public static final String MESSAGE_PROCESS_LATENCY = "tron:message_process_latency_seconds";
     public static final String BLOCK_FETCH_LATENCY = "tron:block_fetch_latency_seconds";
     public static final String BLOCK_RECEIVE_DELAY = "tron:block_receive_delay_seconds";
-    public static final String DO_REWARD_CAL_DELAY = "tron:do_reward_cal_seconds";
-
 
     private Histogram() {
       throw new IllegalStateException("Histogram");

common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java

@@ -48,8 +48,6 @@ public class MetricsHistogram {
     init(MetricKeys.Histogram.BLOCK_FETCH_LATENCY, "fetch block latency.");
     init(MetricKeys.Histogram.BLOCK_RECEIVE_DELAY,
         "receive block delay time, receiveTime - blockTime.");
-    init(MetricKeys.Histogram.DO_REWARD_CAL_DELAY,
-        "do reward cal delay time.", "depth");
   }
 
   private MetricsHistogram() {

common/src/main/java/org/tron/common/utils/MerkleRoot.java

@@ -0,0 +1,68 @@
+package org.tron.common.utils;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import lombok.Getter;
+
+public class MerkleRoot {
+
+  private MerkleRoot() {
+
+  }
+
+  public static Sha256Hash root(List<Sha256Hash> hashList) {
+    List<Leaf> leaves = createLeaves(hashList);
+    while (leaves.size() > 1) {
+      leaves = createParentLeaves(leaves);
+    }
+    return leaves.isEmpty() ? Sha256Hash.ZERO_HASH : leaves.get(0).hash;
+  }
+
+  private static List<Leaf> createParentLeaves(List<Leaf> leaves) {
+    int step = 2;
+    int len = leaves.size();
+    return IntStream.iterate(0, i -> i + step)
+        .limit(len)
+        .filter(i -> i < len)
+        .mapToObj(i -> {
+          Leaf right = i + 1 < len ? leaves.get(i + 1) : null;
+          return createLeaf(leaves.get(i), right);
+        }).collect(Collectors.toList());
+  }
+
+  private static List<Leaf> createLeaves(List<Sha256Hash> hashList) {
+    int step = 2;
+    int len = hashList.size();
+    return IntStream.iterate(0, i -> i + step)
+        .limit(len)
+        .filter(i -> i < len)
+        .mapToObj(i -> {
+          Leaf right = i + 1 < len ? createLeaf(hashList.get(i + 1)) : null;
+          return createLeaf(createLeaf(hashList.get(i)), right);
+        }).collect(Collectors.toList());
+  }
+
+  private static Leaf createLeaf(Leaf left, Leaf right) {
+    Leaf leaf = new Leaf();
+    leaf.hash = right == null ? left.hash : computeHash(left.hash, right.hash);
+    return leaf;
+  }
+
+  private static Leaf createLeaf(Sha256Hash hash) {
+    Leaf leaf = new Leaf();
+    leaf.hash = hash;
+    return leaf;
+  }
+
+  private static Sha256Hash computeHash(Sha256Hash leftHash, Sha256Hash rightHash) {
+    return Sha256Hash.of(true,
+        leftHash.getByteString().concat(rightHash.getByteString()).toByteArray());
+  }
+
+  @Getter
+  private static class Leaf {
+
+    private Sha256Hash hash;
+  }
+}

common/src/main/java/org/tron/core/config/args/Storage.java

@@ -16,6 +16,7 @@
 package org.tron.core.config.args;
 
 import com.google.common.collect.Maps;
+import com.google.protobuf.ByteString;
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigObject;
 import java.io.File;
@@ -32,6 +33,7 @@
 import org.tron.common.utils.DbOptionalsUtils;
 import org.tron.common.utils.FileUtil;
 import org.tron.common.utils.Property;
+import org.tron.common.utils.Sha256Hash;
 
 /**
  * Custom storage configurations
@@ -79,6 +81,8 @@ public class Storage {
   private static final String CACHE_STRATEGIES = "storage.cache.strategies";
   public static final String TX_CACHE_INIT_OPTIMIZATION = "storage.txCache.initOptimization";
 
+  private static final String MERKLE_ROOT = "storage.merkleRoot";
+
   /**
    * Default values of directory
    */
@@ -163,6 +167,9 @@ public class Storage {
   @Getter
   private Map<String, Property> propertyMap;
 
+  // db root
+  private final Map<String, Sha256Hash> dbRoots = Maps.newConcurrentMap();
+
   public static String getDbEngineFromConfig(final Config config) {
     return config.hasPath(DB_ENGINE_CONFIG_KEY)
         ? config.getString(DB_ENGINE_CONFIG_KEY) : DEFAULT_DB_ENGINE;
@@ -258,6 +265,18 @@ public String getCacheStrategy(CacheType dbName) {
     return this.cacheStrategies.getOrDefault(dbName, CacheStrategies.getCacheStrategy(dbName));
   }
 
+  public Sha256Hash getDbRoot(String dbName, Sha256Hash defaultV) {
+    return this.dbRoots.getOrDefault(dbName, defaultV);
+  }
+
+  public void setDbRoots(Config config) {
+    if (config.hasPath(MERKLE_ROOT)) {
+      config.getConfig(MERKLE_ROOT).resolve().entrySet().forEach(c ->
+          this.dbRoots.put(c.getKey(),  Sha256Hash.wrap(
+              ByteString.fromHex(c.getValue().unwrapped().toString()))));
+    }
+  }
+
   private  Property createProperty(final ConfigObject conf) {
 
     Property property = new Property();

framework/src/main/java/org/tron/core/config/args/Args.java

@@ -530,6 +530,7 @@ public static void setParam(final String[] args, final String confFileName) {
     PARAMETER.storage.setDefaultDbOptions(config);
     PARAMETER.storage.setPropertyMapFromConfig(config);
     PARAMETER.storage.setCacheStrategies(config);
+    PARAMETER.storage.setDbRoots(config);
 
     PARAMETER.seedNode = new SeedNode();
     PARAMETER.seedNode.setAddressList(loadSeeds(config));
@@ -1192,9 +1193,17 @@ public static void setParam(final String[] args, final String confFileName) {
       config.hasPath(Constant.MAX_UNSOLIDIFIED_BLOCKS) ? config
         .getInt(Constant.MAX_UNSOLIDIFIED_BLOCKS) : 1000;
 
-    PARAMETER.allowOldRewardOpt =
-        config.hasPath(Constant.COMMITTEE_ALLOW_OLD_REWARD_OPT) ? config
-            .getInt(Constant.COMMITTEE_ALLOW_OLD_REWARD_OPT) : 0;
+    long allowOldRewardOpt = config.hasPath(Constant.COMMITTEE_ALLOW_OLD_REWARD_OPT) ? config
+        .getInt(Constant.COMMITTEE_ALLOW_OLD_REWARD_OPT) : 0;
+    if (allowOldRewardOpt == 1 && PARAMETER.allowNewRewardAlgorithm != 1
+        && PARAMETER.allowNewReward != 1 && PARAMETER.allowTvmVote != 1) {
+      throw new IllegalArgumentException(
+          "At least one of the following proposals is required to be opened first: "
+          + "committee.allowNewRewardAlgorithm = 1"
+          + " or committee.allowNewReward = 1"
+          + " or committee.allowTvmVote = 1.");
+    }
+    PARAMETER.allowOldRewardOpt = allowOldRewardOpt;
 
     logConfig();
   }