diff --git a/README.md b/README.md index fab0f63d303..3a21f3d63f4 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,20 @@ TRON enables large-scale development and engagement. With over 2000 transactions # Building the Source Code -Building java-tron requires `git` package and 64-bit version of `Oracle JDK 1.8` to be installed, other JDK versions are not supported yet. Make sure you operate on `Linux` and `MacOS` operating systems. +Building java-tron requires `git` package +## Operating systems +Make sure you operate on `Linux` or `MacOS` operating systems, other operating systems are not supported yet. + +## Architecture + +### x86_64 +64-bit version of `Oracle JDK 1.8` to be installed, other JDK versions are not supported yet. + +### ARM64 +64-bit version of `JDK 17` to be installed, other JDK versions are not supported yet. + +### build Clone the repo and switch to the `master` branch ```bash @@ -77,8 +89,19 @@ $ ./gradlew clean build -x test # Running java-tron -Running java-tron requires 64-bit version of `Oracle JDK 1.8` to be installed, other JDK versions are not supported yet. Make sure you operate on `Linux` and `MacOS` operating systems. +## Operating systems +Make sure you operate on `Linux` or `MacOS` operating systems, other operating systems are not supported yet. + +## Architecture + +### X86_64 +Requires 64-bit version of `Oracle JDK 1.8` to be installed, other JDK versions are not supported yet. +### ARM64 +Requires 64-bit version of `JDK 17` to be installed, other JDK versions are not supported yet. + + +## Configuration flile Get the mainnet configuration file: [main_net_config.conf](https://github.com/tronprotocol/tron-deployment/blob/master/main_net_config.conf), other network configuration files can be found [here](https://github.com/tronprotocol/tron-deployment). ## Hardware Requirements @@ -100,6 +123,7 @@ Recommended: Full node has full historical data, it is the entry point into the TRON network, it can be used by other processes as a gateway into the TRON network via HTTP and GRPC endpoints. You can interact with the TRON network through full node：transfer assets, deploy contracts, interact with contracts and so on. `-c` parameter specifies a configuration file to run a full node: +### x86_64 JDK 1.8 ```bash $ nohup java -Xms9G -Xmx9G -XX:ReservedCodeCacheSize=256m \ -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m \ @@ -111,6 +135,19 @@ $ nohup java -Xms9G -Xmx9G -XX:ReservedCodeCacheSize=256m \ -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 \ -jar FullNode.jar -c main_net_config.conf >> start.log 2>&1 & ``` +### ARM64 JDK 17 +```bash +$ nohup java -Xms9G -Xmx9G -XX:+UseZGC \ + -Xlog:gc*:file=gc.log:time,uptime,level,tags:filecount=50,filesize=100M \ + -XX:ReservedCodeCacheSize=256m \ + -XX:+UseCodeCacheFlushing \ + -XX:MetaspaceSize=256m \ + -XX:MaxMetaspaceSize=512m \ + -XX:MaxDirectMemorySize=1g \ + -XX:+HeapDumpOnOutOfMemoryError \ + -jar FullNode.jar -c main_net_config.conf >> start.log 2>&1 & +``` + ## Running a super representative node for mainnet @@ -126,6 +163,7 @@ Fill in the private key of a super representative address into the `localwitness then run the following command to start the node: +### x86_64 JDK 1.8 ```bash $ nohup java -Xms9G -Xmx9G -XX:ReservedCodeCacheSize=256m \ -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m \ @@ -137,9 +175,22 @@ $ nohup java -Xms9G -Xmx9G -XX:ReservedCodeCacheSize=256m \ -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 \ -jar FullNode.jar --witness -c main_net_config.conf >> start.log 2>&1 & ``` +### ARM64 JDK 17 +```bash +$ nohup java -Xms9G -Xmx9G -XX:+UseZGC \ + -Xlog:gc*:file=gc.log:time,uptime,level,tags:filecount=50,filesize=100M \ + -XX:ReservedCodeCacheSize=256m \ + -XX:+UseCodeCacheFlushing \ + -XX:MetaspaceSize=256m \ + -XX:MaxMetaspaceSize=512m \ + -XX:MaxDirectMemorySize=1g \ + -XX:+HeapDumpOnOutOfMemoryError \ + -jar FullNode.jar --witness -c main_net_config.conf >> start.log 2>&1 & +``` ## Quick Start Tool +### x86_64 JDK 1.8 An easier way to build and run java-tron is to use `start.sh`. `start.sh` is a quick start script written in the Shell language. You can use it to build and run java-tron quickly and easily. Here are some common use cases of the scripting tool @@ -150,6 +201,18 @@ Here are some common use cases of the scripting tool For more details, please refer to the tool [guide](./shell.md). +### ARM64 JDK 17 +You can refer to the [start.sh.simple](start.sh.simple). + +```bash +# cp start.sh.simple start.sh +# Usage: +# sh start.sh # Start the java-tron FullNode +# sh start.sh -s # Stop the java-tron FullNode +# sh start.sh [options] # Start with additional java-tron options,such as: -c config.conf -d /path_to_data, etc. +# +``` + ## Run inside Docker container One of the quickest ways to get `java-tron` up and running on your machine is by using Docker: diff --git a/actuator/src/main/java/org/tron/core/vm/VM.java b/actuator/src/main/java/org/tron/core/vm/VM.java index 2150df04c64..b1d7b027601 100644 --- a/actuator/src/main/java/org/tron/core/vm/VM.java +++ b/actuator/src/main/java/org/tron/core/vm/VM.java @@ -108,7 +108,10 @@ public static void play(Program program, JumpTable jumpTable) { } catch (JVMStackOverFlowException | OutOfTimeException e) { throw e; } catch (RuntimeException e) { - if (StringUtils.isEmpty(e.getMessage())) { + // https://openjdk.org/jeps/358 + // https://bugs.openjdk.org/browse/JDK-8220715 + // since jdk 14, the NullPointerExceptions message is not empty + if (e instanceof NullPointerException || StringUtils.isEmpty(e.getMessage())) { logger.warn("Unknown Exception occurred, tx id: {}", Hex.toHexString(program.getRootTransactionId()), e); program.setRuntimeFailure(new RuntimeException("Unknown Exception")); diff --git a/actuator/src/main/java/org/tron/core/vm/program/invoke/ProgramInvokeImpl.java b/actuator/src/main/java/org/tron/core/vm/program/invoke/ProgramInvokeImpl.java index 7997aaedcd5..ede20103609 100644 --- a/actuator/src/main/java/org/tron/core/vm/program/invoke/ProgramInvokeImpl.java +++ b/actuator/src/main/java/org/tron/core/vm/program/invoke/ProgramInvokeImpl.java @@ -314,7 +314,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return new Integer(Boolean.valueOf(byTestingSuite).hashCode() + return Boolean.valueOf(byTestingSuite).hashCode() + Boolean.valueOf(byTransaction).hashCode() + address.hashCode() + balance.hashCode() @@ -326,8 +326,7 @@ public int hashCode() { + origin.hashCode() + prevHash.hashCode() + deposit.hashCode() - + timestamp.hashCode() - ).hashCode(); + + timestamp.hashCode(); } @Override diff --git a/actuator/src/main/java/org/tron/core/vm/repository/Type.java b/actuator/src/main/java/org/tron/core/vm/repository/Type.java index e0842e6a593..9dfbd69f9ae 100644 --- a/actuator/src/main/java/org/tron/core/vm/repository/Type.java +++ b/actuator/src/main/java/org/tron/core/vm/repository/Type.java @@ -73,7 +73,7 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return new Integer(type).hashCode(); + return type; } @Override diff --git a/actuator/src/main/java/org/tron/core/vm/repository/Value.java b/actuator/src/main/java/org/tron/core/vm/repository/Value.java index bf5d99c9c94..1df758f0b3e 100644 --- a/actuator/src/main/java/org/tron/core/vm/repository/Value.java +++ b/actuator/src/main/java/org/tron/core/vm/repository/Value.java @@ -58,6 +58,6 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return new Integer(type.hashCode() + Objects.hashCode(value)).hashCode(); + return type.hashCode() + Objects.hashCode(value); } } diff --git a/build.gradle b/build.gradle index e2d90c55db0..e66ee616807 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,4 @@ +import org.gradle.nativeplatform.platform.internal.Architectures allprojects { version = "1.0.0" apply plugin: "java-library" @@ -5,13 +6,47 @@ allprojects { springVersion = "5.3.39" } } +def arch = System.getProperty("os.arch").toLowerCase() +def javaVersion = JavaVersion.current() +def isArm64 = Architectures.AARCH64.isAlias(arch) +def archSource = isArm64 ? "arm" : "x86" + +ext.archInfo = [ + name : arch, + java : javaVersion, + isArm64 : isArm64, + sourceSets: [ + main: [ + java: [ + srcDirs: ["src/main/java/common", "src/main/java/${archSource}"] + ] + ], + test: [ + java: [ + srcDirs: ["src/test/java"] + ] + ] + ], + requires: [ + JavaVersion: isArm64 ? JavaVersion.VERSION_17 : JavaVersion.VERSION_1_8, + RocksdbVersion: isArm64 ? '7.7.3' : '5.15.10' + ], + VMOptions: isArm64 ? "${rootDir}/gradle/jdk17/java-tron.vmoptions" : "${rootDir}/gradle/java-tron.vmoptions" +] + +if (!archInfo.java.is(archInfo.requires.JavaVersion)) { + throw new GradleException("Java ${archInfo.requires.JavaVersion} is required for ${archInfo.name}. Detected version ${archInfo.java}") +} + +println "Building for architecture: ${archInfo.name}, Java version: ${archInfo.java}" + subprojects { apply plugin: "jacoco" apply plugin: "maven-publish" sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.current() [compileJava, compileTestJava]*.options*.encoding = 'UTF-8' jacoco { @@ -51,10 +86,16 @@ subprojects { implementation group: 'joda-time', name: 'joda-time', version: '2.3' implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' - compileOnly 'org.projectlombok:lombok:1.18.12' - annotationProcessor 'org.projectlombok:lombok:1.18.12' - testCompileOnly 'org.projectlombok:lombok:1.18.12' - testAnnotationProcessor 'org.projectlombok:lombok:1.18.12' + compileOnly 'org.projectlombok:lombok:1.18.34' + annotationProcessor 'org.projectlombok:lombok:1.18.34' + testCompileOnly 'org.projectlombok:lombok:1.18.34' + testAnnotationProcessor 'org.projectlombok:lombok:1.18.34' + + // https://www.oracle.com/java/technologies/javase/11-relnote-issues.html#JDK-8190378 + implementation group: 'javax.annotation', name: 'javax.annotation-api', version: '1.3.2' + // for json-rpc, see https://github.com/briandilley/jsonrpc4j/issues/278 + implementation group: 'javax.jws', name: 'javax.jws-api', version: '1.1' + annotationProcessor group: 'javax.annotation', name: 'javax.annotation-api', version: '1.3.2' testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation "org.mockito:mockito-core:4.11.0" @@ -73,6 +114,10 @@ subprojects { reproducibleFileOrder = true duplicatesStrategy = DuplicatesStrategy.INCLUDE // allow duplicates } + tasks.withType(Test).configureEach { + // https://docs.gradle.org/current/dsl/org.gradle.api.tasks.testing.Test.html#org.gradle.api.tasks.testing.Test:environment + environment 'CI', 'true' + } } task copyToParent(type: Copy) { diff --git a/chainbase/build.gradle b/chainbase/build.gradle index 187ce9d32d3..1a07ff95fa5 100644 --- a/chainbase/build.gradle +++ b/chainbase/build.gradle @@ -10,8 +10,6 @@ dependencies { api project(":common") api project(":crypto") api "org.fusesource.jansi:jansi:$jansiVersion" - api group: 'commons-io', name: 'commons-io', version: '2.18.0' - api 'io.github.tronprotocol:zksnark-java-sdk:1.0.0' exclude(group: 'commons-io', module: 'commons-io') api 'org.reflections:reflections:0.9.11' } diff --git a/chainbase/src/main/java/org/tron/common/storage/OptionsPicker.java b/chainbase/src/main/java/org/tron/common/storage/OptionsPicker.java new file mode 100644 index 00000000000..acace12b39d --- /dev/null +++ b/chainbase/src/main/java/org/tron/common/storage/OptionsPicker.java @@ -0,0 +1,15 @@ +package org.tron.common.storage; + +import org.tron.common.setting.RocksDbSettings; +import org.tron.common.utils.StorageUtils; + +public class OptionsPicker { + + protected org.iq80.leveldb.Options getOptionsByDbNameForLevelDB(String dbName) { + return StorageUtils.getOptionsByDbName(dbName); + } + + protected org.rocksdb.Options getOptionsByDbNameForRocksDB(String dbName) { + return RocksDbSettings.getOptionsByDbName(dbName); + } +} diff --git a/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java b/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java index 506ecdcb6c7..72bdd80d64e 100644 --- a/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java +++ b/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java @@ -17,7 +17,10 @@ import static org.fusesource.leveldbjni.JniDBFactory.factory; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import com.google.common.collect.Sets; +import com.google.common.primitives.Bytes; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -30,18 +33,14 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; import java.util.Set; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; - -import com.google.common.primitives.Bytes; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.iq80.leveldb.CompressionType; import org.iq80.leveldb.DB; import org.iq80.leveldb.DBIterator; import org.iq80.leveldb.Logger; @@ -54,6 +53,7 @@ import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.storage.metric.DbStat; import org.tron.common.utils.FileUtil; +import org.tron.common.utils.PropUtil; import org.tron.common.utils.StorageUtils; import org.tron.core.db.common.DbSourceInter; import org.tron.core.db.common.iterator.StoreIterator; @@ -75,6 +75,7 @@ public class LevelDbDataSourceImpl extends DbStat implements DbSourceInter allKeys() { resetDbLock.readLock().lock(); @@ -243,6 +272,7 @@ public Set allKeys() { } @Deprecated + @VisibleForTesting @Override public Set allValues() { resetDbLock.readLock().lock(); @@ -362,6 +392,8 @@ public Map prefixQuery(byte[] key) { } } + @Deprecated + @VisibleForTesting @Override public long getTotal() throws RuntimeException { resetDbLock.readLock().lock(); @@ -378,13 +410,6 @@ public long getTotal() throws RuntimeException { } } - private void updateByBatchInner(Map rows) throws Exception { - try (WriteBatch batch = database.createWriteBatch()) { - innerBatchUpdate(rows,batch); - database.write(batch, writeOptions); - } - } - private void updateByBatchInner(Map rows, WriteOptions options) throws Exception { try (WriteBatch batch = database.createWriteBatch()) { innerBatchUpdate(rows,batch); @@ -404,30 +429,23 @@ private void innerBatchUpdate(Map rows, WriteBatch batch) { @Override public void updateByBatch(Map rows, WriteOptionsWrapper options) { - resetDbLock.readLock().lock(); - try { - updateByBatchInner(rows, options.level); - } catch (Exception e) { - try { - updateByBatchInner(rows, options.level); - } catch (Exception e1) { - throw new RuntimeException(e); - } - } finally { - resetDbLock.readLock().unlock(); - } + this.updateByBatch(rows, options.level); } @Override public void updateByBatch(Map rows) { + this.updateByBatch(rows, writeOptions); + } + + private void updateByBatch(Map rows, WriteOptions options) { resetDbLock.readLock().lock(); try { - updateByBatchInner(rows); + updateByBatchInner(rows, options); } catch (Exception e) { try { - updateByBatchInner(rows); + updateByBatchInner(rows, options); } catch (Exception e1) { - throw new RuntimeException(e); + throw new RuntimeException(e1); } } finally { resetDbLock.readLock().unlock(); diff --git a/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java b/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java index 5c051bdc101..af34dff670c 100644 --- a/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java +++ b/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java @@ -1,5 +1,7 @@ package org.tron.common.storage.rocksdb; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import com.google.common.collect.Sets; import com.google.common.primitives.Bytes; import java.io.File; @@ -20,10 +22,7 @@ import java.util.stream.Collectors; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.rocksdb.BlockBasedTableConfig; -import org.rocksdb.BloomFilter; import org.rocksdb.Checkpoint; -import org.rocksdb.DirectComparator; import org.rocksdb.InfoLogLevel; import org.rocksdb.Logger; import org.rocksdb.Options; @@ -31,11 +30,11 @@ import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; -import org.rocksdb.Statistics; import org.rocksdb.Status; import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; import org.slf4j.LoggerFactory; +import org.tron.common.error.TronDBException; import org.tron.common.setting.RocksDbSettings; import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.storage.metric.DbStat; @@ -53,36 +52,28 @@ public class RocksDbDataSourceImpl extends DbStat implements DbSourceInter, Iterable>, Instance { - ReadOptions readOpts; private String dataBaseName; private RocksDB database; + private Options options; private volatile boolean alive; private String parentPath; private ReadWriteLock resetDbLock = new ReentrantReadWriteLock(); private static final String KEY_ENGINE = "ENGINE"; private static final String ROCKSDB = "ROCKSDB"; - private DirectComparator comparator; private static final org.slf4j.Logger rocksDbLogger = LoggerFactory.getLogger(ROCKSDB); - public RocksDbDataSourceImpl(String parentPath, String name, RocksDbSettings settings, - DirectComparator comparator) { + public RocksDbDataSourceImpl(String parentPath, String name, Options options) { this.dataBaseName = name; this.parentPath = parentPath; - this.comparator = comparator; - RocksDbSettings.setRocksDbSettings(settings); - initDB(); - } - - public RocksDbDataSourceImpl(String parentPath, String name, RocksDbSettings settings) { - this.dataBaseName = name; - this.parentPath = parentPath; - RocksDbSettings.setRocksDbSettings(settings); + this.options = options; initDB(); } + @VisibleForTesting public RocksDbDataSourceImpl(String parentPath, String name) { this.parentPath = parentPath; this.dataBaseName = name; + this.options = RocksDbSettings.getOptionsByDbName(name); } public Path getDbPath() { @@ -125,40 +116,65 @@ public void resetDb() { } } - private boolean quitIfNotAlive() { + private void throwIfNotAlive() { if (!isAlive()) { - logger.warn("DB {} is not alive.", dataBaseName); + throw new TronDBException("DB " + this.getDBName() + " is closed."); } - return !isAlive(); } + /** copy from {@link org.fusesource.leveldbjni.internal#checkArgNotNull} */ + private static void checkArgNotNull(Object value, String name) { + if (value == null) { + throw new IllegalArgumentException("The " + name + " argument cannot be null"); + } + } + + @Deprecated + @VisibleForTesting @Override public Set allKeys() throws RuntimeException { resetDbLock.readLock().lock(); - try { - if (quitIfNotAlive()) { - return null; - } + try (final RocksIterator iter = getRocksIterator()) { Set result = Sets.newHashSet(); - try (final RocksIterator iter = getRocksIterator()) { - for (iter.seekToFirst(); iter.isValid(); iter.next()) { - result.add(iter.key()); - } - return result; + for (iter.seekToFirst(); iter.isValid(); iter.next()) { + result.add(iter.key()); } + return result; } finally { resetDbLock.readLock().unlock(); } } + @Deprecated + @VisibleForTesting @Override public Set allValues() throws RuntimeException { - return null; + resetDbLock.readLock().lock(); + try (final RocksIterator iter = getRocksIterator()) { + Set result = Sets.newHashSet(); + for (iter.seekToFirst(); iter.isValid(); iter.next()) { + result.add(iter.value()); + } + return result; + } finally { + resetDbLock.readLock().unlock(); + } } + @Deprecated + @VisibleForTesting @Override public long getTotal() throws RuntimeException { - return 0; + resetDbLock.readLock().lock(); + try (final RocksIterator iter = getRocksIterator()) { + long total = 0; + for (iter.seekToFirst(); iter.isValid(); iter.next()) { + total++; + } + return total; + } finally { + resetDbLock.readLock().unlock(); + } } @Override @@ -168,12 +184,18 @@ public String getDBName() { @Override public void setDBName(String name) { + this.dataBaseName = name; } public boolean checkOrInitEngine() { String dir = getDbPath().toString(); String enginePath = dir + File.separator + "engine.properties"; - + File currentFile = new File(dir, "CURRENT"); + if (currentFile.exists() && !Paths.get(enginePath).toFile().exists()) { + // if the CURRENT file exists, but the engine.properties file does not exist, it is LevelDB + logger.error(" You are trying to open a LevelDB database with RocksDB engine."); + return false; + } if (FileUtil.createDirIfNotExists(dir)) { if (!FileUtil.createFileIfNotExists(enginePath)) { return false; @@ -184,7 +206,7 @@ public boolean checkOrInitEngine() { // for the first init engine String engine = PropUtil.readProperty(enginePath, KEY_ENGINE); - if (engine.isEmpty() && !PropUtil.writeProperty(enginePath, KEY_ENGINE, ROCKSDB)) { + if (Strings.isNullOrEmpty(engine) && !PropUtil.writeProperty(enginePath, KEY_ENGINE, ROCKSDB)) { return false; } engine = PropUtil.readProperty(enginePath, KEY_ENGINE); @@ -194,13 +216,10 @@ public boolean checkOrInitEngine() { public void initDB() { if (!checkOrInitEngine()) { - throw new RuntimeException( - String.format("failed to check database: %s, engine do not match", dataBaseName)); + throw new RuntimeException(String.format( + "Cannot open LevelDB database '%s' with RocksDB engine." + + " Set db.engine=LEVELDB or use RocksDB database. ", dataBaseName)); } - initDB(RocksDbSettings.getSettings()); - } - - public void initDB(RocksDbSettings settings) { resetDbLock.writeLock().lock(); try { if (isAlive()) { @@ -209,81 +228,40 @@ public void initDB(RocksDbSettings settings) { if (dataBaseName == null) { throw new IllegalArgumentException("No name set to the dbStore"); } + options.setLogger(new Logger(options) { + @Override + protected void log(InfoLogLevel infoLogLevel, String logMsg) { + rocksDbLogger.info("{} {}", dataBaseName, logMsg); + } + }); - try (Options options = new Options()) { - - // most of these options are suggested by https://github.com/facebook/rocksdb/wiki/Set-Up-Options + try { + logger.debug("Opening database {}.", dataBaseName); + final Path dbPath = getDbPath(); - // general options - if (settings.isEnableStatistics()) { - options.setStatistics(new Statistics()); - options.setStatsDumpPeriodSec(60); - } - options.setCreateIfMissing(true); - options.setIncreaseParallelism(1); - options.setLevelCompactionDynamicLevelBytes(true); - options.setMaxOpenFiles(settings.getMaxOpenFiles()); - - // general options supported user config - options.setNumLevels(settings.getLevelNumber()); - options.setMaxBytesForLevelMultiplier(settings.getMaxBytesForLevelMultiplier()); - options.setMaxBytesForLevelBase(settings.getMaxBytesForLevelBase()); - options.setMaxBackgroundCompactions(settings.getCompactThreads()); - options.setLevel0FileNumCompactionTrigger(settings.getLevel0FileNumCompactionTrigger()); - options.setTargetFileSizeMultiplier(settings.getTargetFileSizeMultiplier()); - options.setTargetFileSizeBase(settings.getTargetFileSizeBase()); - if (comparator != null) { - options.setComparator(comparator); + if (!Files.isSymbolicLink(dbPath.getParent())) { + Files.createDirectories(dbPath.getParent()); } - options.setLogger(new Logger(options) { - @Override - protected void log(InfoLogLevel infoLogLevel, String logMsg) { - rocksDbLogger.info("{} {}", dataBaseName, logMsg); - } - }); - - // table options - final BlockBasedTableConfig tableCfg; - options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig()); - tableCfg.setBlockSize(settings.getBlockSize()); - tableCfg.setBlockCache(RocksDbSettings.getCache()); - tableCfg.setCacheIndexAndFilterBlocks(true); - tableCfg.setPinL0FilterAndIndexBlocksInCache(true); - tableCfg.setFilter(new BloomFilter(10, false)); - - // read options - readOpts = new ReadOptions(); - readOpts = readOpts.setPrefixSameAsStart(true) - .setVerifyChecksums(false); try { - logger.debug("Opening database {}.", dataBaseName); - final Path dbPath = getDbPath(); - - if (!Files.isSymbolicLink(dbPath.getParent())) { - Files.createDirectories(dbPath.getParent()); - } - - try { - database = RocksDB.open(options, dbPath.toString()); - } catch (RocksDBException e) { - if (Objects.equals(e.getStatus().getCode(), Status.Code.Corruption)) { - logger.error("Database {} corrupted, please delete database directory({}) " + - "and restart.", dataBaseName, parentPath, e); - } else { - logger.error("Open Database {} failed", dataBaseName, e); - } - throw new TronError(e, TronError.ErrCode.ROCKSDB_INIT); + database = RocksDB.open(options, dbPath.toString()); + } catch (RocksDBException e) { + if (Objects.equals(e.getStatus().getCode(), Status.Code.Corruption)) { + logger.error("Database {} corrupted, please delete database directory({}) " + + "and restart.", dataBaseName, parentPath, e); + } else { + logger.error("Open Database {} failed", dataBaseName, e); } - - alive = true; - } catch (IOException ioe) { - throw new RuntimeException( - String.format("failed to init database: %s", dataBaseName), ioe); + throw new TronError(e, TronError.ErrCode.ROCKSDB_INIT); } - logger.debug("Init DB {} done.", dataBaseName); + alive = true; + } catch (IOException ioe) { + throw new RuntimeException( + String.format("failed to init database: %s", dataBaseName), ioe); } + + logger.debug("Init DB {} done.", dataBaseName); } finally { resetDbLock.writeLock().unlock(); } @@ -293,9 +271,9 @@ protected void log(InfoLogLevel infoLogLevel, String logMsg) { public void putData(byte[] key, byte[] value) { resetDbLock.readLock().lock(); try { - if (quitIfNotAlive()) { - return; - } + throwIfNotAlive(); + checkArgNotNull(key, "key"); + checkArgNotNull(value, "value"); database.put(key, value); } catch (RocksDBException e) { throw new RuntimeException(dataBaseName, e); @@ -308,9 +286,8 @@ public void putData(byte[] key, byte[] value) { public byte[] getData(byte[] key) { resetDbLock.readLock().lock(); try { - if (quitIfNotAlive()) { - return null; - } + throwIfNotAlive(); + checkArgNotNull(key, "key"); return database.get(key); } catch (RocksDBException e) { throw new RuntimeException(dataBaseName, e); @@ -323,9 +300,8 @@ public byte[] getData(byte[] key) { public void deleteData(byte[] key) { resetDbLock.readLock().lock(); try { - if (quitIfNotAlive()) { - return; - } + throwIfNotAlive(); + checkArgNotNull(key, "key"); database.delete(key); } catch (RocksDBException e) { throw new RuntimeException(dataBaseName, e); @@ -344,69 +320,39 @@ public org.tron.core.db.common.iterator.DBIterator iterator() { return new RockStoreIterator(getRocksIterator()); } - private void updateByBatchInner(Map rows) throws Exception { - if (quitIfNotAlive()) { - return; - } - try (WriteBatch batch = new WriteBatch()) { - for (Map.Entry entry : rows.entrySet()) { - if (entry.getValue() == null) { - batch.delete(entry.getKey()); - } else { - batch.put(entry.getKey(), entry.getValue()); - } - } - database.write(new WriteOptions(), batch); - } - } - private void updateByBatchInner(Map rows, WriteOptions options) throws Exception { - if (quitIfNotAlive()) { - return; - } try (WriteBatch batch = new WriteBatch()) { for (Map.Entry entry : rows.entrySet()) { + checkArgNotNull(entry.getKey(), "key"); if (entry.getValue() == null) { batch.delete(entry.getKey()); } else { batch.put(entry.getKey(), entry.getValue()); } } + throwIfNotAlive(); database.write(options, batch); } } @Override public void updateByBatch(Map rows, WriteOptionsWrapper optionsWrapper) { - resetDbLock.readLock().lock(); - try { - if (quitIfNotAlive()) { - return; - } - updateByBatchInner(rows, optionsWrapper.rocks); - } catch (Exception e) { - try { - updateByBatchInner(rows); - } catch (Exception e1) { - throw new RuntimeException(dataBaseName, e1); - } - } finally { - resetDbLock.readLock().unlock(); - } + this.updateByBatch(rows, optionsWrapper.rocks); } @Override public void updateByBatch(Map rows) { + this.updateByBatch(rows, new WriteOptions()); + } + + private void updateByBatch(Map rows, WriteOptions options) { resetDbLock.readLock().lock(); try { - if (quitIfNotAlive()) { - return; - } - updateByBatchInner(rows); + updateByBatchInner(rows, options); } catch (Exception e) { try { - updateByBatchInner(rows); + updateByBatchInner(rows, options); } catch (Exception e1) { throw new RuntimeException(dataBaseName, e1); } @@ -416,45 +362,34 @@ public void updateByBatch(Map rows) { } public List getKeysNext(byte[] key, long limit) { + if (limit <= 0) { + return new ArrayList<>(); + } resetDbLock.readLock().lock(); - try { - if (quitIfNotAlive()) { - return new ArrayList<>(); - } - if (limit <= 0) { - return new ArrayList<>(); - } - - try (RocksIterator iter = getRocksIterator()) { - List result = new ArrayList<>(); - long i = 0; - for (iter.seek(key); iter.isValid() && i < limit; iter.next(), i++) { - result.add(iter.key()); - } - return result; + try (RocksIterator iter = getRocksIterator()) { + List result = new ArrayList<>(); + long i = 0; + for (iter.seek(key); iter.isValid() && i < limit; iter.next(), i++) { + result.add(iter.key()); } + return result; } finally { resetDbLock.readLock().unlock(); } } public Map getNext(byte[] key, long limit) { + if (limit <= 0) { + return Collections.emptyMap(); + } resetDbLock.readLock().lock(); - try { - if (quitIfNotAlive()) { - return null; - } - if (limit <= 0) { - return Collections.emptyMap(); - } - try (RocksIterator iter = getRocksIterator()) { - Map result = new HashMap<>(); - long i = 0; - for (iter.seek(key); iter.isValid() && i < limit; iter.next(), i++) { - result.put(iter.key(), iter.value()); - } - return result; + try (RocksIterator iter = getRocksIterator()) { + Map result = new HashMap<>(); + long i = 0; + for (iter.seek(key); iter.isValid() && i < limit; iter.next(), i++) { + result.put(iter.key(), iter.value()); } + return result; } finally { resetDbLock.readLock().unlock(); } @@ -463,78 +398,64 @@ public Map getNext(byte[] key, long limit) { @Override public Map prefixQuery(byte[] key) { resetDbLock.readLock().lock(); - try { - if (quitIfNotAlive()) { - return null; - } - try (RocksIterator iterator = getRocksIterator()) { - Map result = new HashMap<>(); - for (iterator.seek(key); iterator.isValid(); iterator.next()) { - if (Bytes.indexOf(iterator.key(), key) == 0) { - result.put(WrappedByteArray.of(iterator.key()), iterator.value()); - } else { - return result; - } + try (RocksIterator iterator = getRocksIterator()) { + Map result = new HashMap<>(); + for (iterator.seek(key); iterator.isValid(); iterator.next()) { + if (Bytes.indexOf(iterator.key(), key) == 0) { + result.put(WrappedByteArray.of(iterator.key()), iterator.value()); + } else { + return result; } - return result; } + return result; } finally { resetDbLock.readLock().unlock(); } } public Set getlatestValues(long limit) { + if (limit <= 0) { + return Sets.newHashSet(); + } resetDbLock.readLock().lock(); - try { - if (quitIfNotAlive()) { - return null; - } - if (limit <= 0) { - return Sets.newHashSet(); - } - try (RocksIterator iter = getRocksIterator()) { - Set result = Sets.newHashSet(); - long i = 0; - for (iter.seekToLast(); iter.isValid() && i < limit; iter.prev(), i++) { - result.add(iter.value()); - } - return result; + try (RocksIterator iter = getRocksIterator()) { + Set result = Sets.newHashSet(); + long i = 0; + for (iter.seekToLast(); iter.isValid() && i < limit; iter.prev(), i++) { + result.add(iter.value()); } + return result; } finally { resetDbLock.readLock().unlock(); } } - public Set getValuesNext(byte[] key, long limit) { + if (limit <= 0) { + return Sets.newHashSet(); + } resetDbLock.readLock().lock(); - try { - if (quitIfNotAlive()) { - return null; - } - if (limit <= 0) { - return Sets.newHashSet(); - } - try (RocksIterator iter = getRocksIterator()) { - Set result = Sets.newHashSet(); - long i = 0; - for (iter.seek(key); iter.isValid() && i < limit; iter.next(), i++) { - result.add(iter.value()); - } - return result; + try (RocksIterator iter = getRocksIterator()) { + Set result = Sets.newHashSet(); + long i = 0; + for (iter.seek(key); iter.isValid() && i < limit; iter.next(), i++) { + result.add(iter.value()); } + return result; } finally { resetDbLock.readLock().unlock(); } } public void backup(String dir) throws RocksDBException { + throwIfNotAlive(); Checkpoint cp = Checkpoint.create(database); cp.createCheckpoint(dir + this.getDBName()); } private RocksIterator getRocksIterator() { try ( ReadOptions readOptions = new ReadOptions().setFillCache(false)) { + throwIfNotAlive(); return database.newIterator(readOptions); } } @@ -545,7 +466,8 @@ public boolean deleteDbBakPath(String dir) { @Override public RocksDbDataSourceImpl newInstance() { - return new RocksDbDataSourceImpl(parentPath, dataBaseName, RocksDbSettings.getSettings()); + return new RocksDbDataSourceImpl(parentPath, dataBaseName, + this.options); } diff --git a/chainbase/src/main/java/org/tron/common/utils/StorageUtils.java b/chainbase/src/main/java/org/tron/common/utils/StorageUtils.java index 16df43f1534..375f0ba92fb 100644 --- a/chainbase/src/main/java/org/tron/common/utils/StorageUtils.java +++ b/chainbase/src/main/java/org/tron/common/utils/StorageUtils.java @@ -6,6 +6,7 @@ import org.apache.commons.lang3.StringUtils; import org.iq80.leveldb.Options; import org.tron.common.parameter.CommonParameter; +import org.tron.core.Constant; public class StorageUtils { @@ -52,9 +53,15 @@ public static String getOutputDirectory() { } public static Options getOptionsByDbName(String dbName) { + Options options; if (hasProperty(dbName)) { - return getProperty(dbName).getDbOptions(); + options = getProperty(dbName).getDbOptions(); + } else { + options = CommonParameter.getInstance().getStorage().newDefaultDbOptions(dbName); } - return CommonParameter.getInstance().getStorage().newDefaultDbOptions(dbName); + if (Constant.MARKET_PAIR_PRICE_TO_ORDER.equals(dbName)) { + options.comparator(new MarketOrderPriceComparatorForLevelDB()); + } + return options; } } diff --git a/chainbase/src/main/java/org/tron/core/capsule/utils/MarketUtils.java b/chainbase/src/main/java/org/tron/core/capsule/utils/MarketUtils.java index d711ac0d63b..e98bba9db08 100644 --- a/chainbase/src/main/java/org/tron/core/capsule/utils/MarketUtils.java +++ b/chainbase/src/main/java/org/tron/core/capsule/utils/MarketUtils.java @@ -25,6 +25,7 @@ import org.tron.common.crypto.Hash; import org.tron.common.utils.ByteArray; import org.tron.common.utils.ByteUtil; +import org.tron.common.utils.MarketComparator; import org.tron.core.capsule.AccountCapsule; import org.tron.core.capsule.MarketAccountOrderCapsule; import org.tron.core.capsule.MarketOrderCapsule; @@ -227,29 +228,6 @@ public static byte[] createPairKey(byte[] sellTokenId, byte[] buyTokenId) { return result; } - /** - * Note: the params should be the same token pair, or you should change the order. - * All the quantity should be bigger than 0. - * */ - public static int comparePrice(long price1SellQuantity, long price1BuyQuantity, - long price2SellQuantity, long price2BuyQuantity) { - try { - return Long.compare(multiplyExact(price1BuyQuantity, price2SellQuantity, true), - multiplyExact(price2BuyQuantity, price1SellQuantity, true)); - - } catch (ArithmeticException ex) { - // do nothing here, because we will use BigInteger to compute again - } - - BigInteger price1BuyQuantityBI = BigInteger.valueOf(price1BuyQuantity); - BigInteger price1SellQuantityBI = BigInteger.valueOf(price1SellQuantity); - BigInteger price2BuyQuantityBI = BigInteger.valueOf(price2BuyQuantity); - BigInteger price2SellQuantityBI = BigInteger.valueOf(price2SellQuantity); - - return price1BuyQuantityBI.multiply(price2SellQuantityBI) - .compareTo(price2BuyQuantityBI.multiply(price1SellQuantityBI)); - } - /** * if takerPrice >= makerPrice, return True * note: here are two different token pairs @@ -265,7 +243,8 @@ public static boolean priceMatch(MarketPrice takerPrice, MarketPrice makerPrice) // ==> Price_TRX * sellQuantity_taker/buyQuantity_taker >= Price_TRX * buyQuantity_maker/sellQuantity_maker // ==> sellQuantity_taker * sellQuantity_maker > buyQuantity_taker * buyQuantity_maker - return comparePrice(takerPrice.getBuyTokenQuantity(), takerPrice.getSellTokenQuantity(), + return MarketComparator.comparePrice(takerPrice.getBuyTokenQuantity(), + takerPrice.getSellTokenQuantity(), makerPrice.getSellTokenQuantity(), makerPrice.getBuyTokenQuantity()) >= 0; } @@ -316,57 +295,7 @@ public static void returnSellTokenRemain(MarketOrderCapsule orderCapsule, } public static int comparePriceKey(byte[] o1, byte[] o2) { - //compare pair - byte[] pair1 = new byte[TOKEN_ID_LENGTH * 2]; - byte[] pair2 = new byte[TOKEN_ID_LENGTH * 2]; - - System.arraycopy(o1, 0, pair1, 0, TOKEN_ID_LENGTH * 2); - System.arraycopy(o2, 0, pair2, 0, TOKEN_ID_LENGTH * 2); - - int pairResult = org.bouncycastle.util.Arrays.compareUnsigned(pair1, pair2); - if (pairResult != 0) { - return pairResult; - } - - //compare price - byte[] getSellTokenQuantity1 = new byte[8]; - byte[] getBuyTokenQuantity1 = new byte[8]; - - byte[] getSellTokenQuantity2 = new byte[8]; - byte[] getBuyTokenQuantity2 = new byte[8]; - - int longByteNum = 8; - - System.arraycopy(o1, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, - getSellTokenQuantity1, 0, longByteNum); - System.arraycopy(o1, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum, - getBuyTokenQuantity1, 0, longByteNum); - - System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, - getSellTokenQuantity2, 0, longByteNum); - System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum, - getBuyTokenQuantity2, 0, longByteNum); - - long sellTokenQuantity1 = ByteArray.toLong(getSellTokenQuantity1); - long buyTokenQuantity1 = ByteArray.toLong(getBuyTokenQuantity1); - long sellTokenQuantity2 = ByteArray.toLong(getSellTokenQuantity2); - long buyTokenQuantity2 = ByteArray.toLong(getBuyTokenQuantity2); - - if ((sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0) - && (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0)) { - return 0; - } - - if (sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0) { - return -1; - } - - if (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0) { - return 1; - } - - return comparePrice(sellTokenQuantity1, buyTokenQuantity1, - sellTokenQuantity2, buyTokenQuantity2); + return MarketComparator.comparePriceKey(o1, o2); } diff --git a/chainbase/src/main/java/org/tron/core/db/TronDatabase.java b/chainbase/src/main/java/org/tron/core/db/TronDatabase.java index d791e189eb4..8630fbdcdff 100644 --- a/chainbase/src/main/java/org/tron/core/db/TronDatabase.java +++ b/chainbase/src/main/java/org/tron/core/db/TronDatabase.java @@ -9,9 +9,9 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.iq80.leveldb.WriteOptions; -import org.rocksdb.DirectComparator; import org.springframework.beans.factory.annotation.Autowired; import org.tron.common.parameter.CommonParameter; +import org.tron.common.storage.OptionsPicker; import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.storage.leveldb.LevelDbDataSourceImpl; import org.tron.common.storage.metric.DbStatService; @@ -24,7 +24,7 @@ import org.tron.core.exception.ItemNotFoundException; @Slf4j(topic = "DB") -public abstract class TronDatabase implements ITronChainBase { +public abstract class TronDatabase extends OptionsPicker implements ITronChainBase { protected DbSourceInter dbSource; @Getter @@ -51,8 +51,7 @@ protected TronDatabase(String dbName) { String parentName = Paths.get(StorageUtils.getOutputDirectoryByDbName(dbName), CommonParameter.getInstance().getStorage().getDbDirectory()).toString(); dbSource = - new RocksDbDataSourceImpl(parentName, dbName, CommonParameter.getInstance() - .getRocksDBCustomSettings(), getDirectComparator()); + new RocksDbDataSourceImpl(parentName, dbName, getOptionsByDbNameForRocksDB(dbName)); } dbSource.initDB(); @@ -66,14 +65,6 @@ protected void init() { protected TronDatabase() { } - protected org.iq80.leveldb.Options getOptionsByDbNameForLevelDB(String dbName) { - return StorageUtils.getOptionsByDbName(dbName); - } - - protected DirectComparator getDirectComparator() { - return null; - } - public DbSourceInter getDbSource() { return dbSource; } diff --git a/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java b/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java index 4b75ddee3a4..4952b70478d 100644 --- a/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java +++ b/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java @@ -16,9 +16,9 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.iq80.leveldb.WriteOptions; -import org.rocksdb.DirectComparator; import org.springframework.beans.factory.annotation.Autowired; import org.tron.common.parameter.CommonParameter; +import org.tron.common.storage.OptionsPicker; import org.tron.common.storage.leveldb.LevelDbDataSourceImpl; import org.tron.common.storage.metric.DbStatService; import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl; @@ -38,7 +38,7 @@ @Slf4j(topic = "DB") -public abstract class TronStoreWithRevoking implements ITronChainBase { +public abstract class TronStoreWithRevoking extends OptionsPicker implements ITronChainBase { @Getter // only for unit test protected IRevokingDB revokingDB; @@ -69,22 +69,13 @@ protected TronStoreWithRevoking(String dbName) { .getInstance().getStorage().getDbDirectory()).toString(); this.db = new RocksDB( new RocksDbDataSourceImpl(parentPath, - dbName, CommonParameter.getInstance() - .getRocksDBCustomSettings(), getDirectComparator())); + dbName, getOptionsByDbNameForRocksDB(dbName))); } else { throw new RuntimeException(String.format("db engine %s is error", dbEngine)); } this.revokingDB = new Chainbase(new SnapshotRoot(this.db)); } - protected org.iq80.leveldb.Options getOptionsByDbNameForLevelDB(String dbName) { - return StorageUtils.getOptionsByDbName(dbName); - } - - protected DirectComparator getDirectComparator() { - return null; - } - protected TronStoreWithRevoking(DB db) { this.db = db; this.revokingDB = new Chainbase(new SnapshotRoot(db)); diff --git a/chainbase/src/main/java/org/tron/core/db/common/DbSourceInter.java b/chainbase/src/main/java/org/tron/core/db/common/DbSourceInter.java index 0823b7a7cf4..18461860fc2 100755 --- a/chainbase/src/main/java/org/tron/core/db/common/DbSourceInter.java +++ b/chainbase/src/main/java/org/tron/core/db/common/DbSourceInter.java @@ -17,11 +17,10 @@ */ package org.tron.core.db.common; -import org.tron.core.db2.common.WrappedByteArray; - +import com.google.common.annotations.VisibleForTesting; import java.util.Map; import java.util.Set; - +import org.tron.core.db2.common.WrappedByteArray; public interface DbSourceInter extends BatchSourceInter, Iterable> { @@ -38,10 +37,16 @@ public interface DbSourceInter extends BatchSourceInter, void resetDb(); + @VisibleForTesting + @Deprecated Set allKeys() throws RuntimeException; + @VisibleForTesting + @Deprecated Set allValues() throws RuntimeException; + @VisibleForTesting + @Deprecated long getTotal() throws RuntimeException; void stat(); diff --git a/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java b/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java index 541f71348af..105d845dc64 100644 --- a/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java +++ b/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java @@ -47,7 +47,7 @@ public boolean hasNext() { try { close(); } catch (Exception e1) { - logger.error(e.getMessage(), e); + logger.error(e1.getMessage(), e1); } } return hasNext; @@ -79,6 +79,11 @@ public byte[] setValue(byte[] value) { }; } + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + @Override public void seek(byte[] key) { checkState(); diff --git a/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java b/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java index d771716a7e8..c2803f99637 100755 --- a/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java +++ b/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java @@ -46,6 +46,11 @@ public boolean hasNext() { } } catch (Exception e) { logger.error(e.getMessage(), e); + try { + close(); + } catch (Exception e1) { + logger.error(e1.getMessage(), e1); + } } return hasNext; diff --git a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java index 9d2409685c9..db9516d6203 100644 --- a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java +++ b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java @@ -35,6 +35,7 @@ import org.tron.common.parameter.CommonParameter; import org.tron.common.prometheus.MetricKeys; import org.tron.common.prometheus.Metrics; +import org.tron.common.storage.OptionsPicker; import org.tron.common.storage.leveldb.LevelDbDataSourceImpl; import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl; import org.tron.common.utils.ByteArray; @@ -48,7 +49,7 @@ import org.tron.core.store.DynamicPropertiesStore; @Slf4j(topic = "DB") -public class TxCacheDB implements DB, Flusher { +public class TxCacheDB extends OptionsPicker implements DB, Flusher { // > 65_536(= 2^16) blocks, that is the number of the reference block private static final long MAX_BLOCK_SIZE = 65536; @@ -106,7 +107,7 @@ public TxCacheDB(String name, RecentTransactionStore recentTransactionStore, if ("LEVELDB".equals(dbEngine.toUpperCase())) { this.persistentStore = new LevelDB( new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(name), - name, StorageUtils.getOptionsByDbName(name), + name, getOptionsByDbNameForLevelDB(name), new WriteOptions().sync(CommonParameter.getInstance() .getStorage().isDbSync()))); } else if ("ROCKSDB".equals(dbEngine.toUpperCase())) { @@ -115,9 +116,7 @@ public TxCacheDB(String name, RecentTransactionStore recentTransactionStore, .getInstance().getStorage().getDbDirectory()).toString(); this.persistentStore = new RocksDB( - new RocksDbDataSourceImpl(parentPath, - name, CommonParameter.getInstance() - .getRocksDBCustomSettings())); + new RocksDbDataSourceImpl(parentPath, name, getOptionsByDbNameForRocksDB(name))); } else { throw new RuntimeException(String.format("db type: %s is not supported", dbEngine)); } diff --git a/chainbase/src/main/java/org/tron/core/store/MarketPairPriceToOrderStore.java b/chainbase/src/main/java/org/tron/core/store/MarketPairPriceToOrderStore.java index 605952328ed..391fb4249c8 100644 --- a/chainbase/src/main/java/org/tron/core/store/MarketPairPriceToOrderStore.java +++ b/chainbase/src/main/java/org/tron/core/store/MarketPairPriceToOrderStore.java @@ -3,16 +3,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.iq80.leveldb.Options; -import org.rocksdb.ComparatorOptions; -import org.rocksdb.DirectComparator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.tron.common.utils.ByteUtil; -import org.tron.common.utils.MarketOrderPriceComparatorForLevelDB; -import org.tron.common.utils.MarketOrderPriceComparatorForRockDB; -import org.tron.common.utils.StorageUtils; +import org.tron.core.Constant; import org.tron.core.capsule.MarketOrderIdListCapsule; import org.tron.core.capsule.utils.MarketUtils; import org.tron.core.db.TronStoreWithRevoking; @@ -22,24 +17,10 @@ public class MarketPairPriceToOrderStore extends TronStoreWithRevoking { @Autowired - protected MarketPairPriceToOrderStore(@Value("market_pair_price_to_order") String dbName) { + protected MarketPairPriceToOrderStore(@Value(Constant.MARKET_PAIR_PRICE_TO_ORDER) String dbName) { super(dbName); } - @Override - protected Options getOptionsByDbNameForLevelDB(String dbName) { - Options options = StorageUtils.getOptionsByDbName(dbName); - options.comparator(new MarketOrderPriceComparatorForLevelDB()); - return options; - } - - //todo: to test later - @Override - protected DirectComparator getDirectComparator() { - ComparatorOptions comparatorOptions = new ComparatorOptions(); - return new MarketOrderPriceComparatorForRockDB(comparatorOptions); - } - @Override public MarketOrderIdListCapsule get(byte[] key) throws ItemNotFoundException { byte[] value = revokingDB.get(key); diff --git a/common/build.gradle b/common/build.gradle index 8a1c87c2e46..83fe638ae50 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -6,28 +6,6 @@ version '1.0.0' sourceCompatibility = 1.8 -// Dependency versions -// --------------------------------------- -def leveldbVersion = "1.8" -// -------------------------------------- - -static def isWindows() { - return org.gradle.internal.os.OperatingSystem.current().isWindows() -} - -if (isWindows()) { - ext { - leveldbGroup = "org.ethereum" - leveldbName = "leveldbjni-all" - leveldbVersion = "1.18.3" - } -} else { - ext { - leveldbGroup = "org.fusesource.leveldbjni" - leveldbName = "leveldbjni-all" - leveldbVersion = "1.8" - } -} dependencies { api group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.18.3' // https://github.com/FasterXML/jackson-databind/issues/3627 @@ -36,14 +14,14 @@ dependencies { api group: 'commons-codec', name: 'commons-codec', version: '1.11' api group: 'com.beust', name: 'jcommander', version: '1.78' api group: 'com.typesafe', name: 'config', version: '1.3.2' - api group: leveldbGroup, name: leveldbName, version: leveldbVersion - api group: 'org.rocksdb', name: 'rocksdbjni', version: '5.15.10' api group: 'io.prometheus', name: 'simpleclient', version: '0.15.0' api group: 'io.prometheus', name: 'simpleclient_httpserver', version: '0.15.0' api group: 'io.prometheus', name: 'simpleclient_hotspot', version: '0.15.0' - api 'org.aspectj:aspectjrt:1.8.13' - api 'org.aspectj:aspectjweaver:1.8.13' - api 'org.aspectj:aspectjtools:1.8.13' + // https://openjdk.org/jeps/396, JEP 396: Strongly Encapsulate JDK Internals by Default + // https://eclipse.dev/aspectj/doc/latest/release/JavaVersionCompatibility.html + api 'org.aspectj:aspectjrt:1.9.8' + api 'org.aspectj:aspectjweaver:1.9.8' + api 'org.aspectj:aspectjtools:1.9.8' api group: 'io.github.tronprotocol', name: 'libp2p', version: '2.2.6',{ exclude group: 'io.grpc', module: 'grpc-context' exclude group: 'io.grpc', module: 'grpc-core' @@ -62,6 +40,7 @@ dependencies { exclude group: 'org.bouncycastle', module: 'bcutil-jdk18on' } api project(":protocol") + api project(":platform") } jacocoTestReport { diff --git a/common/src/main/java/org/tron/common/setting/RocksDbSettings.java b/common/src/main/java/org/tron/common/setting/RocksDbSettings.java index 7436150cae2..5c2e5eacddf 100644 --- a/common/src/main/java/org/tron/common/setting/RocksDbSettings.java +++ b/common/src/main/java/org/tron/common/setting/RocksDbSettings.java @@ -1,16 +1,21 @@ package org.tron.common.setting; +import java.util.Arrays; import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.BloomFilter; +import org.rocksdb.ComparatorOptions; import org.rocksdb.LRUCache; +import org.rocksdb.Options; import org.rocksdb.RocksDB; +import org.rocksdb.Statistics; +import org.tron.common.utils.MarketOrderPriceComparatorForRocksDB; +import org.tron.core.Constant; @Slf4j public class RocksDbSettings { - @Setter - @Getter private static RocksDbSettings rocksDbSettings; @Getter @@ -40,6 +45,15 @@ public class RocksDbSettings { private static final LRUCache cache = new LRUCache(1 * 1024 * 1024 * 1024L); + private static final String[] CI_ENVIRONMENT_VARIABLES = { + "CI", + "JENKINS_URL", + "TRAVIS", + "CIRCLECI", + "GITHUB_ACTIONS", + "GITLAB_CI" + }; + private RocksDbSettings() { } @@ -60,9 +74,9 @@ public static RocksDbSettings initCustomSettings(int levelNumber, int compactThr int blockSize, long maxBytesForLevelBase, double maxBytesForLevelMultiplier, int level0FileNumCompactionTrigger, long targetFileSizeBase, - int targetFileSizeMultiplier) { + int targetFileSizeMultiplier, int maxOpenFiles) { rocksDbSettings = new RocksDbSettings() - .withMaxOpenFiles(5000) + .withMaxOpenFiles(maxOpenFiles) .withEnableStatistics(false) .withLevelNumber(levelNumber) .withCompactThreads(compactThreads) @@ -76,16 +90,17 @@ public static RocksDbSettings initCustomSettings(int levelNumber, int compactThr } public static void loggingSettings() { - logger.info(String.format( - "level number: %d, CompactThreads: %d, Blocksize: %d, maxBytesForLevelBase: %d," - + " withMaxBytesForLevelMultiplier: %f, level0FileNumCompactionTrigger: %d, " - + "withTargetFileSizeBase: %d, withTargetFileSizeMultiplier: %d", + logger.info( + "level number: {}, CompactThreads: {}, Blocksize:{}, maxBytesForLevelBase: {}," + + " withMaxBytesForLevelMultiplier: {}, level0FileNumCompactionTrigger: {}, " + + "withTargetFileSizeBase: {}, withTargetFileSizeMultiplier: {}, maxOpenFiles: {}", rocksDbSettings.getLevelNumber(), rocksDbSettings.getCompactThreads(), rocksDbSettings.getBlockSize(), rocksDbSettings.getMaxBytesForLevelBase(), rocksDbSettings.getMaxBytesForLevelMultiplier(), rocksDbSettings.getLevel0FileNumCompactionTrigger(), - rocksDbSettings.getTargetFileSizeBase(), rocksDbSettings.getTargetFileSizeMultiplier())); + rocksDbSettings.getTargetFileSizeBase(), rocksDbSettings.getTargetFileSizeMultiplier(), + rocksDbSettings.getMaxOpenFiles()); } public RocksDbSettings withMaxOpenFiles(int maxOpenFiles) { @@ -140,4 +155,63 @@ public RocksDbSettings withTargetFileSizeMultiplier(int targetFileSizeMultiplier public static LRUCache getCache() { return cache; } + + public static Options getOptionsByDbName(String dbName) { + RocksDbSettings settings = getSettings(); + + Options options = new Options(); + + // most of these options are suggested by https://github.com/facebook/rocksdb/wiki/Set-Up-Options + + // general options + if (settings.isEnableStatistics()) { + options.setStatistics(new Statistics()); + options.setStatsDumpPeriodSec(60); + } + options.setCreateIfMissing(true); + options.setIncreaseParallelism(1); + options.setLevelCompactionDynamicLevelBytes(true); + options.setMaxOpenFiles(settings.getMaxOpenFiles()); + + // general options supported user config + options.setNumLevels(settings.getLevelNumber()); + options.setMaxBytesForLevelMultiplier(settings.getMaxBytesForLevelMultiplier()); + options.setMaxBytesForLevelBase(settings.getMaxBytesForLevelBase()); + options.setMaxBackgroundCompactions(settings.getCompactThreads()); + options.setLevel0FileNumCompactionTrigger(settings.getLevel0FileNumCompactionTrigger()); + options.setTargetFileSizeMultiplier(settings.getTargetFileSizeMultiplier()); + options.setTargetFileSizeBase(settings.getTargetFileSizeBase()); + + // table options + final BlockBasedTableConfig tableCfg; + options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig()); + tableCfg.setBlockSize(settings.getBlockSize()); + tableCfg.setBlockCache(RocksDbSettings.getCache()); + tableCfg.setCacheIndexAndFilterBlocks(true); + tableCfg.setPinL0FilterAndIndexBlocksInCache(true); + tableCfg.setFilter(new BloomFilter(10, false)); + if (Constant.MARKET_PAIR_PRICE_TO_ORDER.equals(dbName)) { + ComparatorOptions comparatorOptions = new ComparatorOptions(); + options.setComparator(new MarketOrderPriceComparatorForRocksDB(comparatorOptions)); + } + + if (isRunningInCI()) { + // Disable fallocate calls to avoid issues with disk space + options.setAllowFAllocate(false); + // Set WAL size limits to avoid excessive disk + options.setMaxTotalWalSize(2 * 1024 * 1024); + // Set recycle log file + options.setRecycleLogFileNum(1); + // Enable creation of missing column families + options.setCreateMissingColumnFamilies(true); + // Set max background flushes to 1 to reduce resource usage + options.setMaxBackgroundFlushes(1); + } + + return options; + } + + private static boolean isRunningInCI() { + return Arrays.stream(CI_ENVIRONMENT_VARIABLES).anyMatch(System.getenv()::containsKey); + } } diff --git a/common/src/main/java/org/tron/core/Constant.java b/common/src/main/java/org/tron/core/Constant.java index 2a6104fe6e6..c5e0ddc0ecc 100644 --- a/common/src/main/java/org/tron/core/Constant.java +++ b/common/src/main/java/org/tron/core/Constant.java @@ -24,6 +24,10 @@ public class Constant { public static final int NODE_TYPE_FULL_NODE = 0; public static final int NODE_TYPE_LIGHT_NODE = 1; + // DB NAME + public static final String MARKET_PAIR_PRICE_TO_ORDER = "market_pair_price_to_order"; + // DB NAME + // config for transaction public static final long TRANSACTION_MAX_BYTE_SIZE = 500 * 1_024L; public static final int CREATE_ACCOUNT_TRANSACTION_MIN_BYTE_SIZE = 500; diff --git a/common/src/main/java/org/tron/core/config/args/Storage.java b/common/src/main/java/org/tron/core/config/args/Storage.java index 9cf6eb6bab1..655b6b779fe 100644 --- a/common/src/main/java/org/tron/core/config/args/Storage.java +++ b/common/src/main/java/org/tron/core/config/args/Storage.java @@ -25,9 +25,11 @@ import java.util.stream.Collectors; import lombok.Getter; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.iq80.leveldb.CompressionType; import org.iq80.leveldb.Options; +import org.tron.common.arch.Arch; import org.tron.common.cache.CacheStrategies; import org.tron.common.cache.CacheType; import org.tron.common.utils.DbOptionalsUtils; @@ -42,6 +44,7 @@ * @version 1.0 * @since 2018/5/25 */ +@Slf4j(topic = "db") public class Storage { /** @@ -87,6 +90,7 @@ public class Storage { * Default values of directory */ private static final String DEFAULT_DB_ENGINE = "LEVELDB"; + private static final String ROCKS_DB_ENGINE = "ROCKSDB"; private static final boolean DEFAULT_DB_SYNC = false; private static final boolean DEFAULT_EVENT_SUBSCRIBE_CONTRACT_PARSE = true; private static final String DEFAULT_DB_DIRECTORY = "database"; @@ -171,6 +175,11 @@ public class Storage { private final Map dbRoots = Maps.newConcurrentMap(); public static String getDbEngineFromConfig(final Config config) { + if (Arch.isArm64()) { + // if is arm64 but config is leveldb, should throw exception? + logger.warn("Arm64 architecture detected, using RocksDB as db engine, ignore config."); + return ROCKS_DB_ENGINE; + } return config.hasPath(DB_ENGINE_CONFIG_KEY) ? config.getString(DB_ENGINE_CONFIG_KEY) : DEFAULT_DB_ENGINE; } diff --git a/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp12.java b/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp12.java index 26ea708fbe4..f144656b0ba 100644 --- a/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp12.java +++ b/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp12.java @@ -356,7 +356,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return new Integer(a.hashCode() + b.hashCode()).hashCode(); + return a.hashCode() + b.hashCode(); } @Override diff --git a/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp2.java b/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp2.java index ba2a1ceb477..162a5b13b30 100644 --- a/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp2.java +++ b/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp2.java @@ -164,7 +164,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return new Integer(a.hashCode() + b.hashCode()).hashCode(); + return a.hashCode() + b.hashCode(); } Fp2 frobeniusMap(int power) { diff --git a/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp6.java b/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp6.java index 0636cc334f1..fb863bb6b34 100644 --- a/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp6.java +++ b/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp6.java @@ -246,6 +246,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return new Integer(a.hashCode() + b.hashCode() + c.hashCode()).hashCode(); + return a.hashCode() + b.hashCode() + c.hashCode(); } } diff --git a/docker/arm64/Dockerfile b/docker/arm64/Dockerfile new file mode 100644 index 00000000000..6435faf7ead --- /dev/null +++ b/docker/arm64/Dockerfile @@ -0,0 +1,33 @@ +FROM arm64v8/eclipse-temurin:17 + +ENV TMP_DIR="/tron-build" +ENV BASE_DIR="/java-tron" + +RUN set -o errexit -o nounset \ + && apt-get update \ + && apt-get -y install git p7zip-full wget libtcmalloc-minimal4 \ + && echo "git clone" \ + && mkdir -p $TMP_DIR \ + && cd $TMP_DIR \ + && git clone https://github.com/tronprotocol/java-tron.git \ + && cd java-tron \ + && git checkout master \ + && ./gradlew clean build -x test -x check --no-daemon \ + && cd build/distributions \ + && 7za x -y java-tron-1.0.0.zip \ + && mv java-tron-1.0.0 $BASE_DIR \ + && rm -rf $TMP_DIR \ + && rm -rf ~/.gradle \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ENV LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4" +ENV TCMALLOC_RELEASE_RATE=10 + +RUN wget -P $BASE_DIR/config https://raw.githubusercontent.com/tronprotocol/tron-deployment/master/main_net_config.conf + +COPY docker-entrypoint.sh $BASE_DIR/bin + +WORKDIR $BASE_DIR + +ENTRYPOINT ["./bin/docker-entrypoint.sh"] \ No newline at end of file diff --git a/framework/build.gradle b/framework/build.gradle index 9763f2980e2..7637bfe7899 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -116,6 +116,17 @@ test { destinationFile = file("$buildDir/jacoco/jacocoTest.exec") classDumpDir = file("$buildDir/jacoco/classpathdumps") } + if (rootProject.archInfo.isArm64) { + exclude { element -> + element.file.name.toLowerCase().contains('leveldb') + } + filter { + excludeTestsMatching '*.*leveldb*' + excludeTestsMatching '*.*Leveldb*' + excludeTestsMatching '*.*LevelDB*' + excludeTestsMatching '*.*LevelDb*' + } + } if (isWindows()) { exclude '**/ShieldedTransferActuatorTest.class' exclude '**/BackupDbUtilTest.class' @@ -151,7 +162,7 @@ def binaryRelease(taskName, jarName, mainClass) { // explicit_dependency dependsOn (project(':actuator').jar, project(':consensus').jar, project(':chainbase').jar, - project(':crypto').jar, project(':common').jar, project(':protocol').jar) + project(':crypto').jar, project(':common').jar, project(':protocol').jar, project(':platform').jar) from { configurations.runtimeClasspath.collect { @@ -197,8 +208,7 @@ def createScript(project, mainClass, name) { } } } - -applicationDistribution.from("../gradle/java-tron.vmoptions") { +applicationDistribution.from(rootProject.archInfo.VMOptions) { into "bin" } //distZip { diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index b7c7d95cdae..6f9d7ed4cec 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -47,6 +47,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.tron.common.arch.Arch; import org.tron.common.args.Account; import org.tron.common.args.GenesisBlock; import org.tron.common.args.Witness; @@ -376,6 +377,7 @@ private static Map getOptionGroup() { * set parameters. */ public static void setParam(final String[] args, final String confFileName) { + Arch.throwUnsupportedJavaException(); JCommander.newBuilder().addObject(PARAMETER).build().parse(args); if (PARAMETER.version) { printVersion(); @@ -1721,11 +1723,13 @@ private static void initRocksDbSettings(Config config) { .getLong(prefix + "targetFileSizeBase") : 64; int targetFileSizeMultiplier = config.hasPath(prefix + "targetFileSizeMultiplier") ? config .getInt(prefix + "targetFileSizeMultiplier") : 1; + int maxOpenFiles = config.hasPath(prefix + "maxOpenFiles") + ? config.getInt(prefix + "maxOpenFiles") : 5000; PARAMETER.rocksDBCustomSettings = RocksDbSettings .initCustomSettings(levelNumber, compactThreads, blocksize, maxBytesForLevelBase, maxBytesForLevelMultiplier, level0FileNumCompactionTrigger, - targetFileSizeBase, targetFileSizeMultiplier); + targetFileSizeBase, targetFileSizeMultiplier, maxOpenFiles); RocksDbSettings.loggingSettings(); } @@ -1762,6 +1766,8 @@ private static void initBackupProperty(Config config) { public static void logConfig() { CommonParameter parameter = CommonParameter.getInstance(); logger.info("\n"); + logger.info("************************ System info ************************"); + logger.info("{}", Arch.withAll()); logger.info("************************ Net config ************************"); logger.info("P2P version: {}", parameter.getNodeP2pVersion()); logger.info("LAN IP: {}", parameter.getNodeLanIp()); diff --git a/framework/src/main/java/org/tron/core/services/http/GetProposalByIdServlet.java b/framework/src/main/java/org/tron/core/services/http/GetProposalByIdServlet.java index a903a5b4920..9d7805d4f98 100644 --- a/framework/src/main/java/org/tron/core/services/http/GetProposalByIdServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/GetProposalByIdServlet.java @@ -26,7 +26,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { try { boolean visible = Util.getVisible(request); String input = request.getParameter("id"); - long id = new Long(input); + long id = Long.parseLong(input); fillResponse(ByteString.copyFrom(ByteArray.fromLong(id)), visible, response); } catch (Exception e) { Util.processError(e, response); diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java index eb432432a1c..afc401e7e7c 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java @@ -24,7 +24,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.regex.Matcher; import java.util.regex.Pattern; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -230,15 +229,11 @@ public static void handleLogsFilter(LogsFilterCapsule logsFilterCapsule) { @Override public String web3ClientVersion() { - Pattern shortVersion = Pattern.compile("(\\d\\.\\d).*"); - Matcher matcher = shortVersion.matcher(System.getProperty("java.version")); - matcher.matches(); - return String.join("/", Arrays.asList( "TRON", "v" + Version.getVersion(), System.getProperty("os.name"), - "Java" + matcher.group(1))); + "Java" + System.getProperty("java.specification.version"))); } @Override diff --git a/framework/src/main/resources/config.conf b/framework/src/main/resources/config.conf index 7603ef6ffbb..55e4b8c55d9 100644 --- a/framework/src/main/resources/config.conf +++ b/framework/src/main/resources/config.conf @@ -5,7 +5,7 @@ net { storage { # Directory for storing persistent data - db.engine = "LEVELDB", + db.engine = "LEVELDB", // deprecated for arm, because arm only support "ROCKSDB". db.sync = false, db.directory = "database", index.directory = "index", @@ -19,7 +19,7 @@ storage { # Otherwise, db configs will remain default and data will be stored in # the path of "output-directory" or which is set by "-d" ("--output-directory"). - # setting can impove leveldb performance .... start + # setting can impove leveldb performance .... start, deprecated for arm # node: if this will increase process fds,you may be check your ulimit if 'too many open files' error occurs # see https://github.com/tronprotocol/tips/blob/master/tip-343.md for detail # if you find block sync has lower performance,you can try this settings @@ -32,21 +32,21 @@ storage { #defaultL = { # maxOpenFiles = 1000 #} - # setting can impove leveldb performance .... end + # setting can impove leveldb performance .... end, deprecated for arm # Attention: name is a required field that must be set !!! properties = [ // { // name = "account", // path = "storage_directory_test", - // createIfMissing = true, + // createIfMissing = true, // deprecated for arm start // paranoidChecks = true, // verifyChecksums = true, // compressionType = 1, // compressed with snappy // blockSize = 4096, // 4 KB = 4 * 1024 B // writeBufferSize = 10485760, // 10 MB = 10 * 1024 * 1024 B // cacheSize = 10485760, // 10 MB = 10 * 1024 * 1024 B - // maxOpenFiles = 100 + // maxOpenFiles = 100 // deprecated for arm end // }, // { // name = "account-index", diff --git a/framework/src/test/java/org/tron/common/storage/leveldb/LevelDbDataSourceImplTest.java b/framework/src/test/java/org/tron/common/storage/leveldb/LevelDbDataSourceImplTest.java index bf18b988f19..c8b29c3020d 100644 --- a/framework/src/test/java/org/tron/common/storage/leveldb/LevelDbDataSourceImplTest.java +++ b/framework/src/test/java/org/tron/common/storage/leveldb/LevelDbDataSourceImplTest.java @@ -28,6 +28,7 @@ import com.google.common.collect.Sets; import java.io.File; import java.io.IOException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -38,15 +39,24 @@ import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.iq80.leveldb.DBException; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDB; +import org.tron.common.parameter.CommonParameter; +import org.tron.common.storage.WriteOptionsWrapper; +import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl; import org.tron.common.utils.ByteArray; import org.tron.common.utils.FileUtil; +import org.tron.common.utils.PropUtil; import org.tron.common.utils.PublicMethod; +import org.tron.common.utils.StorageUtils; import org.tron.core.Constant; import org.tron.core.config.args.Args; import org.tron.core.db2.common.WrappedByteArray; @@ -73,6 +83,14 @@ public class LevelDbDataSourceImplTest { private byte[] key5 = "00000005aa".getBytes(); private byte[] key6 = "00000006aa".getBytes(); + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + static { + RocksDB.loadLibrary(); + } + /** * Release resources. */ @@ -102,8 +120,19 @@ public void testPutGet() { assertNotNull(dataSourceTest.getData(key)); assertEquals(1, dataSourceTest.allKeys().size()); + assertEquals(1, dataSourceTest.getTotal()); + assertEquals(1, dataSourceTest.allValues().size()); assertEquals("50000", ByteArray.toStr(dataSourceTest.getData(key1.getBytes()))); + dataSourceTest.deleteData(key); + assertNull(dataSourceTest.getData(key)); + assertEquals(0, dataSourceTest.getTotal()); + dataSourceTest.iterator().forEachRemaining(entry -> Assert.fail("iterator should be empty")); + dataSourceTest.stream().forEach(entry -> Assert.fail("stream should be empty")); + dataSourceTest.stat(); dataSourceTest.closeDB(); + dataSourceTest.stat(); // stat again + exception.expect(DBException.class); + dataSourceTest.deleteData(key); } @Test @@ -142,6 +171,23 @@ public void testupdateByBatchInner() { assertEquals("50000", ByteArray.toStr(dataSource.getData(key1.getBytes()))); assertEquals("10000", ByteArray.toStr(dataSource.getData(key2.getBytes()))); assertEquals(2, dataSource.allKeys().size()); + + rows.clear(); + rows.put(key1.getBytes(), null); + rows.put(key2.getBytes(), null); + dataSource.updateByBatch(rows, WriteOptionsWrapper.getInstance()); + assertEquals(0, dataSource.allKeys().size()); + + rows.clear(); + rows.put(key1.getBytes(), value1.getBytes()); + rows.put(key2.getBytes(), null); + dataSource.updateByBatch(rows); + assertEquals("50000", ByteArray.toStr(dataSource.getData(key1.getBytes()))); + assertEquals(1, dataSource.allKeys().size()); + rows.clear(); + rows.put(null, null); + exception.expect(RuntimeException.class); + dataSource.updateByBatch(rows); dataSource.closeDB(); } @@ -354,6 +400,118 @@ public void initDbTest() { assertEquals(TronError.ErrCode.LEVELDB_INIT, thrown.getErrCode()); } + @Test + public void testCheckOrInitEngine() { + String dir = + Args.getInstance().getOutputDirectory() + Args.getInstance().getStorage().getDbDirectory(); + String enginePath = dir + File.separator + "test_engine" + File.separator + "engine.properties"; + FileUtil.createDirIfNotExists(dir + File.separator + "test_engine"); + FileUtil.createFileIfNotExists(enginePath); + PropUtil.writeProperty(enginePath, "ENGINE", "LEVELDB"); + Assert.assertEquals("LEVELDB", PropUtil.readProperty(enginePath, "ENGINE")); + + LevelDbDataSourceImpl dataSource; + dataSource = new LevelDbDataSourceImpl(dir, "test_engine"); + dataSource.initDB(); + dataSource.closeDB(); + + System.gc(); + PropUtil.writeProperty(enginePath, "ENGINE", "ROCKSDB"); + Assert.assertEquals("ROCKSDB", PropUtil.readProperty(enginePath, "ENGINE")); + try { + dataSource = new LevelDbDataSourceImpl(dir, "test_engine"); + dataSource.initDB(); + } catch (Exception e) { + Assert.assertEquals(String.format( + "Cannot open RocksDB database '%s' with LevelDB engine." + + " Set db.engine=ROCKSDB or use LevelDB database. ", "test_engine"), + e.getMessage()); + } + } + + @Test + public void testLevelDbOpenRocksDb() { + String name = "test_openRocksDb"; + String output = Paths + .get(StorageUtils.getOutputDirectoryByDbName(name), CommonParameter + .getInstance().getStorage().getDbDirectory()).toString(); + RocksDbDataSourceImpl rocksDb = new RocksDbDataSourceImpl(output, name); + rocksDb.initDB(); + rocksDb.putData(key1, value1); + rocksDb.closeDB(); + LevelDbDataSourceImpl levelDB = + new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(name), name); + exception.expectMessage(String.format( + "Cannot open RocksDB database '%s' with LevelDB engine." + + " Set db.engine=ROCKSDB or use LevelDB database. ", name)); + levelDB.initDB(); + } + + @Test + public void testNewInstance() { + dataSourceTest.closeDB(); + LevelDbDataSourceImpl newInst = dataSourceTest.newInstance(); + newInst.initDB(); + assertFalse(newInst.flush()); + newInst.closeDB(); + LevelDbDataSourceImpl empty = new LevelDbDataSourceImpl(); + empty.setDBName("empty"); + assertEquals("empty", empty.getDBName()); + String name = "newInst2"; + LevelDbDataSourceImpl newInst2 = new LevelDbDataSourceImpl( + StorageUtils.getOutputDirectoryByDbName(name), + name); + newInst2.initDB(); + newInst2.closeDB(); + } + + @Test + public void testGetNext() { + LevelDbDataSourceImpl dataSource = new LevelDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_getNext_key"); + dataSource.initDB(); + dataSource.resetDb(); + putSomeKeyValue(dataSource); + // case: normal + Map seekKvLimitNext = dataSource.getNext("0000000300".getBytes(), 2); + Map hashMap = Maps.newHashMap(); + hashMap.put(ByteArray.toStr(key3), ByteArray.toStr(value3)); + hashMap.put(ByteArray.toStr(key4), ByteArray.toStr(value4)); + seekKvLimitNext.forEach((key, value) -> { + String keyStr = ByteArray.toStr(key); + Assert.assertTrue("getNext", hashMap.containsKey(keyStr)); + Assert.assertEquals(ByteArray.toStr(value), hashMap.get(keyStr)); + }); + // case: targetKey greater than all existed keys + seekKvLimitNext = dataSource.getNext("0000000700".getBytes(), 2); + Assert.assertEquals(0, seekKvLimitNext.size()); + // case: limit<=0 + seekKvLimitNext = dataSource.getNext("0000000300".getBytes(), 0); + Assert.assertEquals(0, seekKvLimitNext.size()); + dataSource.resetDb(); + dataSource.closeDB(); + } + + @Test + public void testGetlatestValues() { + LevelDbDataSourceImpl dataSource = new LevelDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_getlatestValues_key"); + dataSource.initDB(); + dataSource.resetDb(); + putSomeKeyValue(dataSource); + // case: normal + Set seekKeyLimitNext = dataSource.getlatestValues(2); + Set hashSet = Sets.newHashSet(ByteArray.toStr(value5), ByteArray.toStr(value6)); + seekKeyLimitNext.forEach(value -> { + Assert.assertTrue(hashSet.contains(ByteArray.toStr(value))); + }); + // case: limit<=0 + seekKeyLimitNext = dataSource.getlatestValues(0); + assertEquals(0, seekKeyLimitNext.size()); + dataSource.resetDb(); + dataSource.closeDB(); + } + private void makeExceptionDb(String dbName) { LevelDbDataSourceImpl dataSource = new LevelDbDataSourceImpl( Args.getInstance().getOutputDirectory(), "test_initDb"); diff --git a/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java b/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java index c6fce30e3af..8f42c44e3b9 100644 --- a/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java +++ b/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java @@ -10,6 +10,8 @@ import com.google.common.collect.Sets; import java.io.File; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -24,13 +26,20 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.common.error.TronDBException; +import org.tron.common.parameter.CommonParameter; +import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl; import org.tron.common.utils.ByteArray; import org.tron.common.utils.FileUtil; import org.tron.common.utils.PropUtil; import org.tron.common.utils.PublicMethod; +import org.tron.common.utils.StorageUtils; import org.tron.core.config.args.Args; import org.tron.core.db2.common.WrappedByteArray; import org.tron.core.exception.TronError; @@ -55,6 +64,9 @@ public class RocksDbDataSourceImplTest { private byte[] key5 = "00000005aa".getBytes(); private byte[] key6 = "00000006aa".getBytes(); + @Rule + public final ExpectedException expectedException = ExpectedException.none(); + /** * Release resources. */ @@ -84,8 +96,18 @@ public void testPutGet() { assertNotNull(dataSourceTest.getData(key)); assertEquals(1, dataSourceTest.allKeys().size()); + assertEquals(1, dataSourceTest.getTotal()); + assertEquals(1, dataSourceTest.allValues().size()); assertEquals("50000", ByteArray.toStr(dataSourceTest.getData(key1.getBytes()))); + dataSourceTest.deleteData(key); + assertNull(dataSourceTest.getData(key)); + assertEquals(0, dataSourceTest.getTotal()); + dataSourceTest.iterator().forEachRemaining(entry -> Assert.fail("iterator should be empty")); + dataSourceTest.stat(); dataSourceTest.closeDB(); + dataSourceTest.stat(); // stat again + expectedException.expect(TronDBException.class); + dataSourceTest.deleteData(key); } @Test @@ -124,6 +146,23 @@ public void testupdateByBatchInner() { assertEquals("50000", ByteArray.toStr(dataSource.getData(key1.getBytes()))); assertEquals("10000", ByteArray.toStr(dataSource.getData(key2.getBytes()))); assertEquals(2, dataSource.allKeys().size()); + + rows.clear(); + rows.put(key1.getBytes(), null); + rows.put(key2.getBytes(), null); + dataSource.updateByBatch(rows, WriteOptionsWrapper.getInstance()); + assertEquals(0, dataSource.allKeys().size()); + + rows.clear(); + rows.put(key1.getBytes(), value1.getBytes()); + rows.put(key2.getBytes(), null); + dataSource.updateByBatch(rows); + assertEquals("50000", ByteArray.toStr(dataSource.getData(key1.getBytes()))); + assertEquals(1, dataSource.allKeys().size()); + rows.clear(); + rows.put(null, null); + expectedException.expect(RuntimeException.class); + dataSource.updateByBatch(rows); dataSource.closeDB(); } @@ -276,8 +315,9 @@ public void testCheckOrInitEngine() { try { dataSource.initDB(); } catch (Exception e) { - Assert.assertEquals(String.format("failed to check database: %s, engine do not match", - "test_engine"), + Assert.assertEquals(String.format( + "Cannot open LevelDB database '%s' with RocksDB engine." + + " Set db.engine=LEVELDB or use RocksDB database. ", "test_engine"), e.getMessage()); } Assert.assertNull(dataSource.getDatabase()); @@ -396,6 +436,103 @@ public void initDbTest() { assertEquals(TronError.ErrCode.ROCKSDB_INIT, thrown.getErrCode()); } + @Test + public void testRocksDbOpenLevelDb() { + String name = "test_openLevelDb"; + String output = Paths + .get(StorageUtils.getOutputDirectoryByDbName(name), CommonParameter + .getInstance().getStorage().getDbDirectory()).toString(); + LevelDbDataSourceImpl levelDb = new LevelDbDataSourceImpl( + StorageUtils.getOutputDirectoryByDbName(name), name); + levelDb.initDB(); + levelDb.putData(key1, value1); + levelDb.closeDB(); + RocksDbDataSourceImpl rocksDb = new RocksDbDataSourceImpl(output, name); + expectedException.expectMessage( + String.format( + "Cannot open LevelDB database '%s' with RocksDB engine." + + " Set db.engine=LEVELDB or use RocksDB database. ", name)); + rocksDb.initDB(); + } + + @Test + public void testRocksDbOpenLevelDb2() { + String name = "test_openLevelDb2"; + String output = Paths + .get(StorageUtils.getOutputDirectoryByDbName(name), CommonParameter + .getInstance().getStorage().getDbDirectory()).toString(); + LevelDbDataSourceImpl levelDb = new LevelDbDataSourceImpl( + StorageUtils.getOutputDirectoryByDbName(name), name); + levelDb.initDB(); + levelDb.putData(key1, value1); + levelDb.closeDB(); + // delete engine.properties file to simulate the case that db.engine is not set. + File engineFile = Paths + .get(output, name, "engine.properties").toFile(); + if (engineFile.exists()) { + engineFile.delete(); + } + Assert.assertFalse(engineFile.exists()); + RocksDbDataSourceImpl rocksDb = new RocksDbDataSourceImpl(output, name); + expectedException.expectMessage( + String.format( + "Cannot open LevelDB database '%s' with RocksDB engine." + + " Set db.engine=LEVELDB or use RocksDB database. ", name)); + rocksDb.initDB(); + } + + @Test + public void testNewInstance() { + dataSourceTest.closeDB(); + RocksDbDataSourceImpl newInst = dataSourceTest.newInstance(); + newInst.initDB(); + assertFalse(newInst.flush()); + newInst.closeDB(); + RocksDbDataSourceImpl empty = new RocksDbDataSourceImpl(); + empty.setDBName("empty"); + assertEquals("empty", empty.getDBName()); + String output = Paths + .get(StorageUtils.getOutputDirectoryByDbName("newInst2"), CommonParameter + .getInstance().getStorage().getDbDirectory()).toString(); + RocksDbDataSourceImpl newInst2 = new RocksDbDataSourceImpl(output, "newInst2"); + newInst2.initDB(); + newInst2.closeDB(); + } + + @Test + public void backupAndDelete() throws RocksDBException { + RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "backupAndDelete"); + dataSource.initDB(); + putSomeKeyValue(dataSource); + Path dir = Paths.get(Args.getInstance().getOutputDirectory(), "backup"); + String path = dir + File.separator; + FileUtil.createDirIfNotExists(path); + dataSource.backup(path); + File backDB = Paths.get(dir.toString(),dataSource.getDBName()).toFile(); + Assert.assertTrue(backDB.exists()); + dataSource.deleteDbBakPath(path); + Assert.assertFalse(backDB.exists()); + dataSource.closeDB(); + } + + @Test + public void testGetTotal() { + RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_getTotal_key"); + dataSource.initDB(); + dataSource.resetDb(); + + Map dataMapset = Maps.newHashMap(); + dataMapset.put(key1, value1); + dataMapset.put(key2, value2); + dataMapset.put(key3, value3); + dataMapset.forEach(dataSource::putData); + Assert.assertEquals(dataMapset.size(), dataSource.getTotal()); + dataSource.resetDb(); + dataSource.closeDB(); + } + private void makeExceptionDb(String dbName) { RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( Args.getInstance().getOutputDirectory(), "test_initDb"); diff --git a/framework/src/test/java/org/tron/common/utils/HashCodeTest.java b/framework/src/test/java/org/tron/common/utils/HashCodeTest.java new file mode 100644 index 00000000000..36f9435c1aa --- /dev/null +++ b/framework/src/test/java/org/tron/common/utils/HashCodeTest.java @@ -0,0 +1,23 @@ +package org.tron.common.utils; + +import java.util.Objects; +import org.junit.Assert; +import org.junit.Test; +import org.tron.core.capsule.AccountCapsule; +import org.tron.core.vm.repository.Type; +import org.tron.core.vm.repository.Value; +import org.tron.protos.Protocol; + +public class HashCodeTest { + + @Test + public void test() { + Type type = new Type(); + type.setType(Type.NORMAL); + Assert.assertEquals(Integer.valueOf(Type.NORMAL).hashCode(), type.hashCode()); + Protocol.Account account = Protocol.Account.newBuilder().setBalance(100).build(); + Value value = Value.create(new AccountCapsule(account.toByteArray())); + Assert.assertEquals(Integer.valueOf( + type.hashCode() + Objects.hashCode(account)).hashCode(), value.hashCode()); + } +} diff --git a/framework/src/test/java/org/tron/core/capsule/utils/ExchangeProcessorTest.java b/framework/src/test/java/org/tron/core/capsule/utils/ExchangeProcessorTest.java index 717c62b01a8..1f0be4b1f7c 100644 --- a/framework/src/test/java/org/tron/core/capsule/utils/ExchangeProcessorTest.java +++ b/framework/src/test/java/org/tron/core/capsule/utils/ExchangeProcessorTest.java @@ -138,12 +138,63 @@ public void testWithdraw() { @Test public void testStrictMath() { long supply = 1_000_000_000_000_000_000L; - ExchangeProcessor processor = new ExchangeProcessor(supply, false); - long anotherTokenQuant = processor.exchange(4732214, 2202692725330L, 29218); - processor = new ExchangeProcessor(supply, true); - long result = processor.exchange(4732214, 2202692725330L, 29218); - Assert.assertNotEquals(anotherTokenQuant, result); + long[][] testData = { + {4732214L, 2202692725330L, 29218L}, + {5618633L, 556559904655L, 1L}, + {9299554L, 1120271441185L, 7000L}, + {62433133L, 12013267997895L, 100000L}, + {64212664L, 725836766395L, 50000L}, + {64126212L, 2895100109660L, 5000L}, + {56459055L, 3288380567368L, 165000L}, + {21084707L, 1589204008960L, 50000L}, + {24120521L, 1243764649177L, 20000L}, + {836877L, 212532333234L, 5293L}, + {55879741L, 13424854054078L, 250000L}, + {66388882L, 11300012790454L, 300000L}, + {94470955L, 7941038150919L, 2000L}, + {13613746L, 5012660712983L, 122L}, + {71852829L, 5262251868618L, 396L}, + {3857658L, 446109245044L, 20637L}, + {35491863L, 3887393269796L, 100L}, + {295632118L, 1265298439004L, 500000L}, + {49320113L, 1692106302503L, 123267L}, + {10966984L, 6222910652894L, 2018L}, + {41634280L, 2004508994767L, 865L}, + {10087714L, 6765558834714L, 1009L}, + {42270078L, 210360843525L, 200000L}, + {571091915L, 655011397250L, 2032520L}, + {51026781L, 1635726339365L, 37L}, + {61594L, 312318864132L, 500L}, + {11616684L, 5875978057357L, 20L}, + {60584529L, 1377717821301L, 78132L}, + {29818073L, 3033545989651L, 182L}, + {3855280L, 834647482043L, 16L}, + {58310711L, 1431562205655L, 200000L}, + {60226263L, 1386036785882L, 178226L}, + {3537634L, 965771433992L, 225L}, + {3760534L, 908700758784L, 328L}, + {80913L, 301864126445L, 4L}, + {3789271L, 901842209723L, 1L}, + {4051904L, 843419481286L, 1005L}, + {89141L, 282107742510L, 100L}, + {90170L, 282854635378L, 26L}, + {4229852L, 787503315944L, 137L}, + {4259884L, 781975090197L, 295L}, + {3627657L, 918682223700L, 34L}, + {813519L, 457546358759L, 173L}, + {89626L, 327856173057L, 27L}, + {97368L, 306386489550L, 50L}, + {93712L, 305866015731L, 4L}, + {3281260L, 723656594544L, 40L}, + {3442652L, 689908773685L, 18L}, + }; + + for (long[] data : testData) { + ExchangeProcessor processor = new ExchangeProcessor(supply, false); + long anotherTokenQuant = processor.exchange(data[0], data[1], data[2]); + processor = new ExchangeProcessor(supply, true); + long result = processor.exchange(data[0], data[1], data[2]); + Assert.assertNotEquals(anotherTokenQuant, result); + } } - - } diff --git a/framework/src/test/java/org/tron/core/db2/CheckpointV2Test.java b/framework/src/test/java/org/tron/core/db2/CheckpointV2Test.java index dff2d376fd5..fb7f1987e9f 100644 --- a/framework/src/test/java/org/tron/core/db2/CheckpointV2Test.java +++ b/framework/src/test/java/org/tron/core/db2/CheckpointV2Test.java @@ -4,7 +4,7 @@ import com.google.common.primitives.Bytes; import com.google.common.primitives.Longs; import com.google.protobuf.ByteString; -import java.io.File; +import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -13,11 +13,12 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; -import org.tron.common.utils.FileUtil; import org.tron.common.utils.Sha256Hash; import org.tron.core.Constant; import org.tron.core.capsule.BlockCapsule; @@ -34,10 +35,12 @@ public class CheckpointV2Test { private TronApplicationContext context; private Application appT; private TestRevokingTronStore tronDatabase; + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void init() { - Args.setParam(new String[]{"-d", "output_SnapshotManager_test"}, + public void init() throws IOException { + Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); Args.getInstance().getStorage().setCheckpointVersion(2); Args.getInstance().getStorage().setCheckpointSync(true); @@ -54,9 +57,6 @@ public void removeDb() { Args.clearParam(); context.destroy(); tronDatabase.close(); - FileUtil.deleteDir(new File("output_SnapshotManager_test")); - revokingDatabase.getCheckTmpStore().close(); - tronDatabase.close(); } @Test diff --git a/framework/src/test/java/org/tron/core/db2/RevokingDbWithCacheNewValueTest.java b/framework/src/test/java/org/tron/core/db2/RevokingDbWithCacheNewValueTest.java index 2290df86978..f371f2348a7 100644 --- a/framework/src/test/java/org/tron/core/db2/RevokingDbWithCacheNewValueTest.java +++ b/framework/src/test/java/org/tron/core/db2/RevokingDbWithCacheNewValueTest.java @@ -1,22 +1,22 @@ package org.tron.core.db2; -import java.io.File; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.RandomStringUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; import org.tron.common.utils.ByteArray; -import org.tron.common.utils.FileUtil; import org.tron.common.utils.SessionOptional; import org.tron.core.Constant; import org.tron.core.capsule.utils.MarketUtils; @@ -34,12 +34,14 @@ public class RevokingDbWithCacheNewValueTest { private TronApplicationContext context; private Application appT; private TestRevokingTronStore tronDatabase; + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); private String databasePath = ""; @Before - public void init() { - databasePath = "output_revokingStore_test_" + RandomStringUtils.randomAlphanumeric(10); + public void init() throws IOException { + databasePath = temporaryFolder.newFolder().toString(); Args.setParam(new String[]{"-d", databasePath}, Constant.TEST_CONF); context = new TronApplicationContext(DefaultConfig.class); @@ -51,7 +53,6 @@ public void removeDb() { Args.clearParam(); context.destroy(); tronDatabase.close(); - FileUtil.deleteDir(new File(databasePath)); } @Test diff --git a/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java b/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java index aab6f656b1f..649056aa151 100644 --- a/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java +++ b/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java @@ -3,16 +3,16 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import java.io.File; +import java.io.IOException; import java.lang.reflect.Constructor; import org.junit.After; -import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; -import org.tron.common.utils.FileUtil; import org.tron.core.Constant; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; @@ -26,10 +26,12 @@ public class SnapshotImplTest { private TronApplicationContext context; private Application appT; private SnapshotManager revokingDatabase; + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void init() { - Args.setParam(new String[]{"-d", "output_revokingStore_test"}, Constant.TEST_CONF); + public void init() throws IOException { + Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); context = new TronApplicationContext(DefaultConfig.class); appT = ApplicationFactory.create(context); @@ -44,10 +46,7 @@ public void init() { public void removeDb() { Args.clearParam(); context.destroy(); - FileUtil.deleteDir(new File("output_revokingStore_test")); - tronDatabase.close(); - revokingDatabase.shutdown(); } /** diff --git a/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java b/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java index 134dc99e51c..d6fd319e6a0 100644 --- a/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java +++ b/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java @@ -6,7 +6,7 @@ import com.google.common.collect.Maps; import com.google.common.primitives.Longs; import com.google.protobuf.ByteString; -import java.io.File; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -15,11 +15,12 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; -import org.tron.common.utils.FileUtil; import org.tron.common.utils.Sha256Hash; import org.tron.core.Constant; import org.tron.core.capsule.BlockCapsule; @@ -40,11 +41,13 @@ public class SnapshotManagerTest { private TronApplicationContext context; private Application appT; private TestRevokingTronStore tronDatabase; + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void init() { - Args.setParam(new String[]{"-d", "output_SnapshotManager_test"}, + public void init() throws IOException { + Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); context = new TronApplicationContext(DefaultConfig.class); appT = ApplicationFactory.create(context); @@ -59,9 +62,6 @@ public void removeDb() { Args.clearParam(); context.destroy(); tronDatabase.close(); - FileUtil.deleteDir(new File("output_SnapshotManager_test")); - revokingDatabase.getCheckTmpStore().close(); - tronDatabase.close(); } @Test diff --git a/framework/src/test/java/org/tron/core/db2/SnapshotRootTest.java b/framework/src/test/java/org/tron/core/db2/SnapshotRootTest.java index 70b4d9eff30..635cc018cc2 100644 --- a/framework/src/test/java/org/tron/core/db2/SnapshotRootTest.java +++ b/framework/src/test/java/org/tron/core/db2/SnapshotRootTest.java @@ -1,10 +1,13 @@ package org.tron.core.db2; import com.google.common.collect.Sets; -import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import lombok.AllArgsConstructor; @@ -13,7 +16,9 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.springframework.util.CollectionUtils; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; @@ -45,11 +50,13 @@ public class SnapshotRootTest { "exchange","market_order","account-trace","contract-state","trans")); private Set allDBNames; private Set allRevokingDBNames; + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void init() { - Args.setParam(new String[]{"-d", "output_revokingStore_test"}, Constant.TEST_CONF); + public void init() throws IOException { + Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); context = new TronApplicationContext(DefaultConfig.class); appT = ApplicationFactory.create(context); } @@ -58,7 +65,6 @@ public void init() { public void removeDb() { Args.clearParam(); context.destroy(); - FileUtil.deleteDir(new File("output_revokingStore_test")); } @Test @@ -133,7 +139,9 @@ public void testSecondCacheCheck() throws ItemNotFoundException { revokingDatabase = context.getBean(SnapshotManager.class); allRevokingDBNames = parseRevokingDBNames(context); - allDBNames = Arrays.stream(new File("output_revokingStore_test/database").list()) + Path path = Paths.get(Args.getInstance().getOutputDirectory(), + Args.getInstance().getStorage().getDbDirectory()); + allDBNames = Arrays.stream(Objects.requireNonNull(path.toFile().list())) .collect(Collectors.toSet()); if (CollectionUtils.isEmpty(allDBNames)) { throw new ItemNotFoundException("No DBs found"); @@ -152,10 +160,13 @@ public void testSecondCacheCheckAddDb() revokingDatabase = context.getBean(SnapshotManager.class); allRevokingDBNames = parseRevokingDBNames(context); allRevokingDBNames.add("secondCheckTestDB"); - FileUtil.createDirIfNotExists("output_revokingStore_test/database/secondCheckTestDB"); - allDBNames = Arrays.stream(new File("output_revokingStore_test/database").list()) + Path path = Paths.get(Args.getInstance().getOutputDirectory(), + Args.getInstance().getStorage().getDbDirectory()); + Path secondCheckTestDB = Paths.get(path.toString(), "secondCheckTestDB"); + FileUtil.createDirIfNotExists(secondCheckTestDB.toString()); + allDBNames = Arrays.stream(Objects.requireNonNull(path.toFile().list())) .collect(Collectors.toSet()); - FileUtil.deleteDir(new File("output_revokingStore_test/database/secondCheckTestDB")); + FileUtil.deleteDir(secondCheckTestDB.toFile()); if (CollectionUtils.isEmpty(allDBNames)) { throw new ItemNotFoundException("No DBs found"); } diff --git a/framework/src/test/java/org/tron/core/jsonrpc/ApiUtilTest.java b/framework/src/test/java/org/tron/core/jsonrpc/ApiUtilTest.java index 570e7ed3498..2c9afbac99b 100644 --- a/framework/src/test/java/org/tron/core/jsonrpc/ApiUtilTest.java +++ b/framework/src/test/java/org/tron/core/jsonrpc/ApiUtilTest.java @@ -4,10 +4,13 @@ import static org.tron.keystore.Wallet.generateRandomBytes; import com.google.protobuf.ByteString; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; import org.tron.common.utils.ByteArray; import org.tron.core.capsule.BlockCapsule; +import org.tron.core.config.args.Args; import org.tron.core.services.jsonrpc.JsonRpcApiUtil; import org.tron.protos.Protocol.Block; import org.tron.protos.Protocol.BlockHeader; @@ -16,6 +19,17 @@ public class ApiUtilTest { + + @BeforeClass + public static void init() { + Args.setParam(new String[]{}, "config-localtest.conf"); + } + + @AfterClass + public static void clear() { + Args.clearParam(); + } + @Test public void testGetBlockID() { byte[] mockedHash = generateRandomBytes(128); diff --git a/framework/src/test/java/org/tron/core/jsonrpc/JsonrpcServiceTest.java b/framework/src/test/java/org/tron/core/jsonrpc/JsonrpcServiceTest.java index 0f2214c5c9c..df0e13d6b6c 100644 --- a/framework/src/test/java/org/tron/core/jsonrpc/JsonrpcServiceTest.java +++ b/framework/src/test/java/org/tron/core/jsonrpc/JsonrpcServiceTest.java @@ -434,7 +434,8 @@ public void testGetByJsonBlockId() { getByJsonBlockId("0xxabc", wallet); Assert.fail("Expected to be thrown"); } catch (Exception e) { - Assert.assertEquals("For input string: \"xabc\"", e.getMessage()); + // https://bugs.openjdk.org/browse/JDK-8176425, from JDK 12, the exception message is changed + Assert.assertTrue(e.getMessage().startsWith("For input string: \"xabc\"")); } } @@ -968,4 +969,15 @@ public void testNewFilterFinalizedBlock() { Assert.assertEquals("invalid block range params", e.getMessage()); } } + + @Test + public void testWeb3ClientVersion() { + try { + String[] versions = tronJsonRpc.web3ClientVersion().split("/"); + String javaVersion = versions[versions.length - 1]; + Assert.assertTrue("Java1.8".equals(javaVersion) || "Java17".equals(javaVersion)); + } catch (Exception e) { + Assert.fail(); + } + } } diff --git a/framework/src/test/java/org/tron/core/net/BaseNet.java b/framework/src/test/java/org/tron/core/net/BaseNet.java index 65771bae952..fc04e210acd 100644 --- a/framework/src/test/java/org/tron/core/net/BaseNet.java +++ b/framework/src/test/java/org/tron/core/net/BaseNet.java @@ -16,6 +16,7 @@ import java.io.File; import java.io.IOException; import java.util.Collection; +import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -123,10 +124,12 @@ public static void init() throws Exception { @AfterClass public static void destroy() { - Collection peerConnections = ReflectUtils - .invokeMethod(tronNetDelegate, "getActivePeer"); - for (PeerConnection peer : peerConnections) { - peer.getChannel().close(); + if (Objects.nonNull(tronNetDelegate)) { + Collection peerConnections = ReflectUtils + .invokeMethod(tronNetDelegate, "getActivePeer"); + for (PeerConnection peer : peerConnections) { + peer.getChannel().close(); + } } Args.clearParam(); context.destroy(); diff --git a/framework/src/test/java/org/tron/core/services/RpcApiServicesTest.java b/framework/src/test/java/org/tron/core/services/RpcApiServicesTest.java index 3ae090d3caf..3af0b8fb9b2 100644 --- a/framework/src/test/java/org/tron/core/services/RpcApiServicesTest.java +++ b/framework/src/test/java/org/tron/core/services/RpcApiServicesTest.java @@ -60,8 +60,6 @@ import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; import org.tron.core.db.Manager; -import org.tron.core.services.interfaceOnPBFT.RpcApiServiceOnPBFT; -import org.tron.core.services.interfaceOnSolidity.RpcApiServiceOnSolidity; import org.tron.protos.Protocol; import org.tron.protos.Protocol.Account; import org.tron.protos.Protocol.Block; @@ -144,11 +142,11 @@ public static void init() throws IOException { getInstance().setMetricsPrometheusPort(PublicMethod.chooseRandomPort()); getInstance().setMetricsPrometheusEnable(true); getInstance().setP2pDisable(true); - String fullNode = String.format("%s:%d", getInstance().getNodeLanIp(), + String fullNode = String.format("%s:%d", Constant.LOCAL_HOST, getInstance().getRpcPort()); - String solidityNode = String.format("%s:%d", getInstance().getNodeLanIp(), + String solidityNode = String.format("%s:%d", Constant.LOCAL_HOST, getInstance().getRpcOnSolidityPort()); - String pBFTNode = String.format("%s:%d", getInstance().getNodeLanIp(), + String pBFTNode = String.format("%s:%d", Constant.LOCAL_HOST, getInstance().getRpcOnPBFTPort()); ManagedChannel channelFull = ManagedChannelBuilder.forTarget(fullNode) diff --git a/framework/src/test/java/org/tron/core/services/filter/LiteFnQueryGrpcInterceptorTest.java b/framework/src/test/java/org/tron/core/services/filter/LiteFnQueryGrpcInterceptorTest.java index 84869ea0750..d98e2c9267e 100644 --- a/framework/src/test/java/org/tron/core/services/filter/LiteFnQueryGrpcInterceptorTest.java +++ b/framework/src/test/java/org/tron/core/services/filter/LiteFnQueryGrpcInterceptorTest.java @@ -25,9 +25,6 @@ import org.tron.core.Constant; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; -import org.tron.core.services.RpcApiService; -import org.tron.core.services.interfaceOnPBFT.RpcApiServiceOnPBFT; -import org.tron.core.services.interfaceOnSolidity.RpcApiServiceOnSolidity; @Slf4j public class LiteFnQueryGrpcInterceptorTest { @@ -62,11 +59,11 @@ public static void init() throws IOException { Args.getInstance().setRpcPBFTEnable(true); Args.getInstance().setRpcOnPBFTPort(PublicMethod.chooseRandomPort()); Args.getInstance().setP2pDisable(true); - String fullnode = String.format("%s:%d", Args.getInstance().getNodeLanIp(), + String fullnode = String.format("%s:%d", Constant.LOCAL_HOST, Args.getInstance().getRpcPort()); - String solidityNode = String.format("%s:%d", Args.getInstance().getNodeLanIp(), + String solidityNode = String.format("%s:%d", Constant.LOCAL_HOST, Args.getInstance().getRpcOnSolidityPort()); - String pBFTNode = String.format("%s:%d", Args.getInstance().getNodeLanIp(), + String pBFTNode = String.format("%s:%d", Constant.LOCAL_HOST, Args.getInstance().getRpcOnPBFTPort()); channelFull = ManagedChannelBuilder.forTarget(fullnode) .usePlaintext() diff --git a/framework/src/test/java/org/tron/core/services/filter/RpcApiAccessInterceptorTest.java b/framework/src/test/java/org/tron/core/services/filter/RpcApiAccessInterceptorTest.java index 900ca304e7d..ce7efabef0c 100644 --- a/framework/src/test/java/org/tron/core/services/filter/RpcApiAccessInterceptorTest.java +++ b/framework/src/test/java/org/tron/core/services/filter/RpcApiAccessInterceptorTest.java @@ -36,8 +36,6 @@ import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; import org.tron.core.services.RpcApiService; -import org.tron.core.services.interfaceOnPBFT.RpcApiServiceOnPBFT; -import org.tron.core.services.interfaceOnSolidity.RpcApiServiceOnSolidity; import org.tron.protos.Protocol.Transaction; @Slf4j @@ -63,11 +61,11 @@ public static void init() throws IOException { Args.getInstance().setRpcPBFTEnable(true); Args.getInstance().setRpcOnPBFTPort(PublicMethod.chooseRandomPort()); Args.getInstance().setP2pDisable(true); - String fullNode = String.format("%s:%d", Args.getInstance().getNodeLanIp(), + String fullNode = String.format("%s:%d", Constant.LOCAL_HOST, Args.getInstance().getRpcPort()); - String solidityNode = String.format("%s:%d", Args.getInstance().getNodeLanIp(), + String solidityNode = String.format("%s:%d", Constant.LOCAL_HOST, Args.getInstance().getRpcOnSolidityPort()); - String pBFTNode = String.format("%s:%d", Args.getInstance().getNodeLanIp(), + String pBFTNode = String.format("%s:%d", Constant.LOCAL_HOST, Args.getInstance().getRpcOnPBFTPort()); ManagedChannel channelFull = ManagedChannelBuilder.forTarget(fullNode) diff --git a/framework/src/test/java/org/tron/keystroe/CredentialsTest.java b/framework/src/test/java/org/tron/keystroe/CredentialsTest.java index ce992c3443f..2642129e00a 100644 --- a/framework/src/test/java/org/tron/keystroe/CredentialsTest.java +++ b/framework/src/test/java/org/tron/keystroe/CredentialsTest.java @@ -1,6 +1,5 @@ package org.tron.keystroe; -import lombok.var; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -11,24 +10,24 @@ public class CredentialsTest { @Test public void test_equality() { - var aObject = new Object(); - var si = Mockito.mock(SignInterface.class); - var si2 = Mockito.mock(SignInterface.class); - var si3 = Mockito.mock(SignInterface.class); - var address = "TQhZ7W1RudxFdzJMw6FvMnujPxrS6sFfmj".getBytes(); - var address2 = "TNCmcTdyrYKMtmE1KU2itzeCX76jGm5Not".getBytes(); + Object aObject = new Object(); + SignInterface si = Mockito.mock(SignInterface.class); + SignInterface si2 = Mockito.mock(SignInterface.class); + SignInterface si3 = Mockito.mock(SignInterface.class); + byte[] address = "TQhZ7W1RudxFdzJMw6FvMnujPxrS6sFfmj".getBytes(); + byte[] address2 = "TNCmcTdyrYKMtmE1KU2itzeCX76jGm5Not".getBytes(); Mockito.when(si.getAddress()).thenReturn(address); Mockito.when(si2.getAddress()).thenReturn(address); Mockito.when(si3.getAddress()).thenReturn(address2); - var aCredential = Credentials.create(si); + Credentials aCredential = Credentials.create(si); Assert.assertFalse(aObject.equals(aCredential)); Assert.assertFalse(aCredential.equals(aObject)); Assert.assertFalse(aCredential.equals(null)); - var anotherCredential = Credentials.create(si); - Assert.assertTrue(aCredential.equals(anotherCredential)); - var aCredential2 = Credentials.create(si2); + Credentials anotherCredential = Credentials.create(si); Assert.assertTrue(aCredential.equals(anotherCredential)); - var aCredential3 = Credentials.create(si3); + Credentials aCredential2 = Credentials.create(si2); + Assert.assertTrue(aCredential.equals(anotherCredential)); + Credentials aCredential3 = Credentials.create(si3); Assert.assertFalse(aCredential.equals(aCredential3)); } } diff --git a/gradle/jdk17/java-tron.vmoptions b/gradle/jdk17/java-tron.vmoptions new file mode 100644 index 00000000000..91accd05016 --- /dev/null +++ b/gradle/jdk17/java-tron.vmoptions @@ -0,0 +1,8 @@ +-XX:+UseZGC +-Xlog:gc*:file=gc.log:time,uptime,level,tags:filecount=50,filesize=100M +-XX:ReservedCodeCacheSize=256m +-XX:+UseCodeCacheFlushing +-XX:MetaspaceSize=256m +-XX:MaxMetaspaceSize=512m +-XX:MaxDirectMemorySize=1g +-XX:+HeapDumpOnOutOfMemoryError \ No newline at end of file diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 5b80b8d706e..53d4958f1e5 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -521,9 +521,16 @@ + + + + + + + @@ -894,9 +901,16 @@ + + + + + + + @@ -1074,6 +1088,14 @@ + + + + + + + + @@ -1106,6 +1128,14 @@ + + + + + + + + @@ -1507,28 +1537,28 @@ - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + @@ -2058,12 +2088,12 @@ - - - + + + - - + + @@ -2090,6 +2120,14 @@ + + + + + + + + diff --git a/platform/build.gradle b/platform/build.gradle new file mode 100644 index 00000000000..a94aad3cf17 --- /dev/null +++ b/platform/build.gradle @@ -0,0 +1,17 @@ +description = "platform – a distributed consensus arithmetic for blockchain." + +sourceSets { + main { + java.srcDirs = rootProject.archInfo.sourceSets.main.java.srcDirs + } + test { + java.srcDirs = rootProject.archInfo.sourceSets.test.java.srcDirs + } +} + +dependencies { + api group: 'org.fusesource.leveldbjni', name: 'leveldbjni-all', version: '1.8' + api group: 'org.rocksdb', name: 'rocksdbjni', version: "${rootProject.archInfo.requires.RocksdbVersion}" + api group: 'commons-io', name: 'commons-io', version: '2.18.0' + api 'io.github.tronprotocol:zksnark-java-sdk:1.0.0' exclude(group: 'commons-io', module: 'commons-io') +} diff --git a/platform/src/main/java/arm/org/tron/common/math/MathWrapper.java b/platform/src/main/java/arm/org/tron/common/math/MathWrapper.java new file mode 100644 index 00000000000..12395dffcea --- /dev/null +++ b/platform/src/main/java/arm/org/tron/common/math/MathWrapper.java @@ -0,0 +1,254 @@ +package org.tron.common.math; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * This class is deprecated and should not be used in new code, + * for cross-platform consistency, please use {@link StrictMathWrapper} instead, + * especially for floating-point calculations. + */ +@Deprecated +public class MathWrapper { + + private static final Map powData = Collections.synchronizedMap(new HashMap<>()); + private static final String EXPONENT = "3f40624dd2f1a9fc"; // 1/2000 = 0.0005 + + public static double pow(double a, double b) { + double strictResult = StrictMath.pow(a, b); + return powData.getOrDefault(new PowData(a, b), strictResult); + } + + /** + * This static block is used to initialize the data map. + */ + static { + // init main-net pow data start + addPowData("3ff0192278704be3", EXPONENT, "3ff000033518c576"); // 4137160(block) + addPowData("3ff000002fc6a33f", EXPONENT, "3ff0000000061d86"); // 4065476 + addPowData("3ff00314b1e73ecf", EXPONENT, "3ff0000064ea3ef8"); // 4071538 + addPowData("3ff0068cd52978ae", EXPONENT, "3ff00000d676966c"); // 4109544 + addPowData("3ff0032fda05447d", EXPONENT, "3ff0000068636fe0"); // 4123826 + addPowData("3ff00051c09cc796", EXPONENT, "3ff000000a76c20e"); // 4166806 + addPowData("3ff00bef8115b65d", EXPONENT, "3ff0000186893de0"); // 4225778 + addPowData("3ff009b0b2616930", EXPONENT, "3ff000013d27849e"); // 4251796 + addPowData("3ff00364ba163146", EXPONENT, "3ff000006f26a9dc"); // 4257157 + addPowData("3ff019be4095d6ae", EXPONENT, "3ff0000348e9f02a"); // 4260583 + addPowData("3ff0123e52985644", EXPONENT, "3ff0000254797fd0"); // 4367125 + addPowData("3ff0126d052860e2", EXPONENT, "3ff000025a6cde26"); // 4402197 + addPowData("3ff0001632cccf1b", EXPONENT, "3ff0000002d76406"); // 4405788 + addPowData("3ff0000965922b01", EXPONENT, "3ff000000133e966"); // 4490332 + addPowData("3ff00005c7692d61", EXPONENT, "3ff0000000bd5d34"); // 4499056 + addPowData("3ff015cba20ec276", EXPONENT, "3ff00002c84cef0e"); // 4518035 + addPowData("3ff00002f453d343", EXPONENT, "3ff000000060cf4e"); // 4533215 + addPowData("3ff006ea73f88946", EXPONENT, "3ff00000e26d4ea2"); // 4647814 + addPowData("3ff00a3632db72be", EXPONENT, "3ff000014e3382a6"); // 4766695 + addPowData("3ff000c0e8df0274", EXPONENT, "3ff0000018b0aeb2"); // 4771494 + addPowData("3ff00015c8f06afe", EXPONENT, "3ff0000002c9d73e"); // 4793587 + addPowData("3ff00068def18101", EXPONENT, "3ff000000d6c3cac"); // 4801947 + addPowData("3ff01349f3ac164b", EXPONENT, "3ff000027693328a"); // 4916843 + addPowData("3ff00e86a7859088", EXPONENT, "3ff00001db256a52"); // 4924111 + addPowData("3ff00000c2a51ab7", EXPONENT, "3ff000000018ea20"); // 5098864 + addPowData("3ff020fb74e9f170", EXPONENT, "3ff00004346fbfa2"); // 5133963 + addPowData("3ff00001ce277ce7", EXPONENT, "3ff00000003b27dc"); // 5139389 + addPowData("3ff005468a327822", EXPONENT, "3ff00000acc20750"); // 5151258 + addPowData("3ff00006666f30ff", EXPONENT, "3ff0000000d1b80e"); // 5185021 + addPowData("3ff000045a0b2035", EXPONENT, "3ff00000008e98e6"); // 5295829 + addPowData("3ff00e00380e10d7", EXPONENT, "3ff00001c9ff83c8"); // 5380897 + addPowData("3ff00c15de2b0d5e", EXPONENT, "3ff000018b6eaab6"); // 5400886 + addPowData("3ff00042afe6956a", EXPONENT, "3ff0000008892244"); // 5864127 + addPowData("3ff0005b7357c2d4", EXPONENT, "3ff000000bb48572"); // 6167339 + addPowData("3ff00033d5ab51c8", EXPONENT, "3ff0000006a279c8"); // 6240974 + addPowData("3ff0000046d74585", EXPONENT, "3ff0000000091150"); // 6279093 + addPowData("3ff0010403f34767", EXPONENT, "3ff0000021472146"); // 6428736 + addPowData("3ff00496fe59bc98", EXPONENT, "3ff000009650a4ca"); // 6432355,6493373 + addPowData("3ff0012e43815868", EXPONENT, "3ff0000026af266e"); // 6555029 + addPowData("3ff00021f6080e3c", EXPONENT, "3ff000000458d16a"); // 7092933 + addPowData("3ff000489c0f28bd", EXPONENT, "3ff00000094b3072"); // 7112412 + addPowData("3ff00009d3df2e9c", EXPONENT, "3ff00000014207b4"); // 7675535 + addPowData("3ff000def05fa9c8", EXPONENT, "3ff000001c887cdc"); // 7860324 + addPowData("3ff0013bca543227", EXPONENT, "3ff00000286a42d2"); // 8292427 + addPowData("3ff0021a2f14a0ee", EXPONENT, "3ff0000044deb040"); // 8517311 + addPowData("3ff0002cc166be3c", EXPONENT, "3ff0000005ba841e"); // 8763101 + addPowData("3ff0000cc84e613f", EXPONENT, "3ff0000001a2da46"); // 9269124 + addPowData("3ff000057b83c83f", EXPONENT, "3ff0000000b3a640"); // 9631452 + // init main-net pow data end + // add pow data + } + + private static void addPowData(String a, String b, String ret) { + powData.put(new PowData(hexToDouble(a), hexToDouble(b)), hexToDouble(ret)); + } + + private static double hexToDouble(String input) { + // Convert the hex string to a long + long hexAsLong = Long.parseLong(input, 16); + // and then convert the long to a double + return Double.longBitsToDouble(hexAsLong); + } + + private static class PowData { + final double a; + final double b; + + public PowData(double a, double b) { + this.a = a; + this.b = b; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PowData powData = (PowData) o; + return Double.compare(powData.a, a) == 0 && Double.compare(powData.b, b) == 0; + } + + @Override + public int hashCode() { + return Objects.hash(a, b); + } + } + + /** + * *** methods are same as {@link java.lang.Math} methods, guaranteed by the call start *** + */ + + /** + * finally calls {@link java.lang.Math#addExact(long, long)} + */ + + public static long addExact(long x, long y) { + return StrictMath.addExact(x, y); + } + + /** + * finally calls {@link java.lang.Math#addExact(int, int)} + */ + + public static int addExact(int x, int y) { + return StrictMath.addExact(x, y); + } + + /** + * finally calls {@link java.lang.Math#subtractExact(long, long)} + */ + + public static long subtractExact(long x, long y) { + return StrictMath.subtractExact(x, y); + } + + /** + * finally calls {@link java.lang.Math#floorMod(long, long)} + */ + public static long multiplyExact(long x, long y) { + return StrictMath.multiplyExact(x, y); + } + + public static long multiplyExact(long x, int y) { + return multiplyExact(x, (long) y); + } + + public static int multiplyExact(int x, int y) { + return StrictMath.multiplyExact(x, y); + } + + /** + * finally calls {@link java.lang.Math#floorDiv(long, long)} + */ + public static long floorDiv(long x, long y) { + return StrictMath.floorDiv(x, y); + } + + public static long floorDiv(long x, int y) { + return floorDiv(x, (long) y); + } + + /** + * finally calls {@link java.lang.Math#min(int, int)} + */ + public static int min(int a, int b) { + return StrictMath.min(a, b); + } + + /** + * finally calls {@link java.lang.Math#min(long, long)} + */ + public static long min(long a, long b) { + return StrictMath.min(a, b); + } + + /** + * finally calls {@link java.lang.Math#max(int, int)} + */ + public static int max(int a, int b) { + return StrictMath.max(a, b); + } + + /** + * finally calls {@link java.lang.Math#max(long, long)} + */ + public static long max(long a, long b) { + return StrictMath.max(a, b); + } + + /** + * finally calls {@link java.lang.Math#round(float)} + */ + public static int round(float a) { + return StrictMath.round(a); + } + + /** + * finally calls {@link java.lang.Math#round(double)} + */ + public static long round(double a) { + return StrictMath.round(a); + } + + /** + * finally calls {@link java.lang.Math#signum(double)} + */ + public static double signum(double d) { + return StrictMath.signum(d); + } + + /** + * finally calls {@link java.lang.Math#signum(float)} + */ + public static long abs(long a) { + return StrictMath.abs(a); + } + + /** + * *** methods are same as {@link java.lang.Math} methods, guaranteed by the call end *** + */ + + /** + * *** methods are same as {@link java.lang.Math} methods by mathematical algorithms*** + * / + + + /** + * mathematical integer: ceil(i) = floor(i) = i + * @return the smallest (closest to negative infinity) double value that is greater + * than or equal to the argument and is equal to a mathematical integer. + */ + public static double ceil(double a) { + return StrictMath.ceil(a); + } + + /** + * *** methods are no matters *** + */ + public static double random() { + return StrictMath.random(); + } + +} diff --git a/platform/src/main/java/arm/org/tron/common/utils/MarketOrderPriceComparatorForRocksDB.java b/platform/src/main/java/arm/org/tron/common/utils/MarketOrderPriceComparatorForRocksDB.java new file mode 100644 index 00000000000..26c246faf0e --- /dev/null +++ b/platform/src/main/java/arm/org/tron/common/utils/MarketOrderPriceComparatorForRocksDB.java @@ -0,0 +1,32 @@ +package org.tron.common.utils; + +import java.nio.ByteBuffer; +import org.rocksdb.AbstractComparator; +import org.rocksdb.ComparatorOptions; + +public class MarketOrderPriceComparatorForRocksDB extends AbstractComparator { + + public MarketOrderPriceComparatorForRocksDB(final ComparatorOptions copt) { + super(copt); + } + + @Override + public String name() { + return "MarketOrderPriceComparator"; + } + + @Override + public int compare(final ByteBuffer a, final ByteBuffer b) { + return MarketComparator.comparePriceKey(convertDataToBytes(a), convertDataToBytes(b)); + } + + /** + * DirectSlice.data().array will throw UnsupportedOperationException. + * */ + public byte[] convertDataToBytes(ByteBuffer buf) { + byte[] bytes = new byte[buf.remaining()]; + buf.get(bytes); + return bytes; + } + +} diff --git a/platform/src/main/java/common/org/tron/common/arch/Arch.java b/platform/src/main/java/common/org/tron/common/arch/Arch.java new file mode 100644 index 00000000000..20744d755a4 --- /dev/null +++ b/platform/src/main/java/common/org/tron/common/arch/Arch.java @@ -0,0 +1,86 @@ +package org.tron.common.arch; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j(topic = "arch") +public final class Arch { + + private Arch() { + } + + public static String withAll() { + final StringBuilder info = new StringBuilder(); + info.append("os.name").append(": ").append(getOsName()).append("\n"); + info.append("os.arch").append(": ").append(getOsArch()).append("\n"); + info.append("bit.model").append(": ").append(getBitModel()).append("\n"); + info.append("java.version").append(": ").append(javaVersion()).append("\n"); + info.append("java.specification.version").append(": ").append(javaSpecificationVersion()) + .append("\n"); + info.append("java.vendor").append(": ").append(javaVendor()).append("\n"); + return info.toString(); + } + + public static String getOsName() { + return System.getProperty("os.name").toLowerCase().trim(); + + } + public static String getOsArch() { + return System.getProperty("os.arch").toLowerCase().trim(); + } + + public static int getBitModel() { + String prop = System.getProperty("sun.arch.data.model"); + if (prop == null) { + prop = System.getProperty("com.ibm.vm.bitmode"); + } + if (prop != null) { + return Integer.parseInt(prop); + } + // GraalVM support, see https://github.com/fusesource/jansi/issues/162 + String arch = System.getProperty("os.arch"); + if (arch.endsWith("64") && "Substrate VM".equals(System.getProperty("java.vm.name"))) { + return 64; + } + return -1; // we don't know... + } + + public static String javaVersion() { + return System.getProperty("java.version").toLowerCase().trim(); + } + + public static String javaSpecificationVersion() { + return System.getProperty("java.specification.version").toLowerCase().trim(); + } + + public static String javaVendor() { + return System.getProperty("java.vendor").toLowerCase().trim(); + } + + public static boolean isArm64() { + String osArch = getOsArch(); + return osArch.contains("arm64") || osArch.contains("aarch64"); + } + + public static boolean isX86() { + return !isArm64(); + } + + public static boolean isJava8() { + return javaSpecificationVersion().equals("1.8"); + } + + public static void throwUnsupportedJavaException() { + if (isX86() && !isJava8()) { + logger.info(withAll()); + throw new UnsupportedOperationException(String.format( + "Java %s is required for %s architecture. Detected version %s", + "1.8 ", getOsArch(), javaSpecificationVersion())); + } + } + + public static void throwUnsupportedArm64Exception() { + if (isArm64()) { + throw new UnsupportedOperationException("unsupported on " + getOsArch() + " architecture"); + } + } +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/MarketUtils.java b/platform/src/main/java/common/org/tron/common/utils/MarketComparator.java similarity index 50% rename from plugins/src/main/java/org/tron/plugins/utils/MarketUtils.java rename to platform/src/main/java/common/org/tron/common/utils/MarketComparator.java index dbd578a59a3..c2742d4d10b 100644 --- a/plugins/src/main/java/org/tron/plugins/utils/MarketUtils.java +++ b/platform/src/main/java/common/org/tron/common/utils/MarketComparator.java @@ -1,71 +1,10 @@ -package org.tron.plugins.utils; +package org.tron.common.utils; import java.math.BigInteger; -import org.tron.plugins.utils.ByteArray; -public class MarketUtils { +public class MarketComparator { - public static final int TOKEN_ID_LENGTH = ByteArray - .fromString(Long.toString(Long.MAX_VALUE)).length; // 19 - - - - /** - * In order to avoid the difference between the data of same key stored and fetched by hashMap and - * levelDB, when creating the price key, we will find the GCD (Greatest Common Divisor) of - * sellTokenQuantity and buyTokenQuantity. - */ - public static byte[] createPairPriceKey(byte[] sellTokenId, byte[] buyTokenId, - long sellTokenQuantity, long buyTokenQuantity) { - - byte[] sellTokenQuantityBytes; - byte[] buyTokenQuantityBytes; - - // cal the GCD - long gcd = findGCD(sellTokenQuantity, buyTokenQuantity); - if (gcd == 0) { - sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity); - buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity); - } else { - sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity / gcd); - buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity / gcd); - } - - return doCreatePairPriceKey(sellTokenId, buyTokenId, - sellTokenQuantityBytes, buyTokenQuantityBytes); - } - - public static long findGCD(long number1, long number2) { - if (number1 == 0 || number2 == 0) { - return 0; - } - return calGCD(number1, number2); - } - - private static long calGCD(long number1, long number2) { - if (number2 == 0) { - return number1; - } - return calGCD(number2, number1 % number2); - } - - - private static byte[] doCreatePairPriceKey(byte[] sellTokenId, byte[] buyTokenId, - byte[] sellTokenQuantity, byte[] buyTokenQuantity) { - byte[] result = new byte[TOKEN_ID_LENGTH + TOKEN_ID_LENGTH - + sellTokenQuantity.length + buyTokenQuantity.length]; - - System.arraycopy(sellTokenId, 0, result, 0, sellTokenId.length); - System.arraycopy(buyTokenId, 0, result, TOKEN_ID_LENGTH, buyTokenId.length); - System.arraycopy(sellTokenQuantity, 0, result, - TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, - sellTokenQuantity.length); - System.arraycopy(buyTokenQuantity, 0, result, - TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + buyTokenQuantity.length, - buyTokenQuantity.length); - - return result; - } + public static final int TOKEN_ID_LENGTH = Long.toString(Long.MAX_VALUE).getBytes().length; // 19 public static int comparePriceKey(byte[] o1, byte[] o2) { @@ -76,7 +15,7 @@ public static int comparePriceKey(byte[] o1, byte[] o2) { System.arraycopy(o1, 0, pair1, 0, TOKEN_ID_LENGTH * 2); System.arraycopy(o2, 0, pair2, 0, TOKEN_ID_LENGTH * 2); - int pairResult = ByteArray.compareUnsigned(pair1, pair2); + int pairResult = compareUnsigned(pair1, pair2); if (pairResult != 0) { return pairResult; } @@ -100,10 +39,10 @@ public static int comparePriceKey(byte[] o1, byte[] o2) { System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum, getBuyTokenQuantity2, 0, longByteNum); - long sellTokenQuantity1 = ByteArray.toLong(getSellTokenQuantity1); - long buyTokenQuantity1 = ByteArray.toLong(getBuyTokenQuantity1); - long sellTokenQuantity2 = ByteArray.toLong(getSellTokenQuantity2); - long buyTokenQuantity2 = ByteArray.toLong(getBuyTokenQuantity2); + long sellTokenQuantity1 = toLong(getSellTokenQuantity1); + long buyTokenQuantity1 = toLong(getBuyTokenQuantity1); + long sellTokenQuantity2 = toLong(getSellTokenQuantity2); + long buyTokenQuantity2 = toLong(getBuyTokenQuantity2); if ((sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0) && (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0)) { @@ -145,4 +84,41 @@ public static int comparePrice(long price1SellQuantity, long price1BuyQuantity, return price1BuyQuantityBI.multiply(price2SellQuantityBI) .compareTo(price2BuyQuantityBI.multiply(price1SellQuantityBI)); } + + /** + * copy from org.bouncycastle.util.Arrays.compareUnsigned + */ + private static int compareUnsigned(byte[] a, byte[] b) { + if (a == b) { + return 0; + } + if (a == null) { + return -1; + } + if (b == null) { + return 1; + } + int minLen = StrictMath.min(a.length, b.length); + for (int i = 0; i < minLen; ++i) { + int aVal = a[i] & 0xFF; + int bVal = b[i] & 0xFF; + if (aVal < bVal) { + return -1; + } + if (aVal > bVal) { + return 1; + } + } + if (a.length < b.length) { + return -1; + } + if (a.length > b.length) { + return 1; + } + return 0; + } + + public static long toLong(byte[] b) { + return (b == null || b.length == 0) ? 0 : new BigInteger(1, b).longValue(); + } } diff --git a/chainbase/src/main/java/org/tron/common/utils/MarketOrderPriceComparatorForLevelDB.java b/platform/src/main/java/common/org/tron/common/utils/MarketOrderPriceComparatorForLevelDB.java similarity index 87% rename from chainbase/src/main/java/org/tron/common/utils/MarketOrderPriceComparatorForLevelDB.java rename to platform/src/main/java/common/org/tron/common/utils/MarketOrderPriceComparatorForLevelDB.java index c4c519f68f1..efb7219b211 100644 --- a/chainbase/src/main/java/org/tron/common/utils/MarketOrderPriceComparatorForLevelDB.java +++ b/platform/src/main/java/common/org/tron/common/utils/MarketOrderPriceComparatorForLevelDB.java @@ -1,7 +1,5 @@ package org.tron.common.utils; -import org.tron.core.capsule.utils.MarketUtils; - public class MarketOrderPriceComparatorForLevelDB implements org.iq80.leveldb.DBComparator { @Override @@ -26,7 +24,7 @@ public byte[] findShortSuccessor(byte[] key) { */ @Override public int compare(byte[] o1, byte[] o2) { - return MarketUtils.comparePriceKey(o1, o2); + return MarketComparator.comparePriceKey(o1, o2); } } diff --git a/common/src/main/java/org/tron/common/math/MathWrapper.java b/platform/src/main/java/x86/org/tron/common/math/MathWrapper.java similarity index 100% rename from common/src/main/java/org/tron/common/math/MathWrapper.java rename to platform/src/main/java/x86/org/tron/common/math/MathWrapper.java diff --git a/chainbase/src/main/java/org/tron/common/utils/MarketOrderPriceComparatorForRockDB.java b/platform/src/main/java/x86/org/tron/common/utils/MarketOrderPriceComparatorForRocksDB.java similarity index 70% rename from chainbase/src/main/java/org/tron/common/utils/MarketOrderPriceComparatorForRockDB.java rename to platform/src/main/java/x86/org/tron/common/utils/MarketOrderPriceComparatorForRocksDB.java index d3812f0f6bc..be406ff658d 100644 --- a/chainbase/src/main/java/org/tron/common/utils/MarketOrderPriceComparatorForRockDB.java +++ b/platform/src/main/java/x86/org/tron/common/utils/MarketOrderPriceComparatorForRocksDB.java @@ -3,11 +3,10 @@ import org.rocksdb.ComparatorOptions; import org.rocksdb.DirectSlice; import org.rocksdb.util.DirectBytewiseComparator; -import org.tron.core.capsule.utils.MarketUtils; -public class MarketOrderPriceComparatorForRockDB extends DirectBytewiseComparator { +public class MarketOrderPriceComparatorForRocksDB extends DirectBytewiseComparator { - public MarketOrderPriceComparatorForRockDB(final ComparatorOptions copt) { + public MarketOrderPriceComparatorForRocksDB(final ComparatorOptions copt) { super(copt); } @@ -18,7 +17,7 @@ public String name() { @Override public int compare(final DirectSlice a, final DirectSlice b) { - return MarketUtils.comparePriceKey(convertDataToBytes(a), convertDataToBytes(b)); + return MarketComparator.comparePriceKey(convertDataToBytes(a), convertDataToBytes(b)); } /** diff --git a/plugins/README.md b/plugins/README.md index 0db6f2e6143..db25811882f 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -2,7 +2,7 @@ This package contains a set of tools for TRON, the followings are the documentation for each tool. -## DB Archive +## DB Archive(Requires x86 + LevelDB) DB archive provides the ability to reformat the manifest according to the current `database`, parameters are compatible with the previous `ArchiveManifest`. @@ -26,7 +26,7 @@ DB archive provides the ability to reformat the manifest according to the curren ``` -## DB Convert +## DB Convert(Requires x86 + LevelDB) DB convert provides a helper which can convert LevelDB data to RocksDB data, parameters are compatible with previous `DBConvert`. @@ -34,15 +34,13 @@ DB convert provides a helper which can convert LevelDB data to RocksDB data, par - ``: Input path for leveldb, default: output-directory/database. - ``: Output path for rocksdb, default: output-directory-dst/database. -- `--safe`: In safe mode, read data from leveldb then put into rocksdb, it's a very time-consuming procedure. If not, just change engine.properties from leveldb to rocksdb, rocksdb - is compatible with leveldb for the current version. This may not be the case in the future, default: false. - `-h | --help`: Provide the help info. ### Examples: ```shell script # full command - java -jar Toolkit.jar db convert [-h] [--safe] + java -jar Toolkit.jar db convert [-h] # examples java -jar Toolkit.jar db convert output-directory/database /tmp/database ``` @@ -66,7 +64,7 @@ DB copy provides a helper which can copy LevelDB or RocksDB data quickly on the java -jar Toolkit.jar db cp output-directory/database /tmp/databse ``` -## DB Lite +## DB Lite(LevelDB unavailable on ARM) DB lite provides lite database, parameters are compatible with previous `LiteFullNodeTool`. @@ -134,7 +132,7 @@ Execute move command. java -jar Toolkit.jar db mv -c main_net_config.conf -d /data/tron/output-directory ``` -## DB Root +## DB Root(LevelDB unavailable on ARM) DB root provides a helper which can compute merkle root for tiny db. diff --git a/plugins/build.gradle b/plugins/build.gradle index 01afaa01708..40d68e19d2f 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -20,6 +20,15 @@ configurations.getByName('checkstyleConfig') { transitive = false } +sourceSets { + main { + java.srcDirs = rootProject.archInfo.sourceSets.main.java.srcDirs + } + test { + java.srcDirs = rootProject.archInfo.sourceSets.test.java.srcDirs + } +} + dependencies { //local libraries implementation fileTree(dir: 'libs', include: '*.jar') @@ -29,9 +38,18 @@ dependencies { implementation group: 'com.typesafe', name: 'config', version: '1.3.2' implementation group: 'me.tongfei', name: 'progressbar', version: '0.9.3' implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' - implementation group: 'org.rocksdb', name: 'rocksdbjni', version: '5.15.10' - implementation 'io.github.tronprotocol:leveldbjni-all:1.18.2' - implementation 'io.github.tronprotocol:leveldb:1.18.2' + if (rootProject.archInfo.isArm64) { + testRuntimeOnly group: 'org.fusesource.hawtjni', name: 'hawtjni-runtime', version: '1.18' // for test + implementation project(":platform") + } else { + implementation project(":platform"), { + exclude(group: 'org.fusesource.leveldbjni', module: 'leveldbjni-all') + exclude(group: 'io.github.tronprotocol', module: 'zksnark-java-sdk') + exclude(group: 'commons-io', module: 'commons-io') + } + implementation 'io.github.tronprotocol:leveldbjni-all:1.18.2' + implementation 'io.github.tronprotocol:leveldb:1.18.2' + } implementation project(":protocol") } @@ -76,6 +94,17 @@ test { destinationFile = file("../framework/build/jacoco/jacocoTest1.exec") classDumpDir = file("$buildDir/jacoco/classpathdumps") } + + if (rootProject.archInfo.isArm64) { + exclude 'org/tron/plugins/leveldb/**' + filter { + excludeTestsMatching '*.*leveldb*' + excludeTestsMatching '*.*Leveldb*' + excludeTestsMatching '*.*LevelDB*' + excludeTestsMatching '*.*LevelDb*' + excludeTestsMatching '*.*Archive*' + } + } } jacocoTestReport { @@ -94,7 +123,7 @@ def binaryRelease(taskName, jarName, mainClass) { from(sourceSets.main.output) { include "/**" } - dependsOn project(':protocol').jar // explicit_dependency + dependsOn (project(':protocol').jar, project(':platform').jar) // explicit_dependency from { configurations.runtimeClasspath.collect { // https://docs.gradle.org/current/userguide/upgrading_version_6.html#changes_6.3 it.isDirectory() ? it : zipTree(it) @@ -127,7 +156,7 @@ def createScript(project, mainClass, name) { } } } -applicationDistribution.from("../gradle/java-tron.vmoptions") { +applicationDistribution.from(rootProject.archInfo.VMOptions) { into "bin" } createScript(project, 'org.tron.plugins.ArchiveManifest', 'ArchiveManifest') diff --git a/plugins/src/main/java/arm/org/tron/plugins/ArchiveManifest.java b/plugins/src/main/java/arm/org/tron/plugins/ArchiveManifest.java new file mode 100644 index 00000000000..b7848cf4c6f --- /dev/null +++ b/plugins/src/main/java/arm/org/tron/plugins/ArchiveManifest.java @@ -0,0 +1,29 @@ +package org.tron.plugins; + +import lombok.extern.slf4j.Slf4j; +import org.tron.common.arch.Arch; + +/** + * ARM architecture only supports RocksDB, + * which does not require manifest rebuilding (manifest rebuilding is a LevelDB-only feature). + * This command is not supported but retained for compatibility. + **/ +@Slf4j(topic = "archive") +public class ArchiveManifest { + + public static void main(String[] args) { + int exitCode = run(args); + System.exit(exitCode); + } + + public static int run(String[] args) { + String tips = String.format( + "%s architecture only supports RocksDB, which does not require manifest rebuilding " + + "(manifest rebuilding is a LevelDB-only feature).", + Arch.getOsArch()); + System.out.println(tips); + logger.warn(tips); + return 0; + } + +} diff --git a/plugins/src/main/java/arm/org/tron/plugins/DbArchive.java b/plugins/src/main/java/arm/org/tron/plugins/DbArchive.java new file mode 100644 index 00000000000..03c52f33f02 --- /dev/null +++ b/plugins/src/main/java/arm/org/tron/plugins/DbArchive.java @@ -0,0 +1,50 @@ +package org.tron.plugins; + +import java.util.concurrent.Callable; +import lombok.extern.slf4j.Slf4j; +import org.tron.common.arch.Arch; +import picocli.CommandLine; +import picocli.CommandLine.Option; + +/** + * ARM architecture only supports RocksDB, + * which does not require manifest rebuilding (manifest rebuilding is a LevelDB-only feature). + * This command is not supported but retained for compatibility. + **/ +@Slf4j(topic = "archive") +@CommandLine.Command(name = "archive", description = "A helper to rewrite leveldb manifest.") +public class DbArchive implements Callable { + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + @Option(names = {"-d", "--database-directory"}, + defaultValue = "output-directory/database", + description = "java-tron database directory. Default: ${DEFAULT-VALUE}") + private String databaseDirectory; + + @Option(names = {"-b", "--batch-size"}, + defaultValue = "80000", + description = "deal manifest batch size. Default: ${DEFAULT-VALUE}") + private int maxBatchSize; + + @Option(names = {"-m", "--manifest-size"}, + defaultValue = "0", + description = "manifest min size(M) to archive. Default: ${DEFAULT-VALUE}") + private int maxManifestSize; + + @Option(names = {"-h", "--help"}) + private boolean help; + + + @Override + public Integer call() throws Exception { + String tips = String.format( + "%s architecture only supports RocksDB, which does not require manifest rebuilding " + + "(manifest rebuilding is a LevelDB-only feature).", + Arch.getOsArch()); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme().errorText(tips)); + logger.warn(tips); + return 0; + } + +} diff --git a/plugins/src/main/java/org/tron/plugins/Db.java b/plugins/src/main/java/common/org/tron/plugins/Db.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/Db.java rename to plugins/src/main/java/common/org/tron/plugins/Db.java diff --git a/plugins/src/main/java/org/tron/plugins/DbConvert.java b/plugins/src/main/java/common/org/tron/plugins/DbConvert.java similarity index 91% rename from plugins/src/main/java/org/tron/plugins/DbConvert.java rename to plugins/src/main/java/common/org/tron/plugins/DbConvert.java index a75b235bbcf..37ea6bdeca4 100644 --- a/plugins/src/main/java/org/tron/plugins/DbConvert.java +++ b/plugins/src/main/java/common/org/tron/plugins/DbConvert.java @@ -20,6 +20,7 @@ import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.Status; +import org.tron.common.arch.Arch; import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.FileUtils; import picocli.CommandLine; @@ -49,20 +50,19 @@ public class DbConvert implements Callable { description = "Output path for rocksdb. Default: ${DEFAULT-VALUE}") private File dest; - @CommandLine.Option(names = {"--safe"}, - description = "In safe mode, read data from leveldb then put rocksdb." - + "If not, just change engine.properties from leveldb to rocksdb," - + "rocksdb is compatible with leveldb for current version." - + "This may not be the case in the future." - + "Default: ${DEFAULT-VALUE}") - private boolean safe; - @CommandLine.Option(names = {"-h", "--help"}) private boolean help; @Override public Integer call() throws Exception { + if (Arch.isArm64()) { + String tips = String.format("This command is not supported on %s architecture.", + Arch.getOsArch()); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme().errorText(tips)); + logger.error(tips); + return 0; + } if (help) { spec.commandLine().usage(System.out); return 0; @@ -95,12 +95,12 @@ public Integer call() throws Exception { final long time = System.currentTimeMillis(); List services = new ArrayList<>(); files.forEach(f -> services.add( - new DbConverter(src.getPath(), dest.getPath(), f.getName(), safe))); + new DbConverter(src.getPath(), dest.getPath(), f.getName()))); cpList.forEach(f -> services.add( new DbConverter( Paths.get(src.getPath(), DBUtils.CHECKPOINT_DB_V2).toString(), Paths.get(dest.getPath(), DBUtils.CHECKPOINT_DB_V2).toString(), - f.getName(), safe))); + f.getName()))); List fails = ProgressBar.wrap(services.stream(), "convert task").parallel().map( dbConverter -> { try { @@ -140,15 +140,12 @@ static class DbConverter implements Converter { private long srcDbValueSum = 0L; private long dstDbValueSum = 0L; - private boolean safe; - - public DbConverter(String srcDir, String dstDir, String name, boolean safe) { + public DbConverter(String srcDir, String dstDir, String name) { this.srcDir = srcDir; this.dstDir = dstDir; this.dbName = name; this.srcDbPath = Paths.get(this.srcDir, name); this.dstDbPath = Paths.get(this.dstDir, name); - this.safe = safe; } @Override @@ -178,23 +175,15 @@ public boolean doConvert() throws Exception { FileUtils.createDirIfNotExists(dstDir); logger.info("Convert database {} start", this.dbName); - if (safe) { - convertLevelToRocks(); - compact(); - } else { - FileUtils.copyDir(Paths.get(srcDir), Paths.get(dstDir), dbName); - } + convertLevelToRocks(); + compact(); + boolean result = check() && createEngine(dstDbPath.toString()); long etime = System.currentTimeMillis(); if (result) { - if (safe) { - logger.info("Convert database {} successful end with {} key-value {} minutes", - this.dbName, this.srcDbKeyCount, (etime - startTime) / 1000.0 / 60); - } else { - logger.info("Convert database {} successful end {} minutes", - this.dbName, (etime - startTime) / 1000.0 / 60); - } + logger.info("Convert database {} successful end with {} key-value {} minutes", + this.dbName, this.srcDbKeyCount, (etime - startTime) / 1000.0 / 60); } else { logger.info("Convert database {} failure", this.dbName); @@ -310,9 +299,6 @@ private void compact() throws RocksDBException { } private boolean check() throws RocksDBException { - if (!safe) { - return true; - } try ( RocksDB rocks = DBUtils.newRocksDbReadOnly(this.dstDbPath); org.rocksdb.ReadOptions r = new org.rocksdb.ReadOptions().setFillCache(false); diff --git a/plugins/src/main/java/org/tron/plugins/DbCopy.java b/plugins/src/main/java/common/org/tron/plugins/DbCopy.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/DbCopy.java rename to plugins/src/main/java/common/org/tron/plugins/DbCopy.java diff --git a/plugins/src/main/java/org/tron/plugins/DbLite.java b/plugins/src/main/java/common/org/tron/plugins/DbLite.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/DbLite.java rename to plugins/src/main/java/common/org/tron/plugins/DbLite.java diff --git a/plugins/src/main/java/org/tron/plugins/DbMove.java b/plugins/src/main/java/common/org/tron/plugins/DbMove.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/DbMove.java rename to plugins/src/main/java/common/org/tron/plugins/DbMove.java diff --git a/plugins/src/main/java/org/tron/plugins/DbRoot.java b/plugins/src/main/java/common/org/tron/plugins/DbRoot.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/DbRoot.java rename to plugins/src/main/java/common/org/tron/plugins/DbRoot.java diff --git a/plugins/src/main/java/org/tron/plugins/Toolkit.java b/plugins/src/main/java/common/org/tron/plugins/Toolkit.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/Toolkit.java rename to plugins/src/main/java/common/org/tron/plugins/Toolkit.java diff --git a/plugins/src/main/java/org/tron/plugins/utils/ByteArray.java b/plugins/src/main/java/common/org/tron/plugins/utils/ByteArray.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/utils/ByteArray.java rename to plugins/src/main/java/common/org/tron/plugins/utils/ByteArray.java diff --git a/plugins/src/main/java/org/tron/plugins/utils/CryptoUitls.java b/plugins/src/main/java/common/org/tron/plugins/utils/CryptoUitls.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/utils/CryptoUitls.java rename to plugins/src/main/java/common/org/tron/plugins/utils/CryptoUitls.java diff --git a/plugins/src/main/java/org/tron/plugins/utils/DBUtils.java b/plugins/src/main/java/common/org/tron/plugins/utils/DBUtils.java similarity index 94% rename from plugins/src/main/java/org/tron/plugins/utils/DBUtils.java rename to plugins/src/main/java/common/org/tron/plugins/utils/DBUtils.java index f8559d5dba8..507a8ad798e 100644 --- a/plugins/src/main/java/org/tron/plugins/utils/DBUtils.java +++ b/plugins/src/main/java/common/org/tron/plugins/utils/DBUtils.java @@ -16,8 +16,9 @@ import org.rocksdb.Options; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; -import org.tron.plugins.comparator.MarketOrderPriceComparatorForLevelDB; -import org.tron.plugins.comparator.MarketOrderPriceComparatorForRockDB; +import org.tron.common.arch.Arch; +import org.tron.common.utils.MarketOrderPriceComparatorForLevelDB; +import org.tron.common.utils.MarketOrderPriceComparatorForRocksDB; import org.tron.protos.Protocol; public class DBUtils { @@ -65,6 +66,7 @@ static Operator valueOf(byte b) { public static final String ROCKSDB = "ROCKSDB"; public static DB newLevelDb(Path db) throws IOException { + Arch.throwUnsupportedArm64Exception(); File file = db.toFile(); org.iq80.leveldb.Options dbOptions = newDefaultLevelDbOptions(); if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(file.getName())) { @@ -115,7 +117,7 @@ private static Options newDefaultRocksDbOptions(boolean forBulkLoad) { public static RocksDB newRocksDb(Path db) throws RocksDBException { try (Options options = newDefaultRocksDbOptions(false)) { if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); + options.setComparator(new MarketOrderPriceComparatorForRocksDB(new ComparatorOptions())); } return RocksDB.open(options, db.toString()); } @@ -124,7 +126,7 @@ public static RocksDB newRocksDb(Path db) throws RocksDBException { public static RocksDB newRocksDbForBulkLoad(Path db) throws RocksDBException { try (Options options = newDefaultRocksDbOptions(true)) { if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); + options.setComparator(new MarketOrderPriceComparatorForRocksDB(new ComparatorOptions())); } return RocksDB.open(options, db.toString()); } @@ -134,7 +136,7 @@ public static RocksDB newRocksDbForBulkLoad(Path db) throws RocksDBException { public static RocksDB newRocksDbReadOnly(Path db) throws RocksDBException { try (Options options = newDefaultRocksDbOptions(false)) { if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); + options.setComparator(new MarketOrderPriceComparatorForRocksDB(new ComparatorOptions())); } return RocksDB.openReadOnly(options, db.toString()); } diff --git a/plugins/src/main/java/org/tron/plugins/utils/FileUtils.java b/plugins/src/main/java/common/org/tron/plugins/utils/FileUtils.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/utils/FileUtils.java rename to plugins/src/main/java/common/org/tron/plugins/utils/FileUtils.java diff --git a/plugins/src/main/java/common/org/tron/plugins/utils/MarketUtils.java b/plugins/src/main/java/common/org/tron/plugins/utils/MarketUtils.java new file mode 100644 index 00000000000..9bcfd5e71ce --- /dev/null +++ b/plugins/src/main/java/common/org/tron/plugins/utils/MarketUtils.java @@ -0,0 +1,68 @@ +package org.tron.plugins.utils; + +public class MarketUtils { + + public static final int TOKEN_ID_LENGTH = ByteArray + .fromString(Long.toString(Long.MAX_VALUE)).length; // 19 + + + + /** + * In order to avoid the difference between the data of same key stored and fetched by hashMap and + * levelDB, when creating the price key, we will find the GCD (Greatest Common Divisor) of + * sellTokenQuantity and buyTokenQuantity. + */ + public static byte[] createPairPriceKey(byte[] sellTokenId, byte[] buyTokenId, + long sellTokenQuantity, long buyTokenQuantity) { + + byte[] sellTokenQuantityBytes; + byte[] buyTokenQuantityBytes; + + // cal the GCD + long gcd = findGCD(sellTokenQuantity, buyTokenQuantity); + if (gcd == 0) { + sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity); + buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity); + } else { + sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity / gcd); + buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity / gcd); + } + + return doCreatePairPriceKey(sellTokenId, buyTokenId, + sellTokenQuantityBytes, buyTokenQuantityBytes); + } + + public static long findGCD(long number1, long number2) { + if (number1 == 0 || number2 == 0) { + return 0; + } + return calGCD(number1, number2); + } + + private static long calGCD(long number1, long number2) { + if (number2 == 0) { + return number1; + } + return calGCD(number2, number1 % number2); + } + + + private static byte[] doCreatePairPriceKey(byte[] sellTokenId, byte[] buyTokenId, + byte[] sellTokenQuantity, byte[] buyTokenQuantity) { + byte[] result = new byte[TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + + sellTokenQuantity.length + buyTokenQuantity.length]; + + System.arraycopy(sellTokenId, 0, result, 0, sellTokenId.length); + System.arraycopy(buyTokenId, 0, result, TOKEN_ID_LENGTH, buyTokenId.length); + System.arraycopy(sellTokenQuantity, 0, result, + TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, + sellTokenQuantity.length); + System.arraycopy(buyTokenQuantity, 0, result, + TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + buyTokenQuantity.length, + buyTokenQuantity.length); + + return result; + } + + +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/MerkleRoot.java b/plugins/src/main/java/common/org/tron/plugins/utils/MerkleRoot.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/utils/MerkleRoot.java rename to plugins/src/main/java/common/org/tron/plugins/utils/MerkleRoot.java diff --git a/plugins/src/main/java/org/tron/plugins/utils/Sha256Hash.java b/plugins/src/main/java/common/org/tron/plugins/utils/Sha256Hash.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/utils/Sha256Hash.java rename to plugins/src/main/java/common/org/tron/plugins/utils/Sha256Hash.java diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/DBInterface.java b/plugins/src/main/java/common/org/tron/plugins/utils/db/DBInterface.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/utils/db/DBInterface.java rename to plugins/src/main/java/common/org/tron/plugins/utils/db/DBInterface.java diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/DBIterator.java b/plugins/src/main/java/common/org/tron/plugins/utils/db/DBIterator.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/utils/db/DBIterator.java rename to plugins/src/main/java/common/org/tron/plugins/utils/db/DBIterator.java diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/DbTool.java b/plugins/src/main/java/common/org/tron/plugins/utils/db/DbTool.java similarity index 85% rename from plugins/src/main/java/org/tron/plugins/utils/db/DbTool.java rename to plugins/src/main/java/common/org/tron/plugins/utils/db/DbTool.java index 429025e8f8b..cf4c69505bc 100644 --- a/plugins/src/main/java/org/tron/plugins/utils/db/DbTool.java +++ b/plugins/src/main/java/common/org/tron/plugins/utils/db/DbTool.java @@ -18,8 +18,8 @@ public class DbTool { private static final String KEY_ENGINE = "ENGINE"; private static final String ENGINE_FILE = "engine.properties"; - private static final String FILE_SEPARATOR = File.separator; private static final String ROCKSDB = "ROCKSDB"; + private static final String LEVELDB = "LEVELDB"; private static final Map dbMap = Maps.newConcurrentMap(); @@ -162,8 +162,7 @@ public static void close() { } private static DbType getDbType(String sourceDir, String dbName) { - String engineFile = String.format("%s%s%s%s%s", sourceDir, FILE_SEPARATOR, - dbName, FILE_SEPARATOR, ENGINE_FILE); + String engineFile = Paths.get(sourceDir, dbName, ENGINE_FILE).toString(); if (!new File(engineFile).exists()) { return DbType.LevelDB; } @@ -175,13 +174,22 @@ private static DbType getDbType(String sourceDir, String dbName) { } } - private static LevelDBImpl openLevelDb(Path db, String name) throws IOException { - return new LevelDBImpl(DBUtils.newLevelDb(db), name); + public static LevelDBImpl openLevelDb(Path db, String name) throws IOException { + LevelDBImpl leveldb = new LevelDBImpl(DBUtils.newLevelDb(db), name); + tryInitEngineFile(db, LEVELDB); + return leveldb; } - private static RocksDBImpl openRocksDb(Path db, String name) throws RocksDBException { - return new RocksDBImpl(DBUtils.newRocksDb(db), name); + public static RocksDBImpl openRocksDb(Path db, String name) throws RocksDBException { + RocksDBImpl rocksdb = new RocksDBImpl(DBUtils.newRocksDb(db), name); + tryInitEngineFile(db, ROCKSDB); + return rocksdb; } - + private static void tryInitEngineFile(Path db, String engine) { + String engineFile = Paths.get(db.toString(), ENGINE_FILE).toString(); + if (FileUtils.createFileIfNotExists(engineFile)) { + FileUtils.writeProperty(engineFile, KEY_ENGINE, engine); + } + } } diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java b/plugins/src/main/java/common/org/tron/plugins/utils/db/LevelDBImpl.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java rename to plugins/src/main/java/common/org/tron/plugins/utils/db/LevelDBImpl.java diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/LevelDBIterator.java b/plugins/src/main/java/common/org/tron/plugins/utils/db/LevelDBIterator.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/utils/db/LevelDBIterator.java rename to plugins/src/main/java/common/org/tron/plugins/utils/db/LevelDBIterator.java diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java b/plugins/src/main/java/common/org/tron/plugins/utils/db/RockDBIterator.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java rename to plugins/src/main/java/common/org/tron/plugins/utils/db/RockDBIterator.java diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java b/plugins/src/main/java/common/org/tron/plugins/utils/db/RocksDBImpl.java similarity index 100% rename from plugins/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java rename to plugins/src/main/java/common/org/tron/plugins/utils/db/RocksDBImpl.java diff --git a/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java b/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java deleted file mode 100644 index 0879f770e1f..00000000000 --- a/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.tron.plugins.comparator; - -import org.iq80.leveldb.DBComparator; -import org.tron.plugins.utils.MarketUtils; - -public class MarketOrderPriceComparatorForLevelDB implements DBComparator { - - @Override - public String name() { - return "MarketOrderPriceComparator"; - } - - @Override - public byte[] findShortestSeparator(byte[] start, byte[] limit) { - return new byte[0]; - } - - @Override - public byte[] findShortSuccessor(byte[] key) { - return new byte[0]; - } - - @Override - public int compare(byte[] o1, byte[] o2) { - return MarketUtils.comparePriceKey(o1, o2); - } - -} \ No newline at end of file diff --git a/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java b/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java deleted file mode 100644 index cd718bdd2d7..00000000000 --- a/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.tron.plugins.comparator; - -import org.rocksdb.ComparatorOptions; -import org.rocksdb.DirectSlice; -import org.rocksdb.util.DirectBytewiseComparator; -import org.tron.plugins.utils.MarketUtils; - -public class MarketOrderPriceComparatorForRockDB extends DirectBytewiseComparator { - - public MarketOrderPriceComparatorForRockDB(final ComparatorOptions copt) { - super(copt); - } - - @Override - public String name() { - return "MarketOrderPriceComparator"; - } - - @Override - public int compare(final DirectSlice a, final DirectSlice b) { - return MarketUtils.comparePriceKey(convertDataToBytes(a), convertDataToBytes(b)); - } - - /** - * DirectSlice.data().array will throw UnsupportedOperationException. - * */ - public byte[] convertDataToBytes(DirectSlice directSlice) { - int capacity = directSlice.data().capacity(); - byte[] bytes = new byte[capacity]; - - for (int i = 0; i < capacity; i++) { - bytes[i] = directSlice.get(i); - } - - return bytes; - } - -} \ No newline at end of file diff --git a/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java b/plugins/src/main/java/x86/org/tron/plugins/ArchiveManifest.java similarity index 98% rename from plugins/src/main/java/org/tron/plugins/ArchiveManifest.java rename to plugins/src/main/java/x86/org/tron/plugins/ArchiveManifest.java index 4d54df6d299..1d7a91027bf 100644 --- a/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java +++ b/plugins/src/main/java/x86/org/tron/plugins/ArchiveManifest.java @@ -35,6 +35,7 @@ import org.iq80.leveldb.DB; import org.iq80.leveldb.Options; import org.iq80.leveldb.impl.Filename; +import org.tron.plugins.utils.DBUtils; import picocli.CommandLine; import picocli.CommandLine.Option; @@ -183,7 +184,7 @@ public boolean checkManifest(String dir) throws IOException { return false; } logger.info("CurrentName {}/{},size {} kb.", dir, currentName, current.length() / 1024); - if ("market_pair_price_to_order".equalsIgnoreCase(this.name)) { + if (DBUtils.MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(this.name)) { logger.info("Db {} ignored.", this.name); return false; } diff --git a/plugins/src/main/java/org/tron/plugins/DbArchive.java b/plugins/src/main/java/x86/org/tron/plugins/DbArchive.java similarity index 98% rename from plugins/src/main/java/org/tron/plugins/DbArchive.java rename to plugins/src/main/java/x86/org/tron/plugins/DbArchive.java index e3032731ede..15bb281babf 100644 --- a/plugins/src/main/java/org/tron/plugins/DbArchive.java +++ b/plugins/src/main/java/x86/org/tron/plugins/DbArchive.java @@ -19,6 +19,7 @@ import org.iq80.leveldb.DB; import org.iq80.leveldb.Options; import org.iq80.leveldb.impl.Filename; +import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.FileUtils; import picocli.CommandLine; import picocli.CommandLine.Option; @@ -163,7 +164,7 @@ public boolean checkManifest(String dir) throws IOException { return false; } logger.info("CurrentName {}/{},size {} kb.", dir, currentName, current.length() / 1024); - if ("market_pair_price_to_order".equalsIgnoreCase(this.name)) { + if (DBUtils.MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(this.name)) { logger.info("Db {} ignored.", this.name); return false; } diff --git a/plugins/src/test/java/org/tron/plugins/DbCopyTest.java b/plugins/src/test/java/org/tron/plugins/DbCopyTest.java index 9e488a592aa..7548812b799 100644 --- a/plugins/src/test/java/org/tron/plugins/DbCopyTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbCopyTest.java @@ -4,12 +4,23 @@ import java.util.UUID; import org.junit.Assert; import org.junit.Test; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; public class DbCopyTest extends DbTest { @Test - public void testRun() { + public void testRunForLevelDB() throws RocksDBException, IOException { + init(DbTool.DbType.LevelDB); + String[] args = new String[] { "db", "cp", INPUT_DIRECTORY, + genarateTmpDir()}; + Assert.assertEquals(0, cli.execute(args)); + } + + @Test + public void testRunForRocksDB() throws RocksDBException, IOException { + init(DbTool.DbType.RocksDB); String[] args = new String[] { "db", "cp", INPUT_DIRECTORY, genarateTmpDir()}; Assert.assertEquals(0, cli.execute(args)); diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java index b4c66c9820f..b0c7f3b96be 100644 --- a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java @@ -67,9 +67,10 @@ public void shutdown() throws InterruptedException { context.close(); } - public void init() throws IOException { + public void init(String dbType) throws IOException { dbPath = folder.newFolder().toString(); - Args.setParam(new String[]{"-d", dbPath, "-w", "--p2p-disable", "true"}, + Args.setParam(new String[]{ + "-d", dbPath, "-w", "--p2p-disable", "true", "--storage-db-engine", dbType}, "config-localtest.conf"); // allow account root Args.getInstance().setAllowAccountStateRoot(1); @@ -85,11 +86,11 @@ public void clear() { Args.clearParam(); } - void testTools(String dbType, int checkpointVersion) + public void testTools(String dbType, int checkpointVersion) throws InterruptedException, IOException { logger.info("dbType {}, checkpointVersion {}", dbType, checkpointVersion); dbPath = String.format("%s_%s_%d", dbPath, dbType, System.currentTimeMillis()); - init(); + init(dbType); final String[] argsForSnapshot = new String[]{"-o", "split", "-t", "snapshot", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", @@ -101,7 +102,6 @@ void testTools(String dbType, int checkpointVersion) final String[] argsForMerge = new String[]{"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", dbPath + File.separator + "history"}; - Args.getInstance().getStorage().setDbEngine(dbType); Args.getInstance().getStorage().setCheckpointVersion(checkpointVersion); DbLite.setRecentBlks(3); // start fullNode diff --git a/plugins/src/test/java/org/tron/plugins/DbMoveTest.java b/plugins/src/test/java/org/tron/plugins/DbMoveTest.java index c1bc6b470fc..5b25739f272 100644 --- a/plugins/src/test/java/org/tron/plugins/DbMoveTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbMoveTest.java @@ -1,52 +1,41 @@ package org.tron.plugins; -import static org.iq80.leveldb.impl.Iq80DBFactory.factory; - import java.io.File; import java.io.IOException; import java.net.URL; import java.nio.file.Paths; -import java.util.UUID; import lombok.extern.slf4j.Slf4j; -import org.junit.AfterClass; +import org.junit.After; import org.junit.Assert; -import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; -import org.tron.plugins.utils.FileUtils; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; @Slf4j public class DbMoveTest { private static final String OUTPUT_DIRECTORY = "output-directory-toolkit"; - private static final String OUTPUT_DIRECTORY_DATABASE = - Paths.get(OUTPUT_DIRECTORY,"ori","database").toString(); - private static final String ENGINE = "ENGINE"; - private static final String LEVELDB = "LEVELDB"; - private static final String ACCOUNT = "account"; - private static final String TRANS = "trans"; - private static final String MARKET = "market_pair_price_to_order"; - private static final String ENGINE_FILE = "engine.properties"; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); - @BeforeClass - public static void init() throws IOException { - File file = new File(OUTPUT_DIRECTORY_DATABASE, ACCOUNT); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); + private static final String ACCOUNT = "account"; + private static final String TRANS = "trans"; - file = new File(OUTPUT_DIRECTORY_DATABASE, MARKET); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); - file = new File(OUTPUT_DIRECTORY_DATABASE, TRANS); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); + private void init(DbTool.DbType dbType, String path) throws IOException, RocksDBException { + DbTool.getDB(path, ACCOUNT, dbType).close(); + DbTool.getDB(path, DBUtils.MARKET_PAIR_PRICE_TO_ORDER, dbType).close(); + DbTool.getDB(path, TRANS, dbType).close(); } - @AfterClass - public static void destroy() { + @After + public void destroy() { deleteDir(new File(OUTPUT_DIRECTORY)); } @@ -74,9 +63,23 @@ private static String getConfig(String config) { } @Test - public void testMv() { + public void testMvForLevelDB() throws RocksDBException, IOException { + File database = temporaryFolder.newFolder("database"); + init(DbTool.DbType.LevelDB, Paths.get(database.getPath()).toString()); + String[] args = new String[] {"db", "mv", "-d", + database.getParent(), "-c", + getConfig("config.conf")}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(0, cli.execute(args)); + Assert.assertEquals(2, cli.execute(args)); + } + + @Test + public void testMvForRocksDB() throws RocksDBException, IOException { + File database = temporaryFolder.newFolder("database"); + init(DbTool.DbType.RocksDB, Paths.get(database.getPath()).toString()); String[] args = new String[] {"db", "mv", "-d", - Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + database.getParent(), "-c", getConfig("config.conf")}; CommandLine cli = new CommandLine(new Toolkit()); Assert.assertEquals(0, cli.execute(args)); @@ -84,9 +87,10 @@ public void testMv() { } @Test - public void testDuplicate() { + public void testDuplicate() throws IOException { + File output = temporaryFolder.newFolder(); String[] args = new String[] {"db", "mv", "-d", - Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + output.getPath(), "-c", getConfig("config-duplicate.conf")}; CommandLine cli = new CommandLine(new Toolkit()); Assert.assertEquals(2, cli.execute(args)); @@ -107,20 +111,19 @@ public void testDicNotExist() { } @Test - public void testConfNotExist() { + public void testConfNotExist() throws IOException { + File output = temporaryFolder.newFolder(); String[] args = new String[] {"db", "mv", "-d", - Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + output.getPath(), "-c", "config.conf"}; CommandLine cli = new CommandLine(new Toolkit()); Assert.assertEquals(2, cli.execute(args)); } @Test - public void testEmpty() { - File file = new File(OUTPUT_DIRECTORY_DATABASE + File.separator + UUID.randomUUID()); - file.mkdirs(); - file.deleteOnExit(); - String[] args = new String[] {"db", "mv", "-d", file.toString(), "-c", + public void testEmpty() throws IOException { + File output = temporaryFolder.newFolder(); + String[] args = new String[] {"db", "mv", "-d", output.getPath(), "-c", getConfig("config.conf")}; CommandLine cli = new CommandLine(new Toolkit()); diff --git a/plugins/src/test/java/org/tron/plugins/DbRootTest.java b/plugins/src/test/java/org/tron/plugins/DbRootTest.java index b86688f77d5..9d032a43103 100644 --- a/plugins/src/test/java/org/tron/plugins/DbRootTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbRootTest.java @@ -9,10 +9,8 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.rocksdb.RocksDBException; -import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.db.DBInterface; import org.tron.plugins.utils.db.DbTool; -import org.tron.plugins.utils.db.LevelDBImpl; import picocli.CommandLine; @Slf4j @@ -27,8 +25,18 @@ public class DbRootTest { private static final String EMPTY_DB = "empty"; private static final String ERROR_DB = "error"; + + @Test + public void testRootForLevelDB() throws RocksDBException, IOException { + testRoot(DbTool.DbType.LevelDB); + } + @Test - public void testRoot() throws IOException, RocksDBException { + public void testRootForRocksDB() throws RocksDBException, IOException { + testRoot(DbTool.DbType.RocksDB); + } + + public void testRoot(DbTool.DbType dbType) throws IOException, RocksDBException { File file = folder.newFolder(); @@ -36,8 +44,8 @@ public void testRoot() throws IOException, RocksDBException { Assert.assertTrue(database.mkdirs()); - try (DBInterface normal = DbTool.getDB(database.toString(), NORMAL_DB, DbTool.DbType.LevelDB); - DBInterface empty = DbTool.getDB(database.toString(), EMPTY_DB, DbTool.DbType.RocksDB)) { + try (DBInterface normal = DbTool.getDB(database.toString(), NORMAL_DB, dbType); + DBInterface empty = DbTool.getDB(database.toString(), EMPTY_DB, dbType)) { for (int i = 0; i < 10; i++) { normal.put(("" + i).getBytes(), (NORMAL_DB + "-" + i).getBytes()); } @@ -53,8 +61,7 @@ public void testRoot() throws IOException, RocksDBException { "--db", EMPTY_DB}; Assert.assertEquals(0, cli.execute(args)); - try (DBInterface errorDb = new LevelDBImpl( - DBUtils.newLevelDb(Paths.get(database.toString(), ERROR_DB)), ERROR_DB)) { + try (DBInterface errorDb = DbTool.getDB(database.toString(), ERROR_DB, dbType)) { for (int i = 0; i < 10; i++) { errorDb.put(("" + i).getBytes(), (ERROR_DB + "-" + i).getBytes()); } diff --git a/plugins/src/test/java/org/tron/plugins/DbTest.java b/plugins/src/test/java/org/tron/plugins/DbTest.java index 8605fa18d50..9337723e8ff 100644 --- a/plugins/src/test/java/org/tron/plugins/DbTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbTest.java @@ -5,42 +5,43 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.UUID; -import org.iq80.leveldb.DB; -import org.junit.Before; import org.junit.Rule; import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; import org.tron.plugins.utils.ByteArray; import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.MarketUtils; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; public class DbTest { - String INPUT_DIRECTORY; + public String INPUT_DIRECTORY; private static final String ACCOUNT = "account"; private static final String MARKET = DBUtils.MARKET_PAIR_PRICE_TO_ORDER; - CommandLine cli = new CommandLine(new Toolkit()); + public CommandLine cli = new CommandLine(new Toolkit()); String tmpDir = System.getProperty("java.io.tmpdir"); @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); - @Before - public void init() throws IOException { + public void init(DbTool.DbType dbType) throws IOException, RocksDBException { INPUT_DIRECTORY = temporaryFolder.newFolder().toString(); - initDB(new File(INPUT_DIRECTORY, ACCOUNT)); - initDB(new File(INPUT_DIRECTORY, MARKET)); - initDB(new File(INPUT_DIRECTORY, DBUtils.CHECKPOINT_DB_V2)); + initDB(INPUT_DIRECTORY, ACCOUNT, dbType); + initDB(INPUT_DIRECTORY, MARKET, dbType); + initDB(INPUT_DIRECTORY, DBUtils.CHECKPOINT_DB_V2, dbType); } - private static void initDB(File file) throws IOException { - if (DBUtils.CHECKPOINT_DB_V2.equalsIgnoreCase(file.getName())) { - File dbFile = new File(file, DBUtils.CHECKPOINT_DB_V2); + private static void initDB(String sourceDir, String dbName, DbTool.DbType dbType) + throws IOException, RocksDBException { + if (DBUtils.CHECKPOINT_DB_V2.equalsIgnoreCase(dbName)) { + File dbFile = new File(sourceDir, DBUtils.CHECKPOINT_DB_V2); if (dbFile.mkdirs()) { for (int i = 0; i < 3; i++) { - try (DB db = DBUtils.newLevelDb(Paths.get(dbFile.getPath(), - System.currentTimeMillis() + ""))) { + try (DBInterface db = DbTool.getDB(dbFile.getPath(), + System.currentTimeMillis() + "", dbType)) { for (int j = 0; j < 100; j++) { byte[] bytes = UUID.randomUUID().toString().getBytes(); db.put(bytes, bytes); @@ -50,8 +51,8 @@ private static void initDB(File file) throws IOException { } return; } - try (DB db = DBUtils.newLevelDb(file.toPath())) { - if (MARKET.equalsIgnoreCase(file.getName())) { + try (DBInterface db = DbTool.getDB(sourceDir, dbName, dbType)) { + if (MARKET.equalsIgnoreCase(dbName)) { byte[] sellTokenID1 = ByteArray.fromString("100"); byte[] buyTokenID1 = ByteArray.fromString("200"); byte[] pairPriceKey1 = MarketUtils.createPairPriceKey( diff --git a/plugins/src/test/java/org/tron/plugins/ArchiveManifestTest.java b/plugins/src/test/java/org/tron/plugins/leveldb/ArchiveManifestTest.java similarity index 69% rename from plugins/src/test/java/org/tron/plugins/ArchiveManifestTest.java rename to plugins/src/test/java/org/tron/plugins/leveldb/ArchiveManifestTest.java index 4fd9e537d05..f5880d82e39 100644 --- a/plugins/src/test/java/org/tron/plugins/ArchiveManifestTest.java +++ b/plugins/src/test/java/org/tron/plugins/leveldb/ArchiveManifestTest.java @@ -1,6 +1,4 @@ -package org.tron.plugins; - -import static org.iq80.leveldb.impl.Iq80DBFactory.factory; +package org.tron.plugins.leveldb; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -14,47 +12,41 @@ import java.nio.charset.StandardCharsets; import java.util.Properties; import java.util.UUID; - import lombok.extern.slf4j.Slf4j; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.ArchiveManifest; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.db.DbTool; @Slf4j public class ArchiveManifestTest { - private static final String OUTPUT_DIRECTORY = "output-directory/database/archiveManifest"; + @ClassRule + public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static String OUTPUT_DIRECTORY; - private static final String ENGINE = "ENGINE"; - private static final String LEVELDB = "LEVELDB"; - private static final String ROCKSDB = "ROCKSDB"; private static final String ACCOUNT = "account"; private static final String ACCOUNT_ROCKSDB = "account-rocksdb"; - private static final String MARKET = "market_pair_price_to_order"; - private static final String ENGINE_FILE = "engine.properties"; - @BeforeClass - public static void init() throws IOException { - File file = new File(OUTPUT_DIRECTORY,ACCOUNT); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); - - file = new File(OUTPUT_DIRECTORY,MARKET); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); + public static void init() throws IOException, RocksDBException { + OUTPUT_DIRECTORY = temporaryFolder.newFolder("database").toString(); + File file = new File(OUTPUT_DIRECTORY, ACCOUNT); + DbTool.openLevelDb(file.toPath(),ACCOUNT).close(); - file = new File(OUTPUT_DIRECTORY,ACCOUNT_ROCKSDB); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,ROCKSDB); + file = new File(OUTPUT_DIRECTORY, DBUtils.MARKET_PAIR_PRICE_TO_ORDER); + DbTool.openLevelDb(file.toPath(), DBUtils.MARKET_PAIR_PRICE_TO_ORDER).close(); - } + file = new File(OUTPUT_DIRECTORY, ACCOUNT_ROCKSDB); + DbTool.openRocksDb(file.toPath(), ACCOUNT_ROCKSDB).close(); - @AfterClass - public static void destroy() { - deleteDir(new File(OUTPUT_DIRECTORY)); } @Test diff --git a/plugins/src/test/java/org/tron/plugins/DbArchiveTest.java b/plugins/src/test/java/org/tron/plugins/leveldb/DbArchiveTest.java similarity index 71% rename from plugins/src/test/java/org/tron/plugins/DbArchiveTest.java rename to plugins/src/test/java/org/tron/plugins/leveldb/DbArchiveTest.java index 10bed418764..69dca01e4f8 100644 --- a/plugins/src/test/java/org/tron/plugins/DbArchiveTest.java +++ b/plugins/src/test/java/org/tron/plugins/leveldb/DbArchiveTest.java @@ -1,6 +1,4 @@ -package org.tron.plugins; - -import static org.iq80.leveldb.impl.Iq80DBFactory.factory; +package org.tron.plugins.leveldb; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -12,48 +10,43 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; import java.util.Properties; import java.util.UUID; import lombok.extern.slf4j.Slf4j; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.Toolkit; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; @Slf4j public class DbArchiveTest { - private static final String OUTPUT_DIRECTORY = "output-directory/database/dbArchive"; + @ClassRule + public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static String OUTPUT_DIRECTORY; - private static final String ENGINE = "ENGINE"; - private static final String LEVELDB = "LEVELDB"; - private static final String ROCKSDB = "ROCKSDB"; private static final String ACCOUNT = "account"; private static final String ACCOUNT_ROCKSDB = "account-rocksdb"; - private static final String MARKET = "market_pair_price_to_order"; - private static final String ENGINE_FILE = "engine.properties"; @BeforeClass - public static void init() throws IOException { - File file = new File(OUTPUT_DIRECTORY,ACCOUNT); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); + public static void init() throws IOException, RocksDBException { + OUTPUT_DIRECTORY = temporaryFolder.newFolder("database").toString(); + File file = new File(OUTPUT_DIRECTORY, ACCOUNT); + DbTool.openLevelDb(file.toPath(),ACCOUNT).close(); - file = new File(OUTPUT_DIRECTORY,MARKET); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); + file = new File(OUTPUT_DIRECTORY, DBUtils.MARKET_PAIR_PRICE_TO_ORDER); + DbTool.openLevelDb(file.toPath(), DBUtils.MARKET_PAIR_PRICE_TO_ORDER).close(); - file = new File(OUTPUT_DIRECTORY,ACCOUNT_ROCKSDB); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,ROCKSDB); - - } + file = new File(OUTPUT_DIRECTORY, ACCOUNT_ROCKSDB); + DbTool.openRocksDb(file.toPath(), ACCOUNT_ROCKSDB).close(); - @AfterClass - public static void destroy() { - deleteDir(new File(OUTPUT_DIRECTORY)); } @Test diff --git a/plugins/src/test/java/org/tron/plugins/DbConvertTest.java b/plugins/src/test/java/org/tron/plugins/leveldb/DbConvertTest.java similarity index 76% rename from plugins/src/test/java/org/tron/plugins/DbConvertTest.java rename to plugins/src/test/java/org/tron/plugins/leveldb/DbConvertTest.java index 150e47c9f65..d24604f0a0b 100644 --- a/plugins/src/test/java/org/tron/plugins/DbConvertTest.java +++ b/plugins/src/test/java/org/tron/plugins/leveldb/DbConvertTest.java @@ -1,27 +1,25 @@ -package org.tron.plugins; +package org.tron.plugins.leveldb; import java.io.IOException; import java.util.UUID; import org.junit.Assert; import org.junit.Test; +import org.rocksdb.RocksDBException; +import org.tron.plugins.DbTest; +import org.tron.plugins.Toolkit; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; public class DbConvertTest extends DbTest { @Test - public void testRun() throws IOException { + public void testRun() throws IOException, RocksDBException { + init(DbTool.DbType.LevelDB); String[] args = new String[] { "db", "convert", INPUT_DIRECTORY, temporaryFolder.newFolder().toString() }; Assert.assertEquals(0, cli.execute(args)); } - @Test - public void testRunWithSafe() throws IOException { - String[] args = new String[] { "db", "convert", INPUT_DIRECTORY, - temporaryFolder.newFolder().toString(),"--safe" }; - Assert.assertEquals(0, cli.execute(args)); - } - @Test public void testHelp() { String[] args = new String[] {"db", "convert", "-h"}; diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java b/plugins/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java similarity index 76% rename from plugins/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java rename to plugins/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java index 792f93ad197..7666806e2b5 100644 --- a/plugins/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java +++ b/plugins/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java @@ -1,7 +1,8 @@ -package org.tron.plugins; +package org.tron.plugins.leveldb; import java.io.IOException; import org.junit.Test; +import org.tron.plugins.DbLiteTest; public class DbLiteLevelDbTest extends DbLiteTest { diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java b/plugins/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java similarity index 76% rename from plugins/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java rename to plugins/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java index ae48e1d66e9..de32ae29c7c 100644 --- a/plugins/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java +++ b/plugins/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java @@ -1,7 +1,8 @@ -package org.tron.plugins; +package org.tron.plugins.leveldb; import java.io.IOException; import org.junit.Test; +import org.tron.plugins.DbLiteTest; public class DbLiteLevelDbV2Test extends DbLiteTest { diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java similarity index 76% rename from plugins/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java rename to plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java index e6910b1103a..2f9c92f9679 100644 --- a/plugins/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java +++ b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java @@ -1,7 +1,8 @@ -package org.tron.plugins; +package org.tron.plugins.rocksdb; import java.io.IOException; import org.junit.Test; +import org.tron.plugins.DbLiteTest; public class DbLiteRocksDbTest extends DbLiteTest { diff --git a/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java new file mode 100644 index 00000000000..ab1067fefc3 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java @@ -0,0 +1,13 @@ +package org.tron.plugins.rocksdb; + +import java.io.IOException; +import org.junit.Test; +import org.tron.plugins.DbLiteTest; + +public class DbLiteRocksDbV2Test extends DbLiteTest { + + @Test + public void testToolsWithRocksDB() throws InterruptedException, IOException { + testTools("ROCKSDB", 2); + } +} diff --git a/plugins/src/test/resources/config.conf b/plugins/src/test/resources/config.conf index 2bfca7dbdd7..77d15d521eb 100644 --- a/plugins/src/test/resources/config.conf +++ b/plugins/src/test/resources/config.conf @@ -1,11 +1,5 @@ storage { - # Directory for storing persistent data - db.engine = "LEVELDB", - db.sync = false, - db.directory = "database", - index.directory = "index", - transHistory.switch = "on", properties = [ { name = "account", diff --git a/settings.gradle b/settings.gradle index eb304444378..af32bfca702 100644 --- a/settings.gradle +++ b/settings.gradle @@ -8,4 +8,5 @@ include 'common' include 'example:actuator-example' include 'crypto' include 'plugins' +include 'platform' diff --git a/start.sh.simple b/start.sh.simple new file mode 100644 index 00000000000..72862d9bdfc --- /dev/null +++ b/start.sh.simple @@ -0,0 +1,195 @@ +#!/bin/bash +############################################################################# +# +# GNU LESSER GENERAL PUBLIC LICENSE +# Version 3, 29 June 2007 +# +# Copyright (C) [2007] [TRON Foundation], Inc. +# Everyone is permitted to copy and distribute verbatim copies +# of this license document, but changing it is not allowed. +# +# +# This version of the GNU Lesser General Public License incorporates +# the terms and conditions of version 3 of the GNU General Public +# License, supplemented by the additional permissions listed below. +# +# You can find java-tron at https://github.com/tronprotocol/java-tron/ +# +############################################################################## +# TRON Full Node Management Simple Script +# +# NOTE: This is a simple and concise script to start and stop the java-tron full node, +# designed for developers to quickly get started and learn. +# It may not be suitable for production environments. +# +# Usage: +# sh start.sh # Start the java-tron FullNode +# sh start.sh -s # Stop the java-tron FullNode +# sh start.sh [options] # Start with additional java-tron options,such as: -c config.conf -d /path_to_data, etc. +# +############################################################################## + + +# adjust JVM start +# Set the minimum and maximum heap size to 9G, adjust as needed +VM_XMS="9G" +# Set the maximum heap size to 9G, adjust as needed +VM_XMX="9G" +# Set the maximum direct memory size to 1G, adjust as needed +VM_MAX_DIRECT_MEMORY_SIZE="1G" +# adjust JVM end + +FULL_NODE_JAR="FullNode.jar" +FULL_START_OPT=() +PID="" +MAX_STOP_TIME=60 +JAVACMD="" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log() { + local level="$1"; shift + local timestamp color="" + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + case "$level" in + INFO) color="$GREEN" ;; + WARN) color="$YELLOW" ;; + ERROR) color="$RED" ;; + esac + printf "%b[%s] [%s]:%b %s\n" "$color" "$timestamp" "$level" "$NC" "$*" | tee -a "${SCRIPT_DIR}/start.log" +} + +info() { log INFO "$@"; } +warn() { log WARN "$@"; } +error() { log ERROR "$@"; } +die() { error "$@"; exit 1; } + +ulimit -n 65535 || warn "Failed to set ulimit -n 65535" + +findJava() { + if [ -n "${JAVA_HOME:-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + [ -x "$JAVACMD" ] || die "JAVA_HOME is invalid: $JAVA_HOME" + else + JAVACMD="java" + which java >/dev/null 2>&1 || die "JAVA_HOME not set and no 'java' in PATH" + fi + "$JAVACMD" -version > /dev/null 2>&1 || die "Java command not working" +} + +checkPid() { + # shellcheck disable=SC2009 + PID=$(ps -ef |grep $FULL_NODE_JAR |grep -v grep |awk '{print $2}') +} + + +stopService() { + checkPid + + if ! kill -0 "$PID" 2>/dev/null; then + info "java-tron is not running." + return 0 + fi + info "Stopping java-tron service (PID: $PID)" + + local count=1 + + while [ -n "$PID" ] && [ $count -le $MAX_STOP_TIME ]; do + kill -TERM "$PID" 2>/dev/null && info "Sent SIGTERM to java-tron (PID: $PID), attempt $count" + sleep 1 + checkPid + count=$((count + 1)) + done + + if [ -n "$PID" ]; then + warn "Forcing kill java-tron (PID: $PID) after $MAX_STOP_TIME seconds" + kill -KILL "$PID" 2>/dev/null + sleep 1 + checkPid + fi + + if [ -n "$PID" ]; then + die "Failed to stop the service (PID: $PID)" + else + info "java-tron stopped" + wait_with_info 2 "Cleaning up..." + fi +} + +startService() { + if [ -n "${FULL_START_OPT[*]}" ]; then + info "Starting java-tron service with options: ${FULL_START_OPT[*]}" + fi + if [ ! -f "$FULL_NODE_JAR" ]; then + die "$FULL_NODE_JAR not found in path $SCRIPT_DIR." + fi + + nohup "$JAVACMD" \ + -Xms"$VM_XMS" -Xmx"$VM_XMX" -XX:ReservedCodeCacheSize=256m \ + -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m \ + -XX:MaxDirectMemorySize="$VM_MAX_DIRECT_MEMORY_SIZE" \ + -Xloggc:gc.log -XX:+PrintGCDetails \ + -XX:+UseG1GC \ + -XX:MaxGCPauseMillis=40 \ + -XX:InitiatingHeapOccupancyPercent=45 \ + -XX:+HeapDumpOnOutOfMemoryError \ + -jar "$FULL_NODE_JAR" "${FULL_START_OPT[@]}" \ + >> start.log 2>&1 & + + + info "Waiting for the service to start..." + wait_with_info 5 "Starting..." + + checkPid + + if [ -n "$PID" ]; then + info "Started java-tron with PID $PID on $HOSTNAME." + else + die "Failed to start java-tron, see start.log or logs/tron.log for details." + fi +} + +wait_with_info() { + local seconds=$1 + local message=$2 + for i in $(seq "$seconds" -1 1); do + info "$message wait ($i) s" + sleep 1 + done +} + + +start() { + checkPid + if [ -n "$PID" ]; then + info "java-tron is already running (PID: $PID), to stop the service: sh start.sh -s" + return + fi + findJava + startService +} + +while [ -n "$1" ]; do + case "$1" in + -s) + stopService + exit 0 + ;; + *) + FULL_START_OPT+=("$@") + break + ;; + esac +done + +start + +exit 0 \ No newline at end of file