core: separated databases for block data (ethereum#2227)

* core: use finalized block as the chain freeze indicator (ethereum#28683)
* core/rawdb: use max(finality, head-90k) as chain freezing threshold
* core: impl multi database for block data
* core: fix db inspect total size bug
* core: add tips for user who use multi-database
* core: adapt some cmd for multi-database
* core: adapter blockBlobSidecars
* core: fix freezer readHeader bug

---------
Co-authored-by: rjl493456442 <[email protected]>

Author: jingjunLi

cmd/geth/chaincmd.go

@@ -268,12 +268,18 @@ func initGenesis(ctx *cli.Context) error {
 		defer chaindb.Close()
 
 		// if the trie data dir has been set, new trie db with a new state database
-		if ctx.IsSet(utils.SeparateDBFlag.Name) {
+		if ctx.IsSet(utils.MultiDataBaseFlag.Name) {
 			statediskdb, dbErr := stack.OpenDatabaseWithFreezer(name+"/state", 0, 0, "", "", false, false, false, false)
 			if dbErr != nil {
 				utils.Fatalf("Failed to open separate trie database: %v", dbErr)
 			}
 			chaindb.SetStateStore(statediskdb)
+			blockdb, err := stack.OpenDatabaseWithFreezer(name+"/block", 0, 0, "", "", false, false, false, false)
+			if err != nil {
+				utils.Fatalf("Failed to open separate block database: %v", err)
+			}
+			chaindb.SetBlockStore(blockdb)
+			log.Warn("Multi-database is an experimental feature")
 		}
 
 		triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle())
@@ -474,6 +480,13 @@ func dumpGenesis(ctx *cli.Context) error {
 			}
 			continue
 		}
+		// set the separate state & block database
+		if stack.CheckIfMultiDataBase() && err == nil {
+			stateDiskDb := utils.MakeStateDataBase(ctx, stack, true, false)
+			db.SetStateStore(stateDiskDb)
+			blockDb := utils.MakeBlockDatabase(ctx, stack, true, false)
+			db.SetBlockStore(blockDb)
+		}
 		genesis, err := core.ReadGenesis(db)
 		if err != nil {
 			utils.Fatalf("failed to read genesis: %s", err)
@@ -746,7 +759,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
 		arg := ctx.Args().First()
 		if hashish(arg) {
 			hash := common.HexToHash(arg)
-			if number := rawdb.ReadHeaderNumber(db, hash); number != nil {
+			if number := rawdb.ReadHeaderNumber(db.BlockStore(), hash); number != nil {
 				header = rawdb.ReadHeader(db, hash, *number)
 			} else {
 				return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash)
@@ -815,6 +828,7 @@ func dump(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	defer db.Close()
 	triedb := utils.MakeTrieDatabase(ctx, db, true, true, false) // always enable preimage lookup
 	defer triedb.Close()
 

cmd/geth/config.go

@@ -211,9 +211,7 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 	if ctx.IsSet(utils.OverrideDefaultExtraReserveForBlobRequests.Name) {
 		params.DefaultExtraReserveForBlobRequests = ctx.Uint64(utils.OverrideDefaultExtraReserveForBlobRequests.Name)
 	}
-	if ctx.IsSet(utils.SeparateDBFlag.Name) && !stack.IsSeparatedDB() {
-		utils.Fatalf("Failed to locate separate database subdirectory when separatedb parameter has been set")
-	}
+
 	backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
 
 	// Create gauge with geth system and build information

cmd/geth/dbcmd.go

@@ -396,8 +396,8 @@ func inspectTrie(ctx *cli.Context) error {
 	var headerBlockHash common.Hash
 	if ctx.NArg() >= 1 {
 		if ctx.Args().Get(0) == "latest" {
-			headerHash := rawdb.ReadHeadHeaderHash(db)
-			blockNumber = *(rawdb.ReadHeaderNumber(db, headerHash))
+			headerHash := rawdb.ReadHeadHeaderHash(db.BlockStore())
+			blockNumber = *(rawdb.ReadHeaderNumber(db.BlockStore(), headerHash))
 		} else if ctx.Args().Get(0) == "snapshot" {
 			trieRootHash = rawdb.ReadSnapshotRoot(db)
 			blockNumber = math.MaxUint64
@@ -519,14 +519,19 @@ func checkStateContent(ctx *cli.Context) error {
 	db := utils.MakeChainDatabase(ctx, stack, true, false)
 	defer db.Close()
 	var (
-		it        = rawdb.NewKeyLengthIterator(db.NewIterator(prefix, start), 32)
+		it        ethdb.Iterator
 		hasher    = crypto.NewKeccakState()
 		got       = make([]byte, 32)
 		errs      int
 		count     int
 		startTime = time.Now()
 		lastLog   = time.Now()
 	)
+	if stack.CheckIfMultiDataBase() {
+		it = rawdb.NewKeyLengthIterator(db.StateStore().NewIterator(prefix, start), 32)
+	} else {
+		it = rawdb.NewKeyLengthIterator(db.NewIterator(prefix, start), 32)
+	}
 	for it.Next() {
 		count++
 		k := it.Key()
@@ -573,9 +578,11 @@ func dbStats(ctx *cli.Context) error {
 	defer db.Close()
 
 	showLeveldbStats(db)
-	if db.StateStore() != nil {
+	if stack.CheckIfMultiDataBase() {
 		fmt.Println("show stats of state store")
 		showLeveldbStats(db.StateStore())
+		fmt.Println("show stats of block store")
+		showLeveldbStats(db.BlockStore())
 	}
 
 	return nil
@@ -591,10 +598,11 @@ func dbCompact(ctx *cli.Context) error {
 	log.Info("Stats before compaction")
 	showLeveldbStats(db)
 
-	statediskdb := db.StateStore()
-	if statediskdb != nil {
+	if stack.CheckIfMultiDataBase() {
 		fmt.Println("show stats of state store")
-		showLeveldbStats(statediskdb)
+		showLeveldbStats(db.StateStore())
+		fmt.Println("show stats of block store")
+		showLeveldbStats(db.BlockStore())
 	}
 
 	log.Info("Triggering compaction")
@@ -603,18 +611,24 @@ func dbCompact(ctx *cli.Context) error {
 		return err
 	}
 
-	if statediskdb != nil {
-		if err := statediskdb.Compact(nil, nil); err != nil {
+	if stack.CheckIfMultiDataBase() {
+		if err := db.StateStore().Compact(nil, nil); err != nil {
+			log.Error("Compact err", "error", err)
+			return err
+		}
+		if err := db.BlockStore().Compact(nil, nil); err != nil {
 			log.Error("Compact err", "error", err)
 			return err
 		}
 	}
 
 	log.Info("Stats after compaction")
 	showLeveldbStats(db)
-	if statediskdb != nil {
+	if stack.CheckIfMultiDataBase() {
 		fmt.Println("show stats of state store after compaction")
-		showLeveldbStats(statediskdb)
+		showLeveldbStats(db.StateStore())
+		fmt.Println("show stats of block store after compaction")
+		showLeveldbStats(db.BlockStore())
 	}
 	return nil
 }
@@ -635,18 +649,18 @@ func dbGet(ctx *cli.Context) error {
 		log.Info("Could not decode the key", "error", err)
 		return err
 	}
+	opDb := db
+	if stack.CheckIfMultiDataBase() {
+		keyType := rawdb.DataTypeByKey(key)
+		if keyType == rawdb.StateDataType {
+			opDb = db.StateStore()
+		} else if keyType == rawdb.BlockDataType {
+			opDb = db.BlockStore()
+		}
+	}
 
-	statediskdb := db.StateStore()
-	data, err := db.Get(key)
+	data, err := opDb.Get(key)
 	if err != nil {
-		// if separate trie db exist, try to get it from separate db
-		if statediskdb != nil {
-			statedata, dberr := statediskdb.Get(key)
-			if dberr == nil {
-				fmt.Printf("key %#x: %#x\n", key, statedata)
-				return nil
-			}
-		}
 		log.Info("Get operation failed", "key", fmt.Sprintf("%#x", key), "error", err)
 		return err
 	}
@@ -809,11 +823,21 @@ func dbDelete(ctx *cli.Context) error {
 		log.Info("Could not decode the key", "error", err)
 		return err
 	}
-	data, err := db.Get(key)
+	opDb := db
+	if stack.CheckIfMultiDataBase() {
+		keyType := rawdb.DataTypeByKey(key)
+		if keyType == rawdb.StateDataType {
+			opDb = db.StateStore()
+		} else if keyType == rawdb.BlockDataType {
+			opDb = db.BlockStore()
+		}
+	}
+
+	data, err := opDb.Get(key)
 	if err == nil {
 		fmt.Printf("Previous value: %#x\n", data)
 	}
-	if err = db.Delete(key); err != nil {
+	if err = opDb.Delete(key); err != nil {
 		log.Info("Delete operation returned an error", "key", fmt.Sprintf("%#x", key), "error", err)
 		return err
 	}
@@ -923,11 +947,22 @@ func dbPut(ctx *cli.Context) error {
 		log.Info("Could not decode the value", "error", err)
 		return err
 	}
-	data, err = db.Get(key)
+
+	opDb := db
+	if stack.CheckIfMultiDataBase() {
+		keyType := rawdb.DataTypeByKey(key)
+		if keyType == rawdb.StateDataType {
+			opDb = db.StateStore()
+		} else if keyType == rawdb.BlockDataType {
+			opDb = db.BlockStore()
+		}
+	}
+
+	data, err = opDb.Get(key)
 	if err == nil {
 		fmt.Printf("Previous value: %#x\n", data)
 	}
-	return db.Put(key, value)
+	return opDb.Put(key, value)
 }
 
 // dbDumpTrie shows the key-value slots of a given storage trie
@@ -940,7 +975,6 @@ func dbDumpTrie(ctx *cli.Context) error {
 
 	db := utils.MakeChainDatabase(ctx, stack, true, false)
 	defer db.Close()
-
 	triedb := utils.MakeTrieDatabase(ctx, db, false, true, false)
 	defer triedb.Close()
 
@@ -1019,7 +1053,7 @@ func freezerInspect(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	ancient := stack.ResolveAncient("chaindata", ctx.String(utils.AncientFlag.Name))
 	stack.Close()
-	return rawdb.InspectFreezerTable(ancient, freezer, table, start, end)
+	return rawdb.InspectFreezerTable(ancient, freezer, table, start, end, stack.CheckIfMultiDataBase())
 }
 
 func importLDBdata(ctx *cli.Context) error {
@@ -1159,11 +1193,11 @@ func showMetaData(ctx *cli.Context) error {
 	defer stack.Close()
 	db := utils.MakeChainDatabase(ctx, stack, true, false)
 	defer db.Close()
-	ancients, err := db.Ancients()
+	ancients, err := db.BlockStore().Ancients()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error accessing ancients: %v", err)
 	}
-	data := rawdb.ReadChainMetadata(db)
+	data := rawdb.ReadChainMetadataFromMultiDatabase(db)
 	data = append(data, []string{"frozen", fmt.Sprintf("%d items", ancients)})
 	data = append(data, []string{"snapshotGenerator", snapshot.ParseGeneratorStatus(rawdb.ReadSnapshotGenerator(db))})
 	if b := rawdb.ReadHeadBlock(db); b != nil {
@@ -1224,8 +1258,8 @@ func hbss2pbss(ctx *cli.Context) error {
 		log.Info("hbss2pbss triedb", "scheme", triedb.Scheme())
 		defer triedb.Close()
 
-		headerHash := rawdb.ReadHeadHeaderHash(db)
-		blockNumber := rawdb.ReadHeaderNumber(db, headerHash)
+		headerHash := rawdb.ReadHeadHeaderHash(db.BlockStore())
+		blockNumber := rawdb.ReadHeaderNumber(db.BlockStore(), headerHash)
 		if blockNumber == nil {
 			log.Error("read header number failed.")
 			return fmt.Errorf("read header number failed")

cmd/geth/pruneblock_test.go

@@ -178,10 +178,11 @@ func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemai
 
 	// Force run a freeze cycle
 	type freezer interface {
-		Freeze(threshold uint64) error
+		Freeze() error
 		Ancients() (uint64, error)
 	}
-	db.(freezer).Freeze(10)
+	blockchain.SetFinalized(blocks[len(blocks)-1].Header())
+	db.(freezer).Freeze()
 
 	frozen, err := db.Ancients()
 	//make sure there're frozen items

cmd/geth/snapshot.go

@@ -355,7 +355,15 @@ func pruneBlock(ctx *cli.Context) error {
 	if !ctx.IsSet(utils.AncientFlag.Name) {
 		return errors.New("datadir.ancient must be set")
 	} else {
-		oldAncientPath = ctx.String(utils.AncientFlag.Name)
+		if stack.CheckIfMultiDataBase() {
+			ancientPath := ctx.String(utils.AncientFlag.Name)
+			index := strings.LastIndex(ancientPath, "/ancient/chain")
+			if index != -1 {
+				oldAncientPath = ancientPath[:index] + "/block/ancient/chain"
+			}
+		} else {
+			oldAncientPath = ctx.String(utils.AncientFlag.Name)
+		}
 		if !filepath.IsAbs(oldAncientPath) {
 			// force absolute paths, which often fail due to the splicing of relative paths
 			return errors.New("datadir.ancient not abs path")
@@ -852,6 +860,7 @@ func dumpState(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	defer db.Close()
 	triedb := utils.MakeTrieDatabase(ctx, db, false, true, false)
 	defer triedb.Close()
 

cmd/utils/flags.go

@@ -93,10 +93,10 @@ var (
 		Value:    flags.DirectoryString(node.DefaultDataDir()),
 		Category: flags.EthCategory,
 	}
-	SeparateDBFlag = &cli.BoolFlag{
-		Name: "separatedb",
-		Usage: "Enable a separated trie database, it will be created within a subdirectory called state, " +
-			"Users can copy this state directory to another directory or disk, and then create a symbolic link to the state directory under the chaindata",
+	MultiDataBaseFlag = &cli.BoolFlag{
+		Name: "multidatabase",
+		Usage: "Enable a separated state and block database, it will be created within two subdirectory called state and block, " +
+			"Users can copy this state or block directory to another directory or disk, and then create a symbolic link to the state directory under the chaindata",
 		Category: flags.EthCategory,
 	}
 	DirectBroadcastFlag = &cli.BoolFlag{
@@ -1150,7 +1150,7 @@ var (
 		DBEngineFlag,
 		StateSchemeFlag,
 		HttpHeaderFlag,
-		SeparateDBFlag,
+		MultiDataBaseFlag,
 	}
 )
 
@@ -2387,9 +2387,11 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly, disableFree
 	default:
 		chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "", readonly, disableFreeze, false, false)
 		// set the separate state database
-		if stack.IsSeparatedDB() && err == nil {
+		if stack.CheckIfMultiDataBase() && err == nil {
 			stateDiskDb := MakeStateDataBase(ctx, stack, readonly, false)
 			chainDb.SetStateStore(stateDiskDb)
+			blockDb := MakeBlockDatabase(ctx, stack, readonly, false)
+			chainDb.SetBlockStore(blockDb)
 		}
 	}
 	if err != nil {
@@ -2401,14 +2403,25 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly, disableFree
 // MakeStateDataBase open a separate state database using the flags passed to the client and will hard crash if it fails.
 func MakeStateDataBase(ctx *cli.Context, stack *node.Node, readonly, disableFreeze bool) ethdb.Database {
 	cache := ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100
-	handles := MakeDatabaseHandles(ctx.Int(FDLimitFlag.Name)) / 2
+	handles := MakeDatabaseHandles(ctx.Int(FDLimitFlag.Name)) * 90 / 100
 	statediskdb, err := stack.OpenDatabaseWithFreezer("chaindata/state", cache, handles, "", "", readonly, disableFreeze, false, false)
 	if err != nil {
 		Fatalf("Failed to open separate trie database: %v", err)
 	}
 	return statediskdb
 }
 
+// MakeBlockDatabase open a separate block database using the flags passed to the client and will hard crash if it fails.
+func MakeBlockDatabase(ctx *cli.Context, stack *node.Node, readonly, disableFreeze bool) ethdb.Database {
+	cache := ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100
+	handles := MakeDatabaseHandles(ctx.Int(FDLimitFlag.Name)) / 10
+	blockDb, err := stack.OpenDatabaseWithFreezer("chaindata/block", cache, handles, "", "", readonly, disableFreeze, false, false)
+	if err != nil {
+		Fatalf("Failed to open separate block database: %v", err)
+	}
+	return blockDb
+}
+
 // tryMakeReadOnlyDatabase try to open the chain database in read-only mode,
 // or fallback to write mode if the database is not initialized.
 //