[WIP] all: implement chain freezer

XDCxDAO/interfaces.go

@@ -42,9 +42,16 @@ type XDCXDAO interface {
 	HasAncient(kind string, number uint64) (bool, error)
 	Ancient(kind string, number uint64) ([]byte, error)
 	Ancients() (uint64, error)
+	Tail() (uint64, error)
 	AncientSize(kind string) (uint64, error)
 	AppendAncient(number uint64, hash, header, body, receipt, td []byte) error
-	TruncateAncients(n uint64) error
+	TruncateHead(n uint64) error
+	TruncateTail(n uint64) error
+	AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error)
+	ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error)
+	MigrateTable(string, func([]byte) ([]byte, error)) error
+	AncientDatadir() (string, error)
+	ModifyAncients(fn func(ethdb.AncientWriteOp) error) (writeSize int64, err error)
 	Sync() error
 	NewIterator(prefix []byte, start []byte) ethdb.Iterator
 

XDCxDAO/leveldb.go

@@ -145,6 +145,11 @@ func (db *BatchDatabase) Ancients() (uint64, error) {
 	return 0, errNotSupported
 }
 
+// Tail returns an error as we don't have a backing chain freezer.
+func (db *BatchDatabase) Tail() (uint64, error) {
+	return 0, errNotSupported
+}
+
 // AncientSize returns an error as we don't have a backing chain freezer.
 func (db *BatchDatabase) AncientSize(kind string) (uint64, error) {
 	return 0, errNotSupported
@@ -155,11 +160,39 @@ func (db *BatchDatabase) AppendAncient(number uint64, hash, header, body, receip
 	return errNotSupported
 }
 
-// TruncateAncients returns an error as we don't have a backing chain freezer.
-func (db *BatchDatabase) TruncateAncients(items uint64) error {
+// TruncateHead returns an error as we don't have a backing chain freezer.
+func (db *BatchDatabase) TruncateHead(items uint64) error {
+	return errNotSupported
+}
+
+// TruncateTail returns an error as we don't have a backing chain freezer.
+func (db *BatchDatabase) TruncateTail(items uint64) error {
+	return errNotSupported
+}
+
+func (db *BatchDatabase) AncientRange(kind string, start, max, maxByteSize uint64) ([][]byte, error) {
+	return nil, errNotSupported
+}
+
+func (db *BatchDatabase) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {
+	return fn(db)
+}
+
+// MigrateTable processes the entries in a given table in sequence
+// converting them to a new format if they're of an old format.
+func (db *BatchDatabase) MigrateTable(kind string, convert func([]byte) ([]byte, error)) error {
 	return errNotSupported
 }
 
+// AncientDatadir returns an error as we don't have a backing chain freezer.
+func (db *BatchDatabase) AncientDatadir() (string, error) {
+	return "", errNotSupported
+}
+
+func (db *BatchDatabase) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (writeSize int64, err error) {
+	return 0, errNotSupported
+}
+
 // Sync returns an error as we don't have a backing chain freezer.
 func (db *BatchDatabase) Sync() error {
 	return errNotSupported

XDCxDAO/mongodb.go

@@ -845,6 +845,11 @@ func (db *MongoDatabase) Ancients() (uint64, error) {
 	return 0, errNotSupported
 }
 
+// Tail returns an error as we don't have a backing chain freezer.
+func (db *MongoDatabase) Tail() (uint64, error) {
+	return 0, errNotSupported
+}
+
 // AncientSize returns an error as we don't have a backing chain freezer.
 func (db *MongoDatabase) AncientSize(kind string) (uint64, error) {
 	return 0, errNotSupported
@@ -855,11 +860,39 @@ func (db *MongoDatabase) AppendAncient(number uint64, hash, header, body, receip
 	return errNotSupported
 }
 
-// TruncateAncients returns an error as we don't have a backing chain freezer.
-func (db *MongoDatabase) TruncateAncients(items uint64) error {
+// TruncateHead returns an error as we don't have a backing chain freezer.
+func (db *MongoDatabase) TruncateHead(items uint64) error {
+	return errNotSupported
+}
+
+// TruncateTail returns an error as we don't have a backing chain freezer.
+func (db *MongoDatabase) TruncateTail(items uint64) error {
+	return errNotSupported
+}
+
+func (db *MongoDatabase) AncientRange(kind string, start, max, maxByteSize uint64) ([][]byte, error) {
+	return nil, errNotSupported
+}
+
+func (db *MongoDatabase) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {
+	return fn(db)
+}
+
+// MigrateTable processes the entries in a given table in sequence
+// converting them to a new format if they're of an old format.
+func (db *MongoDatabase) MigrateTable(kind string, convert func([]byte) ([]byte, error)) error {
 	return errNotSupported
 }
 
+// AncientDatadir returns an error as we don't have a backing chain freezer.
+func (db *MongoDatabase) AncientDatadir() (string, error) {
+	return "", errNotSupported
+}
+
+func (db *MongoDatabase) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (writeSize int64, err error) {
+	return 0, errNotSupported
+}
+
 // Sync returns an error as we don't have a backing chain freezer.
 func (db *MongoDatabase) Sync() error {
 	return errNotSupported

accounts/abi/bind/backends/simulated.go

@@ -138,7 +138,7 @@ func NewXDCSimulatedBackend(alloc types.GenesisAlloc, gasLimit uint64, chainConf
 		return lendingServ
 	}
 
-	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, consensus, vm.Config{})
+	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, consensus, vm.Config{}, nil)
 
 	backend := &SimulatedBackend{
 		database:   database,
@@ -162,7 +162,7 @@ func NewSimulatedBackend(alloc types.GenesisAlloc, gasLimit uint64) *SimulatedBa
 	database := rawdb.NewMemoryDatabase()
 	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
 	genesis.MustCommit(database)
-	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil)
 
 	backend := &SimulatedBackend{
 		database:   database,

cmd/XDC/chaincmd.go

@@ -29,6 +29,7 @@ import (
 	"github.com/XinFinOrg/XDPoSChain/cmd/utils"
 	"github.com/XinFinOrg/XDPoSChain/common"
 	"github.com/XinFinOrg/XDPoSChain/core"
+	"github.com/XinFinOrg/XDPoSChain/core/rawdb"
 	"github.com/XinFinOrg/XDPoSChain/core/state"
 	"github.com/XinFinOrg/XDPoSChain/core/types"
 	xdc_genesis "github.com/XinFinOrg/XDPoSChain/genesis"
@@ -73,6 +74,7 @@ It expects the genesis file or the network name [ mainnet | testnet | devnet ] a
 			utils.MetricsInfluxDBTokenFlag,
 			utils.MetricsInfluxDBBucketFlag,
 			utils.MetricsInfluxDBOrganizationFlag,
+			utils.TxLookupLimitFlag,
 		}, utils.DatabaseFlags),
 		Description: `
 The import command imports blocks from an RLP-encoded form. The form can be one file
@@ -201,7 +203,7 @@ func importChain(ctx *cli.Context) error {
 	// Start metrics export if enabled
 	utils.SetupMetrics(&cfg.Metrics)
 
-	chain, db := utils.MakeChain(ctx, stack, false)
+	chain, db := utils.MakeChain(ctx, stack)
 	defer db.Close()
 
 	// Start periodically gathering memory profiles
@@ -276,7 +278,7 @@ func exportChain(ctx *cli.Context) error {
 	stack, _, _ := makeFullNode(ctx)
 	defer stack.Close()
 
-	chain, db := utils.MakeChain(ctx, stack, true)
+	chain, db := utils.MakeChain(ctx, stack)
 	defer db.Close()
 	start := time.Now()
 
@@ -347,22 +349,27 @@ func dump(ctx *cli.Context) error {
 	stack, _, _ := makeFullNode(ctx)
 	defer stack.Close()
 
-	chain, chainDb := utils.MakeChain(ctx, stack, true)
-	defer chainDb.Close()
-
+	db := utils.MakeChainDatabase(ctx, stack, true)
 	for _, arg := range ctx.Args().Slice() {
-		var block *types.Block
+		var header *types.Header
 		if hashish(arg) {
-			block = chain.GetBlockByHash(common.HexToHash(arg))
+			hash := common.HexToHash(arg)
+			number := rawdb.ReadHeaderNumber(db, hash)
+			if number != nil {
+				header = rawdb.ReadHeader(db, hash, *number)
+			}
 		} else {
-			num, _ := strconv.Atoi(arg)
-			block = chain.GetBlockByNumber(uint64(num))
+			number, _ := strconv.Atoi(arg)
+			hash := rawdb.ReadCanonicalHash(db, uint64(number))
+			if hash != (common.Hash{}) {
+				header = rawdb.ReadHeader(db, hash, uint64(number))
+			}
 		}
-		if block == nil {
+		if header == nil {
 			fmt.Println("{}")
 			utils.Fatalf("block not found")
 		} else {
-			state, err := state.New(block.Root(), state.NewDatabase(chainDb))
+			state, err := state.New(header.Root, state.NewDatabase(db))
 			if err != nil {
 				utils.Fatalf("could not create new state: %v", err)
 			}

cmd/XDC/config.go

@@ -243,6 +243,18 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend, XDCConfig) {
 		})
 	}
 
+	// Warn users to migrate if they have a legacy freezer format.
+	if eth != nil {
+		firstIdx := uint64(0)
+		isLegacy, _, err := dbHasLegacyReceipts(eth.ChainDb(), firstIdx)
+		if err != nil {
+			utils.Fatalf("Failed to check db for legacy receipts: %v", err)
+		}
+		if isLegacy {
+			log.Warn("Database has receipts with a legacy format. Please run `XDC db freezer-migrate`.")
+		}
+	}
+
 	// Add the Ethereum Stats daemon if requested.
 	if cfg.Ethstats.URL != "" {
 		utils.RegisterEthStatsService(stack, backend, cfg.Ethstats.URL)