diff --git a/cmd/XDC/dbcmd.go b/cmd/XDC/dbcmd.go index dddd91e7ea32..cec7925ca293 100644 --- a/cmd/XDC/dbcmd.go +++ b/cmd/XDC/dbcmd.go @@ -19,12 +19,14 @@ package main import ( "fmt" "os" + "path/filepath" "time" "github.com/XinFinOrg/XDPoSChain/cmd/utils" "github.com/XinFinOrg/XDPoSChain/common" "github.com/XinFinOrg/XDPoSChain/common/hexutil" "github.com/XinFinOrg/XDPoSChain/console" + "github.com/XinFinOrg/XDPoSChain/core/rawdb" "github.com/XinFinOrg/XDPoSChain/ethdb" "github.com/XinFinOrg/XDPoSChain/log" "github.com/urfave/cli/v2" @@ -38,8 +40,6 @@ var ( ArgsUsage: " ", Flags: []cli.Flag{ utils.DataDirFlag, - utils.XDCXDataDirFlag, - utils.LightModeFlag, }, Description: ` Remove blockchain and state databases`, @@ -49,6 +49,7 @@ Remove blockchain and state databases`, Usage: "Low level database operations", ArgsUsage: "", Subcommands: []*cli.Command{ + dbInspectCmd, dbStatCmd, dbCompactCmd, dbGetCmd, @@ -56,24 +57,61 @@ Remove blockchain and state databases`, dbPutCmd, }, } + dbInspectCmd = &cli.Command{ + Action: inspect, + Name: "inspect", + ArgsUsage: " ", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.TestnetFlag, + utils.DevnetFlag, + }, + Usage: "Inspect the storage size for each type of data in the database", + Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, + } dbStatCmd = &cli.Command{ Action: dbStats, Name: "stats", Usage: "Print leveldb statistics", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.TestnetFlag, + utils.DevnetFlag, + }, } dbCompactCmd = &cli.Command{ Action: dbCompact, Name: "compact", Usage: "Compact leveldb database. WARNING: May take a very long time", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.TestnetFlag, + utils.DevnetFlag, + utils.CacheFlag, + utils.CacheDatabaseFlag, + }, Description: `This command performs a database compaction. WARNING: This operation may take a very long time to finish, and may cause database corruption if it is aborted during execution'!`, } dbGetCmd = &cli.Command{ - Action: dbGet, - Name: "get", - Usage: "Show the value of a database key", - ArgsUsage: "", + Action: dbGet, + Name: "get", + Usage: "Show the value of a database key", + ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.TestnetFlag, + utils.DevnetFlag, + }, Description: "This command looks up the specified database key from the database.", } dbDeleteCmd = &cli.Command{ @@ -81,6 +119,13 @@ corruption if it is aborted during execution'!`, Name: "delete", Usage: "Delete a database key (WARNING: may corrupt your database)", ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.TestnetFlag, + utils.DevnetFlag, + }, Description: `This command deletes the specified database key from the database. WARNING: This is a low-level operation which may cause database corruption!`, } @@ -89,6 +134,13 @@ WARNING: This is a low-level operation which may cause database corruption!`, Name: "put", Usage: "Set the value of a database key (WARNING: may corrupt your database)", ArgsUsage: " ", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.TestnetFlag, + utils.DevnetFlag, + }, Description: `This command sets a given database key to the given value. WARNING: This is a low-level operation which may cause database corruption!`, } @@ -97,33 +149,78 @@ WARNING: This is a low-level operation which may cause database corruption!`, func removeDB(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) for _, name := range []string{"chaindata", "lightchaindata"} { - // Ensure the database exists in the first place - logger := log.New("database", name) - dbdir := stack.ResolvePath(name) - if !common.FileExist(dbdir) { - logger.Info("Database doesn't exist, skipping", "path", dbdir) - continue + if common.FileExist(dbdir) { + confirmAndRemoveDB(dbdir, name) + } else { + log.Info("Database doesn't exist, skipping", "path", dbdir) } - confirmAndRemoveDB(dbdir, name) } return nil } +// removeFolder deletes all files (not folders) inside the directory 'dir' (but +// not files in subfolders). +func removeFolder(dir string) { + filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + // If we're at the top level folder, recurse into + if path == dir { + return nil + } + // Delete all the files, but not subfolders + if !info.IsDir() { + os.Remove(path) + return nil + } + return filepath.SkipDir + }) +} + // confirmAndRemoveDB prompts the user for a last confirmation and removes the // folder if accepted. -func confirmAndRemoveDB(database string, kind string) { - confirm, err := console.Stdin.PromptConfirm(fmt.Sprintf("Remove %s (%s)?", kind, database)) +func confirmAndRemoveDB(path string, kind string) { + confirm, err := console.Stdin.PromptConfirm(fmt.Sprintf("Remove %s (%s)?", kind, path)) switch { case err != nil: utils.Fatalf("%v", err) case !confirm: - log.Warn("Database deletion aborted", "path", database) + log.Warn("Database deletion aborted", "path", path) default: start := time.Now() - os.RemoveAll(database) - log.Info("Database successfully deleted", "path", database, "elapsed", common.PrettyDuration(time.Since(start))) + removeFolder(path) + log.Info("Database successfully deleted", "kind", kind, "path", path, "elapsed", common.PrettyDuration(time.Since(start))) + } +} + +func inspect(ctx *cli.Context) error { + var ( + prefix []byte + start []byte + ) + if ctx.NArg() > 2 { + return fmt.Errorf("max 2 arguments: %v", ctx.Command.ArgsUsage) } + if ctx.NArg() >= 1 { + if d, err := hexutil.Decode(ctx.Args().Get(0)); err != nil { + return fmt.Errorf("failed to hex-decode 'prefix': %v", err) + } else { + prefix = d + } + } + if ctx.NArg() >= 2 { + if d, err := hexutil.Decode(ctx.Args().Get(1)); err != nil { + return fmt.Errorf("failed to hex-decode 'start': %v", err) + } else { + start = d + } + } + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + + return rawdb.InspectDatabase(db, prefix, start) } func showLeveldbStats(db ethdb.Stater) { diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 8b974f5ae443..dabe53a05c79 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -303,6 +303,8 @@ func ExportPreimages(db ethdb.Database, fn string) error { } // Iterate over the preimages and export them it := db.NewIterator([]byte("secure-key-"), nil) + defer it.Release() + for it.Next() { if err := rlp.Encode(writer, it.Value()); err != nil { return err diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 2c3ab34d0d6b..a641f25fc91d 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -148,7 +148,7 @@ func WriteHeadFastBlockHash(db ethdb.KeyValueWriter, hash common.Hash) { // ReadFastTrieProgress retrieves the number of tries nodes fast synced to allow // reportinc correct numbers across restarts. func ReadFastTrieProgress(db ethdb.KeyValueReader) uint64 { - data, _ := db.Get(trieSyncKey) + data, _ := db.Get(fastTrieProgressKey) if len(data) == 0 { return 0 } @@ -158,7 +158,7 @@ func ReadFastTrieProgress(db ethdb.KeyValueReader) uint64 { // WriteFastTrieProgress stores the fast sync trie process counter to support // retrieving it across restarts. func WriteFastTrieProgress(db ethdb.KeyValueWriter, count uint64) error { - if err := db.Put(trieSyncKey, new(big.Int).SetUint64(count).Bytes()); err != nil { + if err := db.Put(fastTrieProgressKey, new(big.Int).SetUint64(count).Bytes()); err != nil { log.Crit("Failed to store fast sync trie progress", "err", err) } return nil diff --git a/core/rawdb/database.go b/core/rawdb/database.go index b75db48ae062..e2376de678f0 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -17,11 +17,17 @@ package rawdb import ( + "bytes" "fmt" + "os" + "time" + "github.com/XinFinOrg/XDPoSChain/common" "github.com/XinFinOrg/XDPoSChain/ethdb" "github.com/XinFinOrg/XDPoSChain/ethdb/leveldb" "github.com/XinFinOrg/XDPoSChain/ethdb/memorydb" + "github.com/XinFinOrg/XDPoSChain/log" + "github.com/olekukonko/tablewriter" ) // freezerdb is a database wrapper that enabled freezer data retrievals. @@ -116,3 +122,138 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r } return NewDatabase(db), nil } + +// InspectDatabase traverses the entire database and checks the size +// of all different categories of data. +func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { + it := db.NewIterator(keyPrefix, keyStart) + defer it.Release() + + var ( + count int64 + start = time.Now() + logged = time.Now() + + // Key-value store statistics + total common.StorageSize + headerSize common.StorageSize + bodySize common.StorageSize + receiptSize common.StorageSize + tdSize common.StorageSize + numHashPairing common.StorageSize + hashNumPairing common.StorageSize + trieSize common.StorageSize + txlookupSize common.StorageSize + preimageSize common.StorageSize + bloomBitsSize common.StorageSize + cliqueSnapsSize common.StorageSize + + // Ancient store statistics + ancientHeaders common.StorageSize + ancientBodies common.StorageSize + ancientReceipts common.StorageSize + ancientHashes common.StorageSize + ancientTds common.StorageSize + + // Les statistic + chtTrieNodes common.StorageSize + bloomTrieNodes common.StorageSize + + // Meta- and unaccounted data + metadata common.StorageSize + unaccounted common.StorageSize + ) + // Inspect key-value database first. + for it.Next() { + var ( + key = it.Key() + size = common.StorageSize(len(key) + len(it.Value())) + ) + total += size + switch { + case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): + tdSize += size + case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): + numHashPairing += size + case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): + headerSize += size + case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): + hashNumPairing += size + case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): + bodySize += size + case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): + receiptSize += size + case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): + txlookupSize += size + case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength): + preimageSize += size + case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): + bloomBitsSize += size + case bytes.HasPrefix(key, []byte("clique-")) && len(key) == 7+common.HashLength: + cliqueSnapsSize += size + case bytes.HasPrefix(key, []byte("cht-")) && len(key) == 4+common.HashLength: + chtTrieNodes += size + case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength: + bloomTrieNodes += size + case len(key) == common.HashLength: + trieSize += size + default: + var accounted bool + for _, meta := range [][]byte{databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey} { + if bytes.Equal(key, meta) { + metadata += size + accounted = true + break + } + } + if !accounted { + unaccounted += size + } + } + count += 1 + if count%1000 == 0 && time.Since(logged) > 8*time.Second { + log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + } + // Inspect append-only file store then. + ancients := []*common.StorageSize{&ancientHeaders, &ancientBodies, &ancientReceipts, &ancientHashes, &ancientTds} + for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} { + if size, err := db.AncientSize(category); err == nil { + *ancients[i] += common.StorageSize(size) + total += common.StorageSize(size) + } + } + // Display the database statistic. + stats := [][]string{ + {"Key-Value store", "Headers", headerSize.String()}, + {"Key-Value store", "Bodies", bodySize.String()}, + {"Key-Value store", "Receipts", receiptSize.String()}, + {"Key-Value store", "Difficulties", tdSize.String()}, + {"Key-Value store", "Block number->hash", numHashPairing.String()}, + {"Key-Value store", "Block hash->number", hashNumPairing.String()}, + {"Key-Value store", "Transaction index", txlookupSize.String()}, + {"Key-Value store", "Bloombit index", bloomBitsSize.String()}, + {"Key-Value store", "Trie nodes", trieSize.String()}, + {"Key-Value store", "Trie preimages", preimageSize.String()}, + {"Key-Value store", "Clique snapshots", cliqueSnapsSize.String()}, + {"Key-Value store", "Singleton metadata", metadata.String()}, + {"Ancient store", "Headers", ancientHeaders.String()}, + {"Ancient store", "Bodies", ancientBodies.String()}, + {"Ancient store", "Receipts", ancientReceipts.String()}, + {"Ancient store", "Difficulties", ancientTds.String()}, + {"Ancient store", "Block number->hash", ancientHashes.String()}, + {"Light client", "CHT trie nodes", chtTrieNodes.String()}, + {"Light client", "Bloom trie nodes", bloomTrieNodes.String()}, + } + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader([]string{"Database", "Category", "Size"}) + table.SetFooter([]string{"", "Total", total.String()}) + table.AppendBulk(stats) + table.Render() + + if unaccounted > 0 { + log.Error("Database contains unaccounted data", "size", unaccounted) + } + return nil +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 18a590615f0d..32e7573976a6 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -38,7 +38,8 @@ var ( // headFastBlockKey tracks the latest known incomplete block's hash during fast sync. headFastBlockKey = []byte("LastFast") - trieSyncKey = []byte("TrieSync") + // fastTrieProgressKey tracks the number of trie entries imported during fast sync. + fastTrieProgressKey = []byte("TrieSync") // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header @@ -80,6 +81,9 @@ const ( // freezerReceiptTable indicates the name of the freezer receipts table. freezerReceiptTable = "receipts" + + // freezerDifficultyTable indicates the name of the freezer total difficulty table. + freezerDifficultyTable = "diffs" ) // LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary