cmd, triedb: implement history inspection (#29267)

This pull request introduces a database tool for inspecting the state history. 
It can be used for either account history or storage slot history, within a 
specific block range.

The state output format can be chosen either with

- the "rlp-encoded" values (those inserted into the merkle trie)
- the "rlp-decoded" value (the raw state value)

The latter one needs --raw flag.

cmd/geth/dbcmd.go

@@ -33,11 +33,14 @@ import (
 	"github.com/ethereum/go-ethereum/console/prompt"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/triedb"
 	"github.com/olekukonko/tablewriter"
 	"github.com/urfave/cli/v2"
 )
@@ -79,6 +82,7 @@ Remove blockchain and state databases`,
 			dbExportCmd,
 			dbMetadataCmd,
 			dbCheckStateContentCmd,
+			dbInspectHistoryCmd,
 		},
 	}
 	dbInspectCmd = &cli.Command{
@@ -203,6 +207,28 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "Shows metadata about the chain status.",
 	}
+	dbInspectHistoryCmd = &cli.Command{
+		Action:    inspectHistory,
+		Name:      "inspect-history",
+		Usage:     "Inspect the state history within block range",
+		ArgsUsage: "<address> [OPTIONAL <storage-slot>]",
+		Flags: flags.Merge([]cli.Flag{
+			utils.SyncModeFlag,
+			&cli.Uint64Flag{
+				Name:  "start",
+				Usage: "block number of the range start, zero means earliest history",
+			},
+			&cli.Uint64Flag{
+				Name:  "end",
+				Usage: "block number of the range end(included), zero means latest history",
+			},
+			&cli.BoolFlag{
+				Name:  "raw",
+				Usage: "display the decoded raw state value (otherwise shows rlp-encoded value)",
+			},
+		}, utils.NetworkFlags, utils.DatabaseFlags),
+		Description: "This command queries the history of the account or storage slot within the specified block range",
+	}
 )
 
 func removeDB(ctx *cli.Context) error {
@@ -759,3 +785,145 @@ func showMetaData(ctx *cli.Context) error {
 	table.Render()
 	return nil
 }
+
+func inspectAccount(db *triedb.Database, start uint64, end uint64, address common.Address, raw bool) error {
+	stats, err := db.AccountHistory(address, start, end)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("Account history:\n\taddress: %s\n\tblockrange: [#%d-#%d]\n", address.Hex(), stats.Start, stats.End)
+
+	from := stats.Start
+	for i := 0; i < len(stats.Blocks); i++ {
+		var content string
+		if len(stats.Origins[i]) == 0 {
+			content = "<empty>"
+		} else {
+			if !raw {
+				content = fmt.Sprintf("%#x", stats.Origins[i])
+			} else {
+				account := new(types.SlimAccount)
+				if err := rlp.DecodeBytes(stats.Origins[i], account); err != nil {
+					panic(err)
+				}
+				code := "<nil>"
+				if len(account.CodeHash) > 0 {
+					code = fmt.Sprintf("%#x", account.CodeHash)
+				}
+				root := "<nil>"
+				if len(account.Root) > 0 {
+					root = fmt.Sprintf("%#x", account.Root)
+				}
+				content = fmt.Sprintf("nonce: %d, balance: %d, codeHash: %s, root: %s", account.Nonce, account.Balance, code, root)
+			}
+		}
+		fmt.Printf("#%d - #%d: %s\n", from, stats.Blocks[i], content)
+		from = stats.Blocks[i]
+	}
+	return nil
+}
+
+func inspectStorage(db *triedb.Database, start uint64, end uint64, address common.Address, slot common.Hash, raw bool) error {
+	// The hash of storage slot key is utilized in the history
+	// rather than the raw slot key, make the conversion.
+	slotHash := crypto.Keccak256Hash(slot.Bytes())
+	stats, err := db.StorageHistory(address, slotHash, start, end)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("Storage history:\n\taddress: %s\n\tslot: %s\n\tblockrange: [#%d-#%d]\n", address.Hex(), slot.Hex(), stats.Start, stats.End)
+
+	from := stats.Start
+	for i := 0; i < len(stats.Blocks); i++ {
+		var content string
+		if len(stats.Origins[i]) == 0 {
+			content = "<empty>"
+		} else {
+			if !raw {
+				content = fmt.Sprintf("%#x", stats.Origins[i])
+			} else {
+				_, data, _, err := rlp.Split(stats.Origins[i])
+				if err != nil {
+					fmt.Printf("Failed to decode storage slot, %v", err)
+					return err
+				}
+				content = fmt.Sprintf("%#x", data)
+			}
+		}
+		fmt.Printf("#%d - #%d: %s\n", from, stats.Blocks[i], content)
+		from = stats.Blocks[i]
+	}
+	return nil
+}
+
+func inspectHistory(ctx *cli.Context) error {
+	if ctx.NArg() == 0 || ctx.NArg() > 2 {
+		return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
+	}
+	var (
+		address common.Address
+		slot    common.Hash
+	)
+	if err := address.UnmarshalText([]byte(ctx.Args().Get(0))); err != nil {
+		return err
+	}
+	if ctx.NArg() > 1 {
+		if err := slot.UnmarshalText([]byte(ctx.Args().Get(1))); err != nil {
+			return err
+		}
+	}
+	// Load the databases.
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+
+	triedb := utils.MakeTrieDatabase(ctx, db, false, false, false)
+	defer triedb.Close()
+
+	var (
+		err   error
+		start uint64 // the id of first history object to query
+		end   uint64 // the id (included) of last history object to query
+	)
+	// State histories are identified by state ID rather than block number.
+	// To address this, load the corresponding block header and perform the
+	// conversion by this function.
+	blockToID := func(blockNumber uint64) (uint64, error) {
+		header := rawdb.ReadHeader(db, rawdb.ReadCanonicalHash(db, blockNumber), blockNumber)
+		if header == nil {
+			return 0, fmt.Errorf("block #%d is not existent", blockNumber)
+		}
+		id := rawdb.ReadStateID(db, header.Root)
+		if id == nil {
+			first, last, err := triedb.HistoryRange()
+			if err == nil {
+				return 0, fmt.Errorf("history of block #%d is not existent, available history range: [#%d-#%d]", blockNumber, first, last)
+			}
+			return 0, fmt.Errorf("history of block #%d is not existent", blockNumber)
+		}
+		return *id, nil
+	}
+	// Parse the starting block number for inspection.
+	startNumber := ctx.Uint64("start")
+	if startNumber != 0 {
+		start, err = blockToID(startNumber)
+		if err != nil {
+			return err
+		}
+	}
+	// Parse the ending block number for inspection.
+	endBlock := ctx.Uint64("end")
+	if endBlock != 0 {
+		end, err = blockToID(endBlock)
+		if err != nil {
+			return err
+		}
+	}
+	// Inspect the state history.
+	if slot == (common.Hash{}) {
+		return inspectAccount(triedb, start, end, address, ctx.Bool("raw"))
+	}
+	return inspectStorage(triedb, start, end, address, slot, ctx.Bool("raw"))
+}

triedb/history.go

@@ -0,0 +1,72 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package triedb
+
+import (
+	"errors"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/triedb/pathdb"
+)
+
+// AccountHistory inspects the account history within the specified range.
+//
+// Start: State ID of the first history object for the query. 0 implies the first
+// available object is selected as the starting point.
+//
+// End: State ID of the last history for the query. 0 implies the last available
+// object is selected as the starting point. Note end is included for query.
+//
+// This function is only supported by path mode database.
+func (db *Database) AccountHistory(address common.Address, start, end uint64) (*pathdb.HistoryStats, error) {
+	pdb, ok := db.backend.(*pathdb.Database)
+	if !ok {
+		return nil, errors.New("not supported")
+	}
+	return pdb.AccountHistory(address, start, end)
+}
+
+// StorageHistory inspects the storage history within the specified range.
+//
+// Start: State ID of the first history object for the query. 0 implies the first
+// available object is selected as the starting point.
+//
+// End: State ID of the last history for the query. 0 implies the last available
+// object is selected as the starting point. Note end is included for query.
+//
+// Note, slot refers to the hash of the raw slot key.
+//
+// This function is only supported by path mode database.
+func (db *Database) StorageHistory(address common.Address, slot common.Hash, start uint64, end uint64) (*pathdb.HistoryStats, error) {
+	pdb, ok := db.backend.(*pathdb.Database)
+	if !ok {
+		return nil, errors.New("not supported")
+	}
+	return pdb.StorageHistory(address, slot, start, end)
+}
+
+// HistoryRange returns the block numbers associated with earliest and latest
+// state history in the local store.
+//
+// This function is only supported by path mode database.
+func (db *Database) HistoryRange() (uint64, uint64, error) {
+	pdb, ok := db.backend.(*pathdb.Database)
+	if !ok {
+		return 0, 0, errors.New("not supported")
+	}
+	return pdb.HistoryRange()
+}

triedb/pathdb/database.go

@@ -487,3 +487,33 @@ func (db *Database) modifyAllowed() error {
 	}
 	return nil
 }
+
+// AccountHistory inspects the account history within the specified range.
+//
+// Start: State ID of the first history object for the query. 0 implies the first
+// available object is selected as the starting point.
+//
+// End: State ID of the last history for the query. 0 implies the last available
+// object is selected as the ending point. Note end is included in the query.
+func (db *Database) AccountHistory(address common.Address, start, end uint64) (*HistoryStats, error) {
+	return accountHistory(db.freezer, address, start, end)
+}
+
+// StorageHistory inspects the storage history within the specified range.
+//
+// Start: State ID of the first history object for the query. 0 implies the first
+// available object is selected as the starting point.
+//
+// End: State ID of the last history for the query. 0 implies the last available
+// object is selected as the ending point. Note end is included in the query.
+//
+// Note, slot refers to the hash of the raw slot key.
+func (db *Database) StorageHistory(address common.Address, slot common.Hash, start uint64, end uint64) (*HistoryStats, error) {
+	return storageHistory(db.freezer, address, slot, start, end)
+}
+
+// HistoryRange returns the block numbers associated with earliest and latest
+// state history in the local store.
+func (db *Database) HistoryRange() (uint64, uint64, error) {
+	return historyRange(db.freezer)
+}

triedb/pathdb/history_inspect.go

@@ -0,0 +1,151 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/
+
+package pathdb
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// HistoryStats wraps the history inspection statistics.
+type HistoryStats struct {
+	Start   uint64   // Block number of the first queried history
+	End     uint64   // Block number of the last queried history
+	Blocks  []uint64 // Blocks refers to the list of block numbers in which the state is mutated
+	Origins [][]byte // Origins refers to the original value of the state before its mutation
+}
+
+// sanitizeRange limits the given range to fit within the local history store.
+func sanitizeRange(start, end uint64, freezer *rawdb.ResettableFreezer) (uint64, uint64, error) {
+	// Load the id of the first history object in local store.
+	tail, err := freezer.Tail()
+	if err != nil {
+		return 0, 0, err
+	}
+	first := tail + 1
+	if start != 0 && start > first {
+		first = start
+	}
+	// Load the id of the last history object in local store.
+	head, err := freezer.Ancients()
+	if err != nil {
+		return 0, 0, err
+	}
+	last := head - 1
+	if end != 0 && end < last {
+		last = end
+	}
+	// Make sure the range is valid
+	if first >= last {
+		return 0, 0, fmt.Errorf("range is invalid, first: %d, last: %d", first, last)
+	}
+	return first, last, nil
+}
+
+func inspectHistory(freezer *rawdb.ResettableFreezer, start, end uint64, onHistory func(*history, *HistoryStats)) (*HistoryStats, error) {
+	var (
+		stats  = &HistoryStats{}
+		init   = time.Now()
+		logged = time.Now()
+	)
+	start, end, err := sanitizeRange(start, end, freezer)
+	if err != nil {
+		return nil, err
+	}
+	for id := start; id <= end; id += 1 {
+		// The entire history object is decoded, although it's unnecessary for
+		// account inspection. TODO(rjl493456442) optimization is worthwhile.
+		h, err := readHistory(freezer, id)
+		if err != nil {
+			return nil, err
+		}
+		if id == start {
+			stats.Start = h.meta.block
+		}
+		if id == end {
+			stats.End = h.meta.block
+		}
+		onHistory(h, stats)
+
+		if time.Since(logged) > time.Second*8 {
+			logged = time.Now()
+			eta := float64(time.Since(init)) / float64(id-start+1) * float64(end-id)
+			log.Info("Inspecting state history", "checked", id-start+1, "left", end-id, "elapsed", common.PrettyDuration(time.Since(init)), "eta", common.PrettyDuration(eta))
+		}
+	}
+	log.Info("Inspected state history", "total", end-start+1, "elapsed", common.PrettyDuration(time.Since(init)))
+	return stats, nil
+}
+
+// accountHistory inspects the account history within the range.
+func accountHistory(freezer *rawdb.ResettableFreezer, address common.Address, start, end uint64) (*HistoryStats, error) {
+	return inspectHistory(freezer, start, end, func(h *history, stats *HistoryStats) {
+		blob, exists := h.accounts[address]
+		if !exists {
+			return
+		}
+		stats.Blocks = append(stats.Blocks, h.meta.block)
+		stats.Origins = append(stats.Origins, blob)
+	})
+}
+
+// storageHistory inspects the storage history within the range.
+func storageHistory(freezer *rawdb.ResettableFreezer, address common.Address, slot common.Hash, start uint64, end uint64) (*HistoryStats, error) {
+	return inspectHistory(freezer, start, end, func(h *history, stats *HistoryStats) {
+		slots, exists := h.storages[address]
+		if !exists {
+			return
+		}
+		blob, exists := slots[slot]
+		if !exists {
+			return
+		}
+		stats.Blocks = append(stats.Blocks, h.meta.block)
+		stats.Origins = append(stats.Origins, blob)
+	})
+}
+
+// historyRange returns the block number range of local state histories.
+func historyRange(freezer *rawdb.ResettableFreezer) (uint64, uint64, error) {
+	// Load the id of the first history object in local store.
+	tail, err := freezer.Tail()
+	if err != nil {
+		return 0, 0, err
+	}
+	first := tail + 1
+
+	// Load the id of the last history object in local store.
+	head, err := freezer.Ancients()
+	if err != nil {
+		return 0, 0, err
+	}
+	last := head - 1
+
+	fh, err := readHistory(freezer, first)
+	if err != nil {
+		return 0, 0, err
+	}
+	lh, err := readHistory(freezer, last)
+	if err != nil {
+		return 0, 0, err
+	}
+	return fh.meta.block, lh.meta.block, nil
+}