core/state, cmd/geth: streaming json output for dump command (#15475)
* core/state, cmd/geth: streaming json output dump cmd + optional code+storage * dump: add option to continue even if preimages are missing * core, evm: lint nits * cmd: use local flags for dump, omit empty code/storage * core/state: fix state dump test
This commit is contained in:
Martin Holst Swende
Péter Szilágyi
parent
e4a1488b2f
commit
1da5e0ebb0
|
|
@ -209,7 +209,7 @@ func runCmd(ctx *cli.Context) error {
|
|||
if ctx.GlobalBool(DumpFlag.Name) {
|
||||
statedb.Commit(true)
|
||||
statedb.IntermediateRoot(true)
|
||||
fmt.Println(string(statedb.Dump()))
|
||||
fmt.Println(string(statedb.Dump(false, false, true)))
|
||||
}
|
||||
|
||||
if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" {
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ func stateTestCmd(ctx *cli.Context) error {
|
|||
// Test failed, mark as so and dump any state to aid debugging
|
||||
result.Pass, result.Error = false, err.Error()
|
||||
if ctx.GlobalBool(DumpFlag.Name) && state != nil {
|
||||
dump := state.RawDump()
|
||||
dump := state.RawDump(false, false, true)
|
||||
result.State = &dump
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -162,6 +162,10 @@ Remove blockchain and state databases`,
|
|||
utils.DataDirFlag,
|
||||
utils.CacheFlag,
|
||||
utils.SyncModeFlag,
|
||||
utils.IterativeOutputFlag,
|
||||
utils.ExcludeCodeFlag,
|
||||
utils.ExcludeStorageFlag,
|
||||
utils.IncludeIncompletesFlag,
|
||||
},
|
||||
Category: "BLOCKCHAIN COMMANDS",
|
||||
Description: `
|
||||
|
|
@ -287,7 +291,7 @@ func importChain(ctx *cli.Context) error {
|
|||
fmt.Printf("Allocations: %.3f million\n", float64(mem.Mallocs)/1000000)
|
||||
fmt.Printf("GC pause: %v\n\n", time.Duration(mem.PauseTotalNs))
|
||||
|
||||
if ctx.GlobalIsSet(utils.NoCompactionFlag.Name) {
|
||||
if ctx.GlobalBool(utils.NoCompactionFlag.Name) {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -504,6 +508,7 @@ func dump(ctx *cli.Context) error {
|
|||
defer stack.Close()
|
||||
|
||||
chain, chainDb := utils.MakeChain(ctx, stack)
|
||||
defer chainDb.Close()
|
||||
for _, arg := range ctx.Args() {
|
||||
var block *types.Block
|
||||
if hashish(arg) {
|
||||
|
|
@ -520,10 +525,20 @@ func dump(ctx *cli.Context) error {
|
|||
if err != nil {
|
||||
utils.Fatalf("could not create new state: %v", err)
|
||||
}
|
||||
fmt.Printf("%s\n", state.Dump())
|
||||
excludeCode := ctx.Bool(utils.ExcludeCodeFlag.Name)
|
||||
excludeStorage := ctx.Bool(utils.ExcludeStorageFlag.Name)
|
||||
includeMissing := ctx.Bool(utils.IncludeIncompletesFlag.Name)
|
||||
if ctx.Bool(utils.IterativeOutputFlag.Name) {
|
||||
state.IterativeDump(excludeCode, excludeStorage, !includeMissing, json.NewEncoder(os.Stdout))
|
||||
} else {
|
||||
if includeMissing {
|
||||
fmt.Printf("If you want to include accounts with missing preimages, you need iterative output, since" +
|
||||
" otherwise the accounts will overwrite each other in the resulting mapping.")
|
||||
}
|
||||
fmt.Printf("%v %s\n", includeMissing, state.Dump(excludeCode, excludeStorage, false))
|
||||
}
|
||||
}
|
||||
}
|
||||
chainDb.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -196,6 +196,22 @@ var (
|
|||
Name: "ulc.trusted",
|
||||
Usage: "List of trusted ULC servers",
|
||||
}
|
||||
IterativeOutputFlag = cli.BoolFlag{
|
||||
Name: "iterative",
|
||||
Usage: "Print streaming JSON iteratively, delimited by newlines",
|
||||
}
|
||||
ExcludeStorageFlag = cli.BoolFlag{
|
||||
Name: "nostorage",
|
||||
Usage: "Exclude storage entries (save db lookups)",
|
||||
}
|
||||
IncludeIncompletesFlag = cli.BoolFlag{
|
||||
Name: "incompletes",
|
||||
Usage: "Include accounts for which we don't have the address (missing preimage)",
|
||||
}
|
||||
ExcludeCodeFlag = cli.BoolFlag{
|
||||
Name: "nocode",
|
||||
Usage: "Exclude contract code (save db lookups)",
|
||||
}
|
||||
defaultSyncMode = eth.DefaultConfig.SyncMode
|
||||
SyncModeFlag = TextMarshalerFlag{
|
||||
Name: "syncmode",
|
||||
|
|
|
|||
|
|
@ -21,61 +21,133 @@ import (
|
|||
"fmt"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/common/hexutil"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
)
|
||||
|
||||
// DumpAccount represents an account in the state
|
||||
type DumpAccount struct {
|
||||
Balance string `json:"balance"`
|
||||
Nonce uint64 `json:"nonce"`
|
||||
Root string `json:"root"`
|
||||
CodeHash string `json:"codeHash"`
|
||||
Code string `json:"code"`
|
||||
Storage map[string]string `json:"storage"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Storage map[common.Hash]string `json:"storage,omitempty"`
|
||||
Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode
|
||||
SecureKey hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key
|
||||
|
||||
}
|
||||
|
||||
// Dump represents the full dump in a collected format, as one large map
|
||||
type Dump struct {
|
||||
Root string `json:"root"`
|
||||
Accounts map[string]DumpAccount `json:"accounts"`
|
||||
Accounts map[common.Address]DumpAccount `json:"accounts"`
|
||||
}
|
||||
|
||||
func (self *StateDB) RawDump() Dump {
|
||||
dump := Dump{
|
||||
Root: fmt.Sprintf("%x", self.trie.Hash()),
|
||||
Accounts: make(map[string]DumpAccount),
|
||||
}
|
||||
// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively
|
||||
type iterativeDump json.Encoder
|
||||
|
||||
// Collector interface which the state trie calls during iteration
|
||||
type collector interface {
|
||||
onRoot(common.Hash)
|
||||
onAccount(common.Address, DumpAccount)
|
||||
}
|
||||
|
||||
func (self *Dump) onRoot(root common.Hash) {
|
||||
self.Root = fmt.Sprintf("%x", root)
|
||||
}
|
||||
|
||||
func (self *Dump) onAccount(addr common.Address, account DumpAccount) {
|
||||
self.Accounts[addr] = account
|
||||
}
|
||||
|
||||
func (self iterativeDump) onAccount(addr common.Address, account DumpAccount) {
|
||||
dumpAccount := &DumpAccount{
|
||||
Balance: account.Balance,
|
||||
Nonce: account.Nonce,
|
||||
Root: account.Root,
|
||||
CodeHash: account.CodeHash,
|
||||
Code: account.Code,
|
||||
Storage: account.Storage,
|
||||
SecureKey: account.SecureKey,
|
||||
Address: nil,
|
||||
}
|
||||
if addr != (common.Address{}) {
|
||||
dumpAccount.Address = &addr
|
||||
}
|
||||
(*json.Encoder)(&self).Encode(dumpAccount)
|
||||
}
|
||||
func (self iterativeDump) onRoot(root common.Hash) {
|
||||
(*json.Encoder)(&self).Encode(struct {
|
||||
Root common.Hash `json:"root"`
|
||||
}{root})
|
||||
}
|
||||
|
||||
func (self *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) {
|
||||
emptyAddress := (common.Address{})
|
||||
missingPreimages := 0
|
||||
c.onRoot(self.trie.Hash())
|
||||
it := trie.NewIterator(self.trie.NodeIterator(nil))
|
||||
for it.Next() {
|
||||
addr := self.trie.GetKey(it.Key)
|
||||
var data Account
|
||||
if err := rlp.DecodeBytes(it.Value, &data); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
obj := newObject(nil, common.BytesToAddress(addr), data)
|
||||
addr := common.BytesToAddress(self.trie.GetKey(it.Key))
|
||||
obj := newObject(nil, addr, data)
|
||||
account := DumpAccount{
|
||||
Balance: data.Balance.String(),
|
||||
Nonce: data.Nonce,
|
||||
Root: common.Bytes2Hex(data.Root[:]),
|
||||
CodeHash: common.Bytes2Hex(data.CodeHash),
|
||||
Code: common.Bytes2Hex(obj.Code(self.db)),
|
||||
Storage: make(map[string]string),
|
||||
}
|
||||
if emptyAddress == addr {
|
||||
// Preimage missing
|
||||
missingPreimages++
|
||||
if excludeMissingPreimages {
|
||||
continue
|
||||
}
|
||||
account.SecureKey = it.Key
|
||||
}
|
||||
if !excludeCode {
|
||||
account.Code = common.Bytes2Hex(obj.Code(self.db))
|
||||
}
|
||||
if !excludeStorage {
|
||||
account.Storage = make(map[common.Hash]string)
|
||||
storageIt := trie.NewIterator(obj.getTrie(self.db).NodeIterator(nil))
|
||||
for storageIt.Next() {
|
||||
account.Storage[common.Bytes2Hex(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
|
||||
account.Storage[common.BytesToHash(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
|
||||
}
|
||||
dump.Accounts[common.Bytes2Hex(addr)] = account
|
||||
}
|
||||
return dump
|
||||
c.onAccount(addr, account)
|
||||
}
|
||||
if missingPreimages > 0 {
|
||||
log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
|
||||
}
|
||||
}
|
||||
|
||||
func (self *StateDB) Dump() []byte {
|
||||
json, err := json.MarshalIndent(self.RawDump(), "", " ")
|
||||
// RawDump returns the entire state an a single large object
|
||||
func (self *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages bool) Dump {
|
||||
dump := &Dump{
|
||||
Accounts: make(map[common.Address]DumpAccount),
|
||||
}
|
||||
self.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages)
|
||||
return *dump
|
||||
}
|
||||
|
||||
// Dump returns a JSON string representing the entire state as a single json-object
|
||||
func (self *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool) []byte {
|
||||
dump := self.RawDump(excludeCode, excludeStorage, excludeMissingPreimages)
|
||||
json, err := json.MarshalIndent(dump, "", " ")
|
||||
if err != nil {
|
||||
fmt.Println("dump err", err)
|
||||
}
|
||||
|
||||
return json
|
||||
}
|
||||
|
||||
// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
|
||||
func (self *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) {
|
||||
self.dump(iterativeDump(*output), excludeCode, excludeStorage, excludeMissingPreimages)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,33 +52,28 @@ func (s *StateSuite) TestDump(c *checker.C) {
|
|||
s.state.Commit(false)
|
||||
|
||||
// check that dump contains the state objects that are in trie
|
||||
got := string(s.state.Dump())
|
||||
got := string(s.state.Dump(false, false, true))
|
||||
want := `{
|
||||
"root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2",
|
||||
"accounts": {
|
||||
"0000000000000000000000000000000000000001": {
|
||||
"0x0000000000000000000000000000000000000001": {
|
||||
"balance": "22",
|
||||
"nonce": 0,
|
||||
"root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
|
||||
"codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
|
||||
"code": "",
|
||||
"storage": {}
|
||||
"codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
|
||||
},
|
||||
"0000000000000000000000000000000000000002": {
|
||||
"0x0000000000000000000000000000000000000002": {
|
||||
"balance": "44",
|
||||
"nonce": 0,
|
||||
"root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
|
||||
"codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
|
||||
"code": "",
|
||||
"storage": {}
|
||||
"codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
|
||||
},
|
||||
"0000000000000000000000000000000000000102": {
|
||||
"0x0000000000000000000000000000000000000102": {
|
||||
"balance": "0",
|
||||
"nonce": 0,
|
||||
"root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
|
||||
"codeHash": "87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3",
|
||||
"code": "03030303030303",
|
||||
"storage": {}
|
||||
"code": "03030303030303"
|
||||
}
|
||||
}
|
||||
}`
|
||||
|
|
|
|||
|
|
@ -266,7 +266,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error
|
|||
// both the pending block as well as the pending state from
|
||||
// the miner and operate on those
|
||||
_, stateDb := api.eth.miner.Pending()
|
||||
return stateDb.RawDump(), nil
|
||||
return stateDb.RawDump(false, false, true), nil
|
||||
}
|
||||
var block *types.Block
|
||||
if blockNr == rpc.LatestBlockNumber {
|
||||
|
|
@ -281,7 +281,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error
|
|||
if err != nil {
|
||||
return state.Dump{}, err
|
||||
}
|
||||
return stateDb.RawDump(), nil
|
||||
return stateDb.RawDump(false, false, true), nil
|
||||
}
|
||||
|
||||
// PrivateDebugAPI is the collection of Ethereum full node APIs exposed over
|
||||
|
|
|
|||
Loading…
Reference in New Issue