2015-10-05 11:37:56 -05:00
|
|
|
// Copyright 2015 The go-ethereum Authors
|
|
|
|
// This file is part of the go-ethereum library.
|
|
|
|
//
|
|
|
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
|
|
|
// it under the terms of the GNU Lesser General Public License as published by
|
|
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
|
|
// (at your option) any later version.
|
|
|
|
//
|
|
|
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
// GNU Lesser General Public License for more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU Lesser General Public License
|
|
|
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
package trie
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
2022-08-04 03:03:20 -05:00
|
|
|
"fmt"
|
2015-10-05 11:37:56 -05:00
|
|
|
"testing"
|
|
|
|
|
|
|
|
"github.com/ethereum/go-ethereum/common"
|
2022-11-28 07:31:28 -06:00
|
|
|
"github.com/ethereum/go-ethereum/core/rawdb"
|
2023-02-21 05:12:27 -06:00
|
|
|
"github.com/ethereum/go-ethereum/core/types"
|
2020-08-28 02:50:37 -05:00
|
|
|
"github.com/ethereum/go-ethereum/crypto"
|
2023-05-09 02:11:04 -05:00
|
|
|
"github.com/ethereum/go-ethereum/ethdb"
|
2018-09-24 07:57:49 -05:00
|
|
|
"github.com/ethereum/go-ethereum/ethdb/memorydb"
|
2023-05-09 02:11:04 -05:00
|
|
|
"github.com/ethereum/go-ethereum/trie/trienode"
|
2015-10-05 11:37:56 -05:00
|
|
|
)
|
|
|
|
|
|
|
|
// makeTestTrie create a sample test trie to test node-wise reconstruction.
|
2023-05-09 02:11:04 -05:00
|
|
|
func makeTestTrie(scheme string) (ethdb.Database, *Database, *StateTrie, map[string][]byte) {
|
2015-10-05 11:37:56 -05:00
|
|
|
// Create an empty trie
|
2023-05-09 02:11:04 -05:00
|
|
|
db := rawdb.NewMemoryDatabase()
|
|
|
|
triedb := newTestDatabase(db, scheme)
|
2023-05-11 02:19:42 -05:00
|
|
|
trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb)
|
2015-10-05 11:37:56 -05:00
|
|
|
|
|
|
|
// Fill it with some arbitrary data
|
|
|
|
content := make(map[string][]byte)
|
|
|
|
for i := byte(0); i < 255; i++ {
|
2015-12-28 07:20:37 -06:00
|
|
|
// Map the same data under multiple keys
|
2015-10-05 11:37:56 -05:00
|
|
|
key, val := common.LeftPadBytes([]byte{1, i}, 32), []byte{i}
|
|
|
|
content[string(key)] = val
|
2023-04-20 05:57:24 -05:00
|
|
|
trie.MustUpdate(key, val)
|
2015-10-05 11:37:56 -05:00
|
|
|
|
|
|
|
key, val = common.LeftPadBytes([]byte{2, i}, 32), []byte{i}
|
|
|
|
content[string(key)] = val
|
2023-04-20 05:57:24 -05:00
|
|
|
trie.MustUpdate(key, val)
|
2015-12-28 07:20:37 -06:00
|
|
|
|
2017-06-12 07:45:17 -05:00
|
|
|
// Add some other data to inflate the trie
|
2015-12-28 07:20:37 -06:00
|
|
|
for j := byte(3); j < 13; j++ {
|
|
|
|
key, val = common.LeftPadBytes([]byte{j, i}, 32), []byte{j, i}
|
|
|
|
content[string(key)] = val
|
2023-04-20 05:57:24 -05:00
|
|
|
trie.MustUpdate(key, val)
|
2015-12-28 07:20:37 -06:00
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2023-06-27 07:36:38 -05:00
|
|
|
root, nodes, _ := trie.Commit(false)
|
2023-07-24 05:22:09 -05:00
|
|
|
if err := triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil); err != nil {
|
2022-08-04 03:03:20 -05:00
|
|
|
panic(fmt.Errorf("failed to commit db %v", err))
|
|
|
|
}
|
2023-05-09 02:11:04 -05:00
|
|
|
if err := triedb.Commit(root, false); err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
2022-08-04 03:03:20 -05:00
|
|
|
// Re-create the trie based on the new state
|
cmd, core, eth, les, light: track deleted nodes (#25757)
* cmd, core, eth, les, light: track deleted nodes
* trie: add docs
* trie: address comments
* cmd, core, eth, les, light, trie: trie id
* trie: add tests
* trie, core: updates
* trie: fix imports
* trie: add utility print-method for nodeset
* trie: import err
* trie: fix go vet warnings
Co-authored-by: Martin Holst Swende <martin@swende.se>
2022-09-27 03:01:02 -05:00
|
|
|
trie, _ = NewStateTrie(TrieID(root), triedb)
|
2023-05-09 02:11:04 -05:00
|
|
|
return db, triedb, trie, content
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
// checkTrieContents cross references a reconstructed trie with an expected data
|
|
|
|
// content map.
|
2023-05-09 02:11:04 -05:00
|
|
|
func checkTrieContents(t *testing.T, db ethdb.Database, scheme string, root []byte, content map[string][]byte) {
|
2015-12-28 07:20:37 -06:00
|
|
|
// Check root availability and trie contents
|
2023-05-09 02:11:04 -05:00
|
|
|
ndb := newTestDatabase(db, scheme)
|
|
|
|
trie, err := NewStateTrie(TrieID(common.BytesToHash(root)), ndb)
|
2015-10-05 11:37:56 -05:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("failed to create trie at %x: %v", root, err)
|
|
|
|
}
|
2023-05-09 02:11:04 -05:00
|
|
|
if err := checkTrieConsistency(db, scheme, common.BytesToHash(root)); err != nil {
|
2015-12-28 07:20:37 -06:00
|
|
|
t.Fatalf("inconsistent trie at %x: %v", root, err)
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
for key, val := range content {
|
2023-04-20 05:57:24 -05:00
|
|
|
if have := trie.MustGet([]byte(key)); !bytes.Equal(have, val) {
|
2015-10-05 11:37:56 -05:00
|
|
|
t.Errorf("entry %x: content mismatch: have %x, want %x", key, have, val)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-02-16 04:37:00 -06:00
|
|
|
// checkTrieConsistency checks that all nodes in a trie are indeed present.
|
2023-05-09 02:11:04 -05:00
|
|
|
func checkTrieConsistency(db ethdb.Database, scheme string, root common.Hash) error {
|
|
|
|
ndb := newTestDatabase(db, scheme)
|
|
|
|
trie, err := NewStateTrie(TrieID(root), ndb)
|
2015-12-28 07:20:37 -06:00
|
|
|
if err != nil {
|
2017-06-12 07:45:17 -05:00
|
|
|
return nil // Consider a non existent state consistent
|
2015-12-28 07:20:37 -06:00
|
|
|
}
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
it := trie.MustNodeIterator(nil)
|
2017-02-22 16:49:34 -06:00
|
|
|
for it.Next(true) {
|
2015-12-28 07:20:37 -06:00
|
|
|
}
|
2017-02-22 16:49:34 -06:00
|
|
|
return it.Error()
|
2015-12-28 07:20:37 -06:00
|
|
|
}
|
|
|
|
|
2022-07-15 06:55:51 -05:00
|
|
|
// trieElement represents the element in the state trie(bytecode or trie node).
|
|
|
|
type trieElement struct {
|
|
|
|
path string
|
|
|
|
hash common.Hash
|
|
|
|
syncPath SyncPath
|
|
|
|
}
|
|
|
|
|
2015-10-05 11:37:56 -05:00
|
|
|
// Tests that an empty trie is not scheduled for syncing.
|
2018-05-29 10:48:43 -05:00
|
|
|
func TestEmptySync(t *testing.T) {
|
all: activate pbss as experimental feature (#26274)
* all: activate pbss
* core/rawdb: fix compilation error
* cma, core, eth, les, trie: address comments
* cmd, core, eth, trie: polish code
* core, cmd, eth: address comments
* cmd, core, eth, les, light, tests: address comment
* cmd/utils: shorten log message
* trie/triedb/pathdb: limit node buffer size to 1gb
* cmd/utils: fix opening non-existing db
* cmd/utils: rename flag name
* cmd, core: group chain history flags and fix tests
* core, eth, trie: fix memory leak in snapshot generation
* cmd, eth, internal: deprecate flags
* all: enable state tests for pathdb, fixes
* cmd, core: polish code
* trie/triedb/pathdb: limit the node buffer size to 256mb
---------
Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Péter Szilágyi <peterke@gmail.com>
2023-08-10 14:21:36 -05:00
|
|
|
dbA := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)
|
|
|
|
dbB := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)
|
2023-08-01 07:17:32 -05:00
|
|
|
dbC := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme)
|
|
|
|
dbD := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme)
|
2023-05-09 02:11:04 -05:00
|
|
|
|
|
|
|
emptyA := NewEmpty(dbA)
|
2023-02-21 05:12:27 -06:00
|
|
|
emptyB, _ := New(TrieID(types.EmptyRootHash), dbB)
|
2023-08-01 07:17:32 -05:00
|
|
|
emptyC := NewEmpty(dbC)
|
|
|
|
emptyD, _ := New(TrieID(types.EmptyRootHash), dbD)
|
2015-10-05 11:37:56 -05:00
|
|
|
|
2023-08-01 07:17:32 -05:00
|
|
|
for i, trie := range []*Trie{emptyA, emptyB, emptyC, emptyD} {
|
|
|
|
sync := NewSync(trie.Hash(), memorydb.New(), nil, []*Database{dbA, dbB, dbC, dbD}[i].Scheme())
|
2022-07-15 06:55:51 -05:00
|
|
|
if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 {
|
|
|
|
t.Errorf("test %d: content requested for empty trie: %v, %v, %v", i, paths, nodes, codes)
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Tests that given a root hash, a trie can sync iteratively on a single thread,
|
|
|
|
// requesting retrieval tasks and returning all of them in one go.
|
2023-05-09 02:11:04 -05:00
|
|
|
func TestIterativeSync(t *testing.T) {
|
|
|
|
testIterativeSync(t, 1, false, rawdb.HashScheme)
|
|
|
|
testIterativeSync(t, 100, false, rawdb.HashScheme)
|
|
|
|
testIterativeSync(t, 1, true, rawdb.HashScheme)
|
|
|
|
testIterativeSync(t, 100, true, rawdb.HashScheme)
|
2023-08-01 07:17:32 -05:00
|
|
|
testIterativeSync(t, 1, false, rawdb.PathScheme)
|
|
|
|
testIterativeSync(t, 100, false, rawdb.PathScheme)
|
|
|
|
testIterativeSync(t, 1, true, rawdb.PathScheme)
|
|
|
|
testIterativeSync(t, 100, true, rawdb.PathScheme)
|
2023-05-09 02:11:04 -05:00
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
|
2023-05-09 02:11:04 -05:00
|
|
|
func testIterativeSync(t *testing.T, count int, bypath bool, scheme string) {
|
2015-10-05 11:37:56 -05:00
|
|
|
// Create a random trie to copy
|
2023-05-09 02:11:04 -05:00
|
|
|
_, srcDb, srcTrie, srcData := makeTestTrie(scheme)
|
2015-10-05 11:37:56 -05:00
|
|
|
|
|
|
|
// Create a destination trie and sync with the scheduler
|
2022-11-28 07:31:28 -06:00
|
|
|
diskdb := rawdb.NewMemoryDatabase()
|
|
|
|
sched := NewSync(srcTrie.Hash(), diskdb, nil, srcDb.Scheme())
|
2015-10-05 11:37:56 -05:00
|
|
|
|
2022-07-15 06:55:51 -05:00
|
|
|
// The code requests are ignored here since there is no code
|
|
|
|
// at the testing trie.
|
|
|
|
paths, nodes, _ := sched.Missing(count)
|
|
|
|
var elements []trieElement
|
|
|
|
for i := 0; i < len(paths); i++ {
|
|
|
|
elements = append(elements, trieElement{
|
|
|
|
path: paths[i],
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(paths[i])),
|
|
|
|
})
|
2020-08-28 02:50:37 -05:00
|
|
|
}
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
reader, err := srcDb.Reader(srcTrie.Hash())
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("State is not available %x", srcTrie.Hash())
|
|
|
|
}
|
2022-07-15 06:55:51 -05:00
|
|
|
for len(elements) > 0 {
|
|
|
|
results := make([]NodeSyncResult, len(elements))
|
|
|
|
if !bypath {
|
|
|
|
for i, element := range elements {
|
2023-05-09 02:11:04 -05:00
|
|
|
owner, inner := ResolvePath([]byte(element.path))
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
data, err := reader.Node(owner, inner, element.hash)
|
2022-07-15 06:55:51 -05:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("failed to retrieve node data for hash %x: %v", element.hash, err)
|
|
|
|
}
|
|
|
|
results[i] = NodeSyncResult{element.path, data}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2022-07-15 06:55:51 -05:00
|
|
|
} else {
|
|
|
|
for i, element := range elements {
|
2023-03-27 03:48:46 -05:00
|
|
|
data, _, err := srcTrie.GetNode(element.syncPath[len(element.syncPath)-1])
|
2022-07-15 06:55:51 -05:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("failed to retrieve node data for path %x: %v", element.path, err)
|
|
|
|
}
|
|
|
|
results[i] = NodeSyncResult{element.path, data}
|
2020-08-28 02:50:37 -05:00
|
|
|
}
|
|
|
|
}
|
2020-08-21 07:10:40 -05:00
|
|
|
for _, result := range results {
|
2022-07-15 06:55:51 -05:00
|
|
|
if err := sched.ProcessNode(result); err != nil {
|
2020-08-21 07:10:40 -05:00
|
|
|
t.Fatalf("failed to process result %v", err)
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2019-10-28 12:50:11 -05:00
|
|
|
batch := diskdb.NewBatch()
|
|
|
|
if err := sched.Commit(batch); err != nil {
|
|
|
|
t.Fatalf("failed to commit data: %v", err)
|
eth/downloader: separate state sync from queue (#14460)
* eth/downloader: separate state sync from queue
Scheduling of state node downloads hogged the downloader queue lock when
new requests were scheduled. This caused timeouts for other requests.
With this change, state sync is fully independent of all other downloads
and doesn't involve the queue at all.
State sync is started and checked on in processContent. This is slightly
awkward because processContent doesn't have a select loop. Instead, the
queue is closed by an auxiliary goroutine when state sync fails. We
tried several alternatives to this but settled on the current approach
because it's the least amount of change overall.
Handling of the pivot block has changed slightly: the queue previously
prevented import of pivot block receipts before the state of the pivot
block was available. In this commit, the receipt will be imported before
the state. This causes an annoyance where the pivot block is committed
as fast block head even when state downloads fail. Stay tuned for more
updates in this area ;)
* eth/downloader: remove cancelTimeout channel
* eth/downloader: retry state requests on timeout
* eth/downloader: improve comment
* eth/downloader: mark peers idle when state sync is done
* eth/downloader: move pivot block splitting to processContent
This change also ensures that pivot block receipts aren't imported
before the pivot block itself.
* eth/downloader: limit state node retries
* eth/downloader: improve state node error handling and retry check
* eth/downloader: remove maxStateNodeRetries
It fails the sync too much.
* eth/downloader: remove last use of cancelCh in statesync.go
Fixes TestDeliverHeadersHang*Fast and (hopefully)
the weird cancellation behaviour at the end of fast sync.
* eth/downloader: fix leak in runStateSync
* eth/downloader: don't run processFullSyncContent in LightSync mode
* eth/downloader: improve comments
* eth/downloader: fix vet, megacheck
* eth/downloader: remove unrequested tasks anyway
* eth/downloader, trie: various polishes around duplicate items
This commit explicitly tracks duplicate and unexpected state
delieveries done against a trie Sync structure, also adding
there to import info logs.
The commit moves the db batch used to commit trie changes one
level deeper so its flushed after every node insertion. This
is needed to avoid a lot of duplicate retrievals caused by
inconsistencies between Sync internals and database. A better
approach is to track not-yet-written states in trie.Sync and
flush on commit, but I'm focuing on correctness first now.
The commit fixes a regression around pivot block fail count.
The counter previously was reset to 1 if and only if a sync
cycle progressed (inserted at least 1 entry to the database).
The current code reset it already if a node was delivered,
which is not stong enough, because unless it ends up written
to disk, an attacker can just loop and attack ad infinitum.
The commit also fixes a regression around state deliveries
and timeouts. The old downloader tracked if a delivery is
stale (none of the deliveries were requestedt), in which
case it didn't mark the node idle and did not send further
requests, since it signals a past timeout. The current code
did mark it idle even on stale deliveries, which eventually
caused two requests to be in flight at the same time, making
the deliveries always stale and mass duplicating retrievals
between multiple peers.
* eth/downloader: fix state request leak
This commit fixes the hang seen sometimes while doing the state
sync. The cause of the hang was a rare combination of events:
request state data from peer, peer drops and reconnects almost
immediately. This caused a new download task to be assigned to
the peer, overwriting the old one still waiting for a timeout,
which in turned leaked the requests out, never to be retried.
The fix is to ensure that a task assignment moves any pending
one back into the retry queue.
The commit also fixes a regression with peer dropping due to
stalls. The current code considered a peer stalling if they
timed out delivering 1 item. However, the downloader never
requests only one, the minimum is 2 (attempt to fine tune
estimated latency/bandwidth). The fix is simply to drop if
a timeout is detected at 2 items.
Apart from the above bugfixes, the commit contains some code
polishes I made while debugging the hang.
* core, eth, trie: support batched trie sync db writes
* trie: rename SyncMemCache to syncMemBatch
2017-06-22 07:26:03 -05:00
|
|
|
}
|
2019-10-28 12:50:11 -05:00
|
|
|
batch.Write()
|
2020-08-28 02:50:37 -05:00
|
|
|
|
2022-07-15 06:55:51 -05:00
|
|
|
paths, nodes, _ = sched.Missing(count)
|
|
|
|
elements = elements[:0]
|
|
|
|
for i := 0; i < len(paths); i++ {
|
|
|
|
elements = append(elements, trieElement{
|
|
|
|
path: paths[i],
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(paths[i])),
|
|
|
|
})
|
2020-08-28 02:50:37 -05:00
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2015-12-28 07:20:37 -06:00
|
|
|
// Cross check that the two tries are in sync
|
2023-05-09 02:11:04 -05:00
|
|
|
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), srcData)
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
// Tests that the trie scheduler can correctly reconstruct the state even if only
|
|
|
|
// partial results are returned, and the others sent only later.
|
2018-05-29 10:48:43 -05:00
|
|
|
func TestIterativeDelayedSync(t *testing.T) {
|
2023-05-09 02:11:04 -05:00
|
|
|
testIterativeDelayedSync(t, rawdb.HashScheme)
|
2023-08-01 07:17:32 -05:00
|
|
|
testIterativeDelayedSync(t, rawdb.PathScheme)
|
2023-05-09 02:11:04 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
func testIterativeDelayedSync(t *testing.T, scheme string) {
|
2015-10-05 11:37:56 -05:00
|
|
|
// Create a random trie to copy
|
2023-05-09 02:11:04 -05:00
|
|
|
_, srcDb, srcTrie, srcData := makeTestTrie(scheme)
|
2015-10-05 11:37:56 -05:00
|
|
|
|
|
|
|
// Create a destination trie and sync with the scheduler
|
2022-11-28 07:31:28 -06:00
|
|
|
diskdb := rawdb.NewMemoryDatabase()
|
|
|
|
sched := NewSync(srcTrie.Hash(), diskdb, nil, srcDb.Scheme())
|
2015-10-05 11:37:56 -05:00
|
|
|
|
2022-07-15 06:55:51 -05:00
|
|
|
// The code requests are ignored here since there is no code
|
|
|
|
// at the testing trie.
|
|
|
|
paths, nodes, _ := sched.Missing(10000)
|
|
|
|
var elements []trieElement
|
|
|
|
for i := 0; i < len(paths); i++ {
|
|
|
|
elements = append(elements, trieElement{
|
|
|
|
path: paths[i],
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(paths[i])),
|
|
|
|
})
|
|
|
|
}
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
reader, err := srcDb.Reader(srcTrie.Hash())
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("State is not available %x", srcTrie.Hash())
|
|
|
|
}
|
2022-07-15 06:55:51 -05:00
|
|
|
for len(elements) > 0 {
|
2015-10-05 11:37:56 -05:00
|
|
|
// Sync only half of the scheduled nodes
|
2022-07-15 06:55:51 -05:00
|
|
|
results := make([]NodeSyncResult, len(elements)/2+1)
|
|
|
|
for i, element := range elements[:len(results)] {
|
2023-05-09 02:11:04 -05:00
|
|
|
owner, inner := ResolvePath([]byte(element.path))
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
data, err := reader.Node(owner, inner, element.hash)
|
2015-10-05 11:37:56 -05:00
|
|
|
if err != nil {
|
2022-07-15 06:55:51 -05:00
|
|
|
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2022-07-15 06:55:51 -05:00
|
|
|
results[i] = NodeSyncResult{element.path, data}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2020-08-21 07:10:40 -05:00
|
|
|
for _, result := range results {
|
2022-07-15 06:55:51 -05:00
|
|
|
if err := sched.ProcessNode(result); err != nil {
|
2020-08-21 07:10:40 -05:00
|
|
|
t.Fatalf("failed to process result %v", err)
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2019-10-28 12:50:11 -05:00
|
|
|
batch := diskdb.NewBatch()
|
|
|
|
if err := sched.Commit(batch); err != nil {
|
|
|
|
t.Fatalf("failed to commit data: %v", err)
|
eth/downloader: separate state sync from queue (#14460)
* eth/downloader: separate state sync from queue
Scheduling of state node downloads hogged the downloader queue lock when
new requests were scheduled. This caused timeouts for other requests.
With this change, state sync is fully independent of all other downloads
and doesn't involve the queue at all.
State sync is started and checked on in processContent. This is slightly
awkward because processContent doesn't have a select loop. Instead, the
queue is closed by an auxiliary goroutine when state sync fails. We
tried several alternatives to this but settled on the current approach
because it's the least amount of change overall.
Handling of the pivot block has changed slightly: the queue previously
prevented import of pivot block receipts before the state of the pivot
block was available. In this commit, the receipt will be imported before
the state. This causes an annoyance where the pivot block is committed
as fast block head even when state downloads fail. Stay tuned for more
updates in this area ;)
* eth/downloader: remove cancelTimeout channel
* eth/downloader: retry state requests on timeout
* eth/downloader: improve comment
* eth/downloader: mark peers idle when state sync is done
* eth/downloader: move pivot block splitting to processContent
This change also ensures that pivot block receipts aren't imported
before the pivot block itself.
* eth/downloader: limit state node retries
* eth/downloader: improve state node error handling and retry check
* eth/downloader: remove maxStateNodeRetries
It fails the sync too much.
* eth/downloader: remove last use of cancelCh in statesync.go
Fixes TestDeliverHeadersHang*Fast and (hopefully)
the weird cancellation behaviour at the end of fast sync.
* eth/downloader: fix leak in runStateSync
* eth/downloader: don't run processFullSyncContent in LightSync mode
* eth/downloader: improve comments
* eth/downloader: fix vet, megacheck
* eth/downloader: remove unrequested tasks anyway
* eth/downloader, trie: various polishes around duplicate items
This commit explicitly tracks duplicate and unexpected state
delieveries done against a trie Sync structure, also adding
there to import info logs.
The commit moves the db batch used to commit trie changes one
level deeper so its flushed after every node insertion. This
is needed to avoid a lot of duplicate retrievals caused by
inconsistencies between Sync internals and database. A better
approach is to track not-yet-written states in trie.Sync and
flush on commit, but I'm focuing on correctness first now.
The commit fixes a regression around pivot block fail count.
The counter previously was reset to 1 if and only if a sync
cycle progressed (inserted at least 1 entry to the database).
The current code reset it already if a node was delivered,
which is not stong enough, because unless it ends up written
to disk, an attacker can just loop and attack ad infinitum.
The commit also fixes a regression around state deliveries
and timeouts. The old downloader tracked if a delivery is
stale (none of the deliveries were requestedt), in which
case it didn't mark the node idle and did not send further
requests, since it signals a past timeout. The current code
did mark it idle even on stale deliveries, which eventually
caused two requests to be in flight at the same time, making
the deliveries always stale and mass duplicating retrievals
between multiple peers.
* eth/downloader: fix state request leak
This commit fixes the hang seen sometimes while doing the state
sync. The cause of the hang was a rare combination of events:
request state data from peer, peer drops and reconnects almost
immediately. This caused a new download task to be assigned to
the peer, overwriting the old one still waiting for a timeout,
which in turned leaked the requests out, never to be retried.
The fix is to ensure that a task assignment moves any pending
one back into the retry queue.
The commit also fixes a regression with peer dropping due to
stalls. The current code considered a peer stalling if they
timed out delivering 1 item. However, the downloader never
requests only one, the minimum is 2 (attempt to fine tune
estimated latency/bandwidth). The fix is simply to drop if
a timeout is detected at 2 items.
Apart from the above bugfixes, the commit contains some code
polishes I made while debugging the hang.
* core, eth, trie: support batched trie sync db writes
* trie: rename SyncMemCache to syncMemBatch
2017-06-22 07:26:03 -05:00
|
|
|
}
|
2019-10-28 12:50:11 -05:00
|
|
|
batch.Write()
|
2020-08-28 02:50:37 -05:00
|
|
|
|
2022-07-15 06:55:51 -05:00
|
|
|
paths, nodes, _ = sched.Missing(10000)
|
|
|
|
elements = elements[len(results):]
|
|
|
|
for i := 0; i < len(paths); i++ {
|
|
|
|
elements = append(elements, trieElement{
|
|
|
|
path: paths[i],
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(paths[i])),
|
|
|
|
})
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2015-12-28 07:20:37 -06:00
|
|
|
// Cross check that the two tries are in sync
|
2023-05-09 02:11:04 -05:00
|
|
|
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), srcData)
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
// Tests that given a root hash, a trie can sync iteratively on a single thread,
|
|
|
|
// requesting retrieval tasks and returning all of them in one go, however in a
|
|
|
|
// random order.
|
2023-05-09 02:11:04 -05:00
|
|
|
func TestIterativeRandomSyncIndividual(t *testing.T) {
|
|
|
|
testIterativeRandomSync(t, 1, rawdb.HashScheme)
|
|
|
|
testIterativeRandomSync(t, 100, rawdb.HashScheme)
|
2023-08-01 07:17:32 -05:00
|
|
|
testIterativeRandomSync(t, 1, rawdb.PathScheme)
|
|
|
|
testIterativeRandomSync(t, 100, rawdb.PathScheme)
|
2023-05-09 02:11:04 -05:00
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
|
2023-05-09 02:11:04 -05:00
|
|
|
func testIterativeRandomSync(t *testing.T, count int, scheme string) {
|
2015-10-05 11:37:56 -05:00
|
|
|
// Create a random trie to copy
|
2023-05-09 02:11:04 -05:00
|
|
|
_, srcDb, srcTrie, srcData := makeTestTrie(scheme)
|
2015-10-05 11:37:56 -05:00
|
|
|
|
|
|
|
// Create a destination trie and sync with the scheduler
|
2022-11-28 07:31:28 -06:00
|
|
|
diskdb := rawdb.NewMemoryDatabase()
|
|
|
|
sched := NewSync(srcTrie.Hash(), diskdb, nil, srcDb.Scheme())
|
2015-10-05 11:37:56 -05:00
|
|
|
|
2022-07-15 06:55:51 -05:00
|
|
|
// The code requests are ignored here since there is no code
|
|
|
|
// at the testing trie.
|
|
|
|
paths, nodes, _ := sched.Missing(count)
|
|
|
|
queue := make(map[string]trieElement)
|
|
|
|
for i, path := range paths {
|
|
|
|
queue[path] = trieElement{
|
|
|
|
path: paths[i],
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(paths[i])),
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
reader, err := srcDb.Reader(srcTrie.Hash())
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("State is not available %x", srcTrie.Hash())
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
for len(queue) > 0 {
|
|
|
|
// Fetch all the queued nodes in a random order
|
2022-07-15 06:55:51 -05:00
|
|
|
results := make([]NodeSyncResult, 0, len(queue))
|
|
|
|
for path, element := range queue {
|
2023-05-09 02:11:04 -05:00
|
|
|
owner, inner := ResolvePath([]byte(element.path))
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
data, err := reader.Node(owner, inner, element.hash)
|
2015-10-05 11:37:56 -05:00
|
|
|
if err != nil {
|
2022-07-15 06:55:51 -05:00
|
|
|
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2022-07-15 06:55:51 -05:00
|
|
|
results = append(results, NodeSyncResult{path, data})
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
|
|
|
// Feed the retrieved results back and queue new tasks
|
2020-08-21 07:10:40 -05:00
|
|
|
for _, result := range results {
|
2022-07-15 06:55:51 -05:00
|
|
|
if err := sched.ProcessNode(result); err != nil {
|
2020-08-21 07:10:40 -05:00
|
|
|
t.Fatalf("failed to process result %v", err)
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2019-10-28 12:50:11 -05:00
|
|
|
batch := diskdb.NewBatch()
|
|
|
|
if err := sched.Commit(batch); err != nil {
|
|
|
|
t.Fatalf("failed to commit data: %v", err)
|
eth/downloader: separate state sync from queue (#14460)
* eth/downloader: separate state sync from queue
Scheduling of state node downloads hogged the downloader queue lock when
new requests were scheduled. This caused timeouts for other requests.
With this change, state sync is fully independent of all other downloads
and doesn't involve the queue at all.
State sync is started and checked on in processContent. This is slightly
awkward because processContent doesn't have a select loop. Instead, the
queue is closed by an auxiliary goroutine when state sync fails. We
tried several alternatives to this but settled on the current approach
because it's the least amount of change overall.
Handling of the pivot block has changed slightly: the queue previously
prevented import of pivot block receipts before the state of the pivot
block was available. In this commit, the receipt will be imported before
the state. This causes an annoyance where the pivot block is committed
as fast block head even when state downloads fail. Stay tuned for more
updates in this area ;)
* eth/downloader: remove cancelTimeout channel
* eth/downloader: retry state requests on timeout
* eth/downloader: improve comment
* eth/downloader: mark peers idle when state sync is done
* eth/downloader: move pivot block splitting to processContent
This change also ensures that pivot block receipts aren't imported
before the pivot block itself.
* eth/downloader: limit state node retries
* eth/downloader: improve state node error handling and retry check
* eth/downloader: remove maxStateNodeRetries
It fails the sync too much.
* eth/downloader: remove last use of cancelCh in statesync.go
Fixes TestDeliverHeadersHang*Fast and (hopefully)
the weird cancellation behaviour at the end of fast sync.
* eth/downloader: fix leak in runStateSync
* eth/downloader: don't run processFullSyncContent in LightSync mode
* eth/downloader: improve comments
* eth/downloader: fix vet, megacheck
* eth/downloader: remove unrequested tasks anyway
* eth/downloader, trie: various polishes around duplicate items
This commit explicitly tracks duplicate and unexpected state
delieveries done against a trie Sync structure, also adding
there to import info logs.
The commit moves the db batch used to commit trie changes one
level deeper so its flushed after every node insertion. This
is needed to avoid a lot of duplicate retrievals caused by
inconsistencies between Sync internals and database. A better
approach is to track not-yet-written states in trie.Sync and
flush on commit, but I'm focuing on correctness first now.
The commit fixes a regression around pivot block fail count.
The counter previously was reset to 1 if and only if a sync
cycle progressed (inserted at least 1 entry to the database).
The current code reset it already if a node was delivered,
which is not stong enough, because unless it ends up written
to disk, an attacker can just loop and attack ad infinitum.
The commit also fixes a regression around state deliveries
and timeouts. The old downloader tracked if a delivery is
stale (none of the deliveries were requestedt), in which
case it didn't mark the node idle and did not send further
requests, since it signals a past timeout. The current code
did mark it idle even on stale deliveries, which eventually
caused two requests to be in flight at the same time, making
the deliveries always stale and mass duplicating retrievals
between multiple peers.
* eth/downloader: fix state request leak
This commit fixes the hang seen sometimes while doing the state
sync. The cause of the hang was a rare combination of events:
request state data from peer, peer drops and reconnects almost
immediately. This caused a new download task to be assigned to
the peer, overwriting the old one still waiting for a timeout,
which in turned leaked the requests out, never to be retried.
The fix is to ensure that a task assignment moves any pending
one back into the retry queue.
The commit also fixes a regression with peer dropping due to
stalls. The current code considered a peer stalling if they
timed out delivering 1 item. However, the downloader never
requests only one, the minimum is 2 (attempt to fine tune
estimated latency/bandwidth). The fix is simply to drop if
a timeout is detected at 2 items.
Apart from the above bugfixes, the commit contains some code
polishes I made while debugging the hang.
* core, eth, trie: support batched trie sync db writes
* trie: rename SyncMemCache to syncMemBatch
2017-06-22 07:26:03 -05:00
|
|
|
}
|
2019-10-28 12:50:11 -05:00
|
|
|
batch.Write()
|
2020-08-28 02:50:37 -05:00
|
|
|
|
2022-07-15 06:55:51 -05:00
|
|
|
paths, nodes, _ = sched.Missing(count)
|
|
|
|
queue = make(map[string]trieElement)
|
|
|
|
for i, path := range paths {
|
|
|
|
queue[path] = trieElement{
|
|
|
|
path: path,
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(path)),
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
|
|
|
}
|
2015-12-28 07:20:37 -06:00
|
|
|
// Cross check that the two tries are in sync
|
2023-05-09 02:11:04 -05:00
|
|
|
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), srcData)
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
// Tests that the trie scheduler can correctly reconstruct the state even if only
|
|
|
|
// partial results are returned (Even those randomly), others sent only later.
|
2018-05-29 10:48:43 -05:00
|
|
|
func TestIterativeRandomDelayedSync(t *testing.T) {
|
2023-05-09 02:11:04 -05:00
|
|
|
testIterativeRandomDelayedSync(t, rawdb.HashScheme)
|
2023-08-01 07:17:32 -05:00
|
|
|
testIterativeRandomDelayedSync(t, rawdb.PathScheme)
|
2023-05-09 02:11:04 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
func testIterativeRandomDelayedSync(t *testing.T, scheme string) {
|
2015-10-05 11:37:56 -05:00
|
|
|
// Create a random trie to copy
|
2023-05-09 02:11:04 -05:00
|
|
|
_, srcDb, srcTrie, srcData := makeTestTrie(scheme)
|
2015-10-05 11:37:56 -05:00
|
|
|
|
|
|
|
// Create a destination trie and sync with the scheduler
|
2022-11-28 07:31:28 -06:00
|
|
|
diskdb := rawdb.NewMemoryDatabase()
|
|
|
|
sched := NewSync(srcTrie.Hash(), diskdb, nil, srcDb.Scheme())
|
2015-10-05 11:37:56 -05:00
|
|
|
|
2022-07-15 06:55:51 -05:00
|
|
|
// The code requests are ignored here since there is no code
|
|
|
|
// at the testing trie.
|
|
|
|
paths, nodes, _ := sched.Missing(10000)
|
|
|
|
queue := make(map[string]trieElement)
|
|
|
|
for i, path := range paths {
|
|
|
|
queue[path] = trieElement{
|
|
|
|
path: path,
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(path)),
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
reader, err := srcDb.Reader(srcTrie.Hash())
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("State is not available %x", srcTrie.Hash())
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
for len(queue) > 0 {
|
|
|
|
// Sync only half of the scheduled nodes, even those in random order
|
2022-07-15 06:55:51 -05:00
|
|
|
results := make([]NodeSyncResult, 0, len(queue)/2+1)
|
|
|
|
for path, element := range queue {
|
2023-05-09 02:11:04 -05:00
|
|
|
owner, inner := ResolvePath([]byte(element.path))
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
data, err := reader.Node(owner, inner, element.hash)
|
2015-10-05 11:37:56 -05:00
|
|
|
if err != nil {
|
2022-07-15 06:55:51 -05:00
|
|
|
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2022-07-15 06:55:51 -05:00
|
|
|
results = append(results, NodeSyncResult{path, data})
|
2015-10-05 11:37:56 -05:00
|
|
|
|
|
|
|
if len(results) >= cap(results) {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Feed the retrieved results back and queue new tasks
|
2020-08-21 07:10:40 -05:00
|
|
|
for _, result := range results {
|
2022-07-15 06:55:51 -05:00
|
|
|
if err := sched.ProcessNode(result); err != nil {
|
2020-08-21 07:10:40 -05:00
|
|
|
t.Fatalf("failed to process result %v", err)
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2019-10-28 12:50:11 -05:00
|
|
|
batch := diskdb.NewBatch()
|
|
|
|
if err := sched.Commit(batch); err != nil {
|
|
|
|
t.Fatalf("failed to commit data: %v", err)
|
eth/downloader: separate state sync from queue (#14460)
* eth/downloader: separate state sync from queue
Scheduling of state node downloads hogged the downloader queue lock when
new requests were scheduled. This caused timeouts for other requests.
With this change, state sync is fully independent of all other downloads
and doesn't involve the queue at all.
State sync is started and checked on in processContent. This is slightly
awkward because processContent doesn't have a select loop. Instead, the
queue is closed by an auxiliary goroutine when state sync fails. We
tried several alternatives to this but settled on the current approach
because it's the least amount of change overall.
Handling of the pivot block has changed slightly: the queue previously
prevented import of pivot block receipts before the state of the pivot
block was available. In this commit, the receipt will be imported before
the state. This causes an annoyance where the pivot block is committed
as fast block head even when state downloads fail. Stay tuned for more
updates in this area ;)
* eth/downloader: remove cancelTimeout channel
* eth/downloader: retry state requests on timeout
* eth/downloader: improve comment
* eth/downloader: mark peers idle when state sync is done
* eth/downloader: move pivot block splitting to processContent
This change also ensures that pivot block receipts aren't imported
before the pivot block itself.
* eth/downloader: limit state node retries
* eth/downloader: improve state node error handling and retry check
* eth/downloader: remove maxStateNodeRetries
It fails the sync too much.
* eth/downloader: remove last use of cancelCh in statesync.go
Fixes TestDeliverHeadersHang*Fast and (hopefully)
the weird cancellation behaviour at the end of fast sync.
* eth/downloader: fix leak in runStateSync
* eth/downloader: don't run processFullSyncContent in LightSync mode
* eth/downloader: improve comments
* eth/downloader: fix vet, megacheck
* eth/downloader: remove unrequested tasks anyway
* eth/downloader, trie: various polishes around duplicate items
This commit explicitly tracks duplicate and unexpected state
delieveries done against a trie Sync structure, also adding
there to import info logs.
The commit moves the db batch used to commit trie changes one
level deeper so its flushed after every node insertion. This
is needed to avoid a lot of duplicate retrievals caused by
inconsistencies between Sync internals and database. A better
approach is to track not-yet-written states in trie.Sync and
flush on commit, but I'm focuing on correctness first now.
The commit fixes a regression around pivot block fail count.
The counter previously was reset to 1 if and only if a sync
cycle progressed (inserted at least 1 entry to the database).
The current code reset it already if a node was delivered,
which is not stong enough, because unless it ends up written
to disk, an attacker can just loop and attack ad infinitum.
The commit also fixes a regression around state deliveries
and timeouts. The old downloader tracked if a delivery is
stale (none of the deliveries were requestedt), in which
case it didn't mark the node idle and did not send further
requests, since it signals a past timeout. The current code
did mark it idle even on stale deliveries, which eventually
caused two requests to be in flight at the same time, making
the deliveries always stale and mass duplicating retrievals
between multiple peers.
* eth/downloader: fix state request leak
This commit fixes the hang seen sometimes while doing the state
sync. The cause of the hang was a rare combination of events:
request state data from peer, peer drops and reconnects almost
immediately. This caused a new download task to be assigned to
the peer, overwriting the old one still waiting for a timeout,
which in turned leaked the requests out, never to be retried.
The fix is to ensure that a task assignment moves any pending
one back into the retry queue.
The commit also fixes a regression with peer dropping due to
stalls. The current code considered a peer stalling if they
timed out delivering 1 item. However, the downloader never
requests only one, the minimum is 2 (attempt to fine tune
estimated latency/bandwidth). The fix is simply to drop if
a timeout is detected at 2 items.
Apart from the above bugfixes, the commit contains some code
polishes I made while debugging the hang.
* core, eth, trie: support batched trie sync db writes
* trie: rename SyncMemCache to syncMemBatch
2017-06-22 07:26:03 -05:00
|
|
|
}
|
2019-10-28 12:50:11 -05:00
|
|
|
batch.Write()
|
2015-10-05 11:37:56 -05:00
|
|
|
for _, result := range results {
|
2022-07-15 06:55:51 -05:00
|
|
|
delete(queue, result.Path)
|
|
|
|
}
|
|
|
|
paths, nodes, _ = sched.Missing(10000)
|
|
|
|
for i, path := range paths {
|
|
|
|
queue[path] = trieElement{
|
|
|
|
path: path,
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(path)),
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
|
|
|
}
|
2015-12-28 07:20:37 -06:00
|
|
|
// Cross check that the two tries are in sync
|
2023-05-09 02:11:04 -05:00
|
|
|
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), srcData)
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
// Tests that a trie sync will not request nodes multiple times, even if they
|
|
|
|
// have such references.
|
2018-05-29 10:48:43 -05:00
|
|
|
func TestDuplicateAvoidanceSync(t *testing.T) {
|
2023-05-09 02:11:04 -05:00
|
|
|
testDuplicateAvoidanceSync(t, rawdb.HashScheme)
|
2023-08-01 07:17:32 -05:00
|
|
|
testDuplicateAvoidanceSync(t, rawdb.PathScheme)
|
2023-05-09 02:11:04 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
func testDuplicateAvoidanceSync(t *testing.T, scheme string) {
|
2015-10-05 11:37:56 -05:00
|
|
|
// Create a random trie to copy
|
2023-05-09 02:11:04 -05:00
|
|
|
_, srcDb, srcTrie, srcData := makeTestTrie(scheme)
|
2015-10-05 11:37:56 -05:00
|
|
|
|
|
|
|
// Create a destination trie and sync with the scheduler
|
2022-11-28 07:31:28 -06:00
|
|
|
diskdb := rawdb.NewMemoryDatabase()
|
|
|
|
sched := NewSync(srcTrie.Hash(), diskdb, nil, srcDb.Scheme())
|
2015-10-05 11:37:56 -05:00
|
|
|
|
2022-07-15 06:55:51 -05:00
|
|
|
// The code requests are ignored here since there is no code
|
|
|
|
// at the testing trie.
|
|
|
|
paths, nodes, _ := sched.Missing(0)
|
|
|
|
var elements []trieElement
|
|
|
|
for i := 0; i < len(paths); i++ {
|
|
|
|
elements = append(elements, trieElement{
|
|
|
|
path: paths[i],
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(paths[i])),
|
|
|
|
})
|
|
|
|
}
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
reader, err := srcDb.Reader(srcTrie.Hash())
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("State is not available %x", srcTrie.Hash())
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
requested := make(map[common.Hash]struct{})
|
2022-07-15 06:55:51 -05:00
|
|
|
for len(elements) > 0 {
|
|
|
|
results := make([]NodeSyncResult, len(elements))
|
|
|
|
for i, element := range elements {
|
2023-05-09 02:11:04 -05:00
|
|
|
owner, inner := ResolvePath([]byte(element.path))
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
data, err := reader.Node(owner, inner, element.hash)
|
2015-10-05 11:37:56 -05:00
|
|
|
if err != nil {
|
2022-07-15 06:55:51 -05:00
|
|
|
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2022-07-15 06:55:51 -05:00
|
|
|
if _, ok := requested[element.hash]; ok {
|
|
|
|
t.Errorf("hash %x already requested once", element.hash)
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2022-07-15 06:55:51 -05:00
|
|
|
requested[element.hash] = struct{}{}
|
2015-10-05 11:37:56 -05:00
|
|
|
|
2022-07-15 06:55:51 -05:00
|
|
|
results[i] = NodeSyncResult{element.path, data}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2020-08-21 07:10:40 -05:00
|
|
|
for _, result := range results {
|
2022-07-15 06:55:51 -05:00
|
|
|
if err := sched.ProcessNode(result); err != nil {
|
2020-08-21 07:10:40 -05:00
|
|
|
t.Fatalf("failed to process result %v", err)
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2019-10-28 12:50:11 -05:00
|
|
|
batch := diskdb.NewBatch()
|
|
|
|
if err := sched.Commit(batch); err != nil {
|
|
|
|
t.Fatalf("failed to commit data: %v", err)
|
eth/downloader: separate state sync from queue (#14460)
* eth/downloader: separate state sync from queue
Scheduling of state node downloads hogged the downloader queue lock when
new requests were scheduled. This caused timeouts for other requests.
With this change, state sync is fully independent of all other downloads
and doesn't involve the queue at all.
State sync is started and checked on in processContent. This is slightly
awkward because processContent doesn't have a select loop. Instead, the
queue is closed by an auxiliary goroutine when state sync fails. We
tried several alternatives to this but settled on the current approach
because it's the least amount of change overall.
Handling of the pivot block has changed slightly: the queue previously
prevented import of pivot block receipts before the state of the pivot
block was available. In this commit, the receipt will be imported before
the state. This causes an annoyance where the pivot block is committed
as fast block head even when state downloads fail. Stay tuned for more
updates in this area ;)
* eth/downloader: remove cancelTimeout channel
* eth/downloader: retry state requests on timeout
* eth/downloader: improve comment
* eth/downloader: mark peers idle when state sync is done
* eth/downloader: move pivot block splitting to processContent
This change also ensures that pivot block receipts aren't imported
before the pivot block itself.
* eth/downloader: limit state node retries
* eth/downloader: improve state node error handling and retry check
* eth/downloader: remove maxStateNodeRetries
It fails the sync too much.
* eth/downloader: remove last use of cancelCh in statesync.go
Fixes TestDeliverHeadersHang*Fast and (hopefully)
the weird cancellation behaviour at the end of fast sync.
* eth/downloader: fix leak in runStateSync
* eth/downloader: don't run processFullSyncContent in LightSync mode
* eth/downloader: improve comments
* eth/downloader: fix vet, megacheck
* eth/downloader: remove unrequested tasks anyway
* eth/downloader, trie: various polishes around duplicate items
This commit explicitly tracks duplicate and unexpected state
delieveries done against a trie Sync structure, also adding
there to import info logs.
The commit moves the db batch used to commit trie changes one
level deeper so its flushed after every node insertion. This
is needed to avoid a lot of duplicate retrievals caused by
inconsistencies between Sync internals and database. A better
approach is to track not-yet-written states in trie.Sync and
flush on commit, but I'm focuing on correctness first now.
The commit fixes a regression around pivot block fail count.
The counter previously was reset to 1 if and only if a sync
cycle progressed (inserted at least 1 entry to the database).
The current code reset it already if a node was delivered,
which is not stong enough, because unless it ends up written
to disk, an attacker can just loop and attack ad infinitum.
The commit also fixes a regression around state deliveries
and timeouts. The old downloader tracked if a delivery is
stale (none of the deliveries were requestedt), in which
case it didn't mark the node idle and did not send further
requests, since it signals a past timeout. The current code
did mark it idle even on stale deliveries, which eventually
caused two requests to be in flight at the same time, making
the deliveries always stale and mass duplicating retrievals
between multiple peers.
* eth/downloader: fix state request leak
This commit fixes the hang seen sometimes while doing the state
sync. The cause of the hang was a rare combination of events:
request state data from peer, peer drops and reconnects almost
immediately. This caused a new download task to be assigned to
the peer, overwriting the old one still waiting for a timeout,
which in turned leaked the requests out, never to be retried.
The fix is to ensure that a task assignment moves any pending
one back into the retry queue.
The commit also fixes a regression with peer dropping due to
stalls. The current code considered a peer stalling if they
timed out delivering 1 item. However, the downloader never
requests only one, the minimum is 2 (attempt to fine tune
estimated latency/bandwidth). The fix is simply to drop if
a timeout is detected at 2 items.
Apart from the above bugfixes, the commit contains some code
polishes I made while debugging the hang.
* core, eth, trie: support batched trie sync db writes
* trie: rename SyncMemCache to syncMemBatch
2017-06-22 07:26:03 -05:00
|
|
|
}
|
2019-10-28 12:50:11 -05:00
|
|
|
batch.Write()
|
2020-08-28 02:50:37 -05:00
|
|
|
|
2022-07-15 06:55:51 -05:00
|
|
|
paths, nodes, _ = sched.Missing(0)
|
|
|
|
elements = elements[:0]
|
|
|
|
for i := 0; i < len(paths); i++ {
|
|
|
|
elements = append(elements, trieElement{
|
|
|
|
path: paths[i],
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(paths[i])),
|
|
|
|
})
|
|
|
|
}
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2015-12-28 07:20:37 -06:00
|
|
|
// Cross check that the two tries are in sync
|
2023-05-09 02:11:04 -05:00
|
|
|
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), srcData)
|
2015-10-05 11:37:56 -05:00
|
|
|
}
|
2015-12-28 07:20:37 -06:00
|
|
|
|
|
|
|
// Tests that at any point in time during a sync, only complete sub-tries are in
|
|
|
|
// the database.
|
2023-05-09 02:11:04 -05:00
|
|
|
func TestIncompleteSyncHash(t *testing.T) {
|
|
|
|
testIncompleteSync(t, rawdb.HashScheme)
|
2023-08-01 07:17:32 -05:00
|
|
|
testIncompleteSync(t, rawdb.PathScheme)
|
2023-05-09 02:11:04 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
func testIncompleteSync(t *testing.T, scheme string) {
|
2015-12-28 07:20:37 -06:00
|
|
|
// Create a random trie to copy
|
2023-05-09 02:11:04 -05:00
|
|
|
_, srcDb, srcTrie, _ := makeTestTrie(scheme)
|
2015-12-28 07:20:37 -06:00
|
|
|
|
|
|
|
// Create a destination trie and sync with the scheduler
|
2022-11-28 07:31:28 -06:00
|
|
|
diskdb := rawdb.NewMemoryDatabase()
|
|
|
|
sched := NewSync(srcTrie.Hash(), diskdb, nil, srcDb.Scheme())
|
2015-12-28 07:20:37 -06:00
|
|
|
|
2022-07-15 06:55:51 -05:00
|
|
|
// The code requests are ignored here since there is no code
|
|
|
|
// at the testing trie.
|
|
|
|
var (
|
2023-05-09 02:11:04 -05:00
|
|
|
addedKeys []string
|
|
|
|
addedHashes []common.Hash
|
|
|
|
elements []trieElement
|
|
|
|
root = srcTrie.Hash()
|
2022-07-15 06:55:51 -05:00
|
|
|
)
|
|
|
|
paths, nodes, _ := sched.Missing(1)
|
|
|
|
for i := 0; i < len(paths); i++ {
|
|
|
|
elements = append(elements, trieElement{
|
|
|
|
path: paths[i],
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(paths[i])),
|
|
|
|
})
|
|
|
|
}
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
reader, err := srcDb.Reader(srcTrie.Hash())
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("State is not available %x", srcTrie.Hash())
|
|
|
|
}
|
2022-07-15 06:55:51 -05:00
|
|
|
for len(elements) > 0 {
|
2015-12-28 07:20:37 -06:00
|
|
|
// Fetch a batch of trie nodes
|
2022-07-15 06:55:51 -05:00
|
|
|
results := make([]NodeSyncResult, len(elements))
|
|
|
|
for i, element := range elements {
|
2023-05-09 02:11:04 -05:00
|
|
|
owner, inner := ResolvePath([]byte(element.path))
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
data, err := reader.Node(owner, inner, element.hash)
|
2015-12-28 07:20:37 -06:00
|
|
|
if err != nil {
|
2022-07-15 06:55:51 -05:00
|
|
|
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
|
2015-12-28 07:20:37 -06:00
|
|
|
}
|
2022-07-15 06:55:51 -05:00
|
|
|
results[i] = NodeSyncResult{element.path, data}
|
2015-12-28 07:20:37 -06:00
|
|
|
}
|
|
|
|
// Process each of the trie nodes
|
2020-08-21 07:10:40 -05:00
|
|
|
for _, result := range results {
|
2022-07-15 06:55:51 -05:00
|
|
|
if err := sched.ProcessNode(result); err != nil {
|
2020-08-21 07:10:40 -05:00
|
|
|
t.Fatalf("failed to process result %v", err)
|
|
|
|
}
|
2015-12-28 07:20:37 -06:00
|
|
|
}
|
2019-10-28 12:50:11 -05:00
|
|
|
batch := diskdb.NewBatch()
|
|
|
|
if err := sched.Commit(batch); err != nil {
|
|
|
|
t.Fatalf("failed to commit data: %v", err)
|
eth/downloader: separate state sync from queue (#14460)
* eth/downloader: separate state sync from queue
Scheduling of state node downloads hogged the downloader queue lock when
new requests were scheduled. This caused timeouts for other requests.
With this change, state sync is fully independent of all other downloads
and doesn't involve the queue at all.
State sync is started and checked on in processContent. This is slightly
awkward because processContent doesn't have a select loop. Instead, the
queue is closed by an auxiliary goroutine when state sync fails. We
tried several alternatives to this but settled on the current approach
because it's the least amount of change overall.
Handling of the pivot block has changed slightly: the queue previously
prevented import of pivot block receipts before the state of the pivot
block was available. In this commit, the receipt will be imported before
the state. This causes an annoyance where the pivot block is committed
as fast block head even when state downloads fail. Stay tuned for more
updates in this area ;)
* eth/downloader: remove cancelTimeout channel
* eth/downloader: retry state requests on timeout
* eth/downloader: improve comment
* eth/downloader: mark peers idle when state sync is done
* eth/downloader: move pivot block splitting to processContent
This change also ensures that pivot block receipts aren't imported
before the pivot block itself.
* eth/downloader: limit state node retries
* eth/downloader: improve state node error handling and retry check
* eth/downloader: remove maxStateNodeRetries
It fails the sync too much.
* eth/downloader: remove last use of cancelCh in statesync.go
Fixes TestDeliverHeadersHang*Fast and (hopefully)
the weird cancellation behaviour at the end of fast sync.
* eth/downloader: fix leak in runStateSync
* eth/downloader: don't run processFullSyncContent in LightSync mode
* eth/downloader: improve comments
* eth/downloader: fix vet, megacheck
* eth/downloader: remove unrequested tasks anyway
* eth/downloader, trie: various polishes around duplicate items
This commit explicitly tracks duplicate and unexpected state
delieveries done against a trie Sync structure, also adding
there to import info logs.
The commit moves the db batch used to commit trie changes one
level deeper so its flushed after every node insertion. This
is needed to avoid a lot of duplicate retrievals caused by
inconsistencies between Sync internals and database. A better
approach is to track not-yet-written states in trie.Sync and
flush on commit, but I'm focuing on correctness first now.
The commit fixes a regression around pivot block fail count.
The counter previously was reset to 1 if and only if a sync
cycle progressed (inserted at least 1 entry to the database).
The current code reset it already if a node was delivered,
which is not stong enough, because unless it ends up written
to disk, an attacker can just loop and attack ad infinitum.
The commit also fixes a regression around state deliveries
and timeouts. The old downloader tracked if a delivery is
stale (none of the deliveries were requestedt), in which
case it didn't mark the node idle and did not send further
requests, since it signals a past timeout. The current code
did mark it idle even on stale deliveries, which eventually
caused two requests to be in flight at the same time, making
the deliveries always stale and mass duplicating retrievals
between multiple peers.
* eth/downloader: fix state request leak
This commit fixes the hang seen sometimes while doing the state
sync. The cause of the hang was a rare combination of events:
request state data from peer, peer drops and reconnects almost
immediately. This caused a new download task to be assigned to
the peer, overwriting the old one still waiting for a timeout,
which in turned leaked the requests out, never to be retried.
The fix is to ensure that a task assignment moves any pending
one back into the retry queue.
The commit also fixes a regression with peer dropping due to
stalls. The current code considered a peer stalling if they
timed out delivering 1 item. However, the downloader never
requests only one, the minimum is 2 (attempt to fine tune
estimated latency/bandwidth). The fix is simply to drop if
a timeout is detected at 2 items.
Apart from the above bugfixes, the commit contains some code
polishes I made while debugging the hang.
* core, eth, trie: support batched trie sync db writes
* trie: rename SyncMemCache to syncMemBatch
2017-06-22 07:26:03 -05:00
|
|
|
}
|
2019-10-28 12:50:11 -05:00
|
|
|
batch.Write()
|
2022-07-15 06:55:51 -05:00
|
|
|
|
2015-12-28 07:20:37 -06:00
|
|
|
for _, result := range results {
|
2022-07-15 06:55:51 -05:00
|
|
|
hash := crypto.Keccak256Hash(result.Data)
|
|
|
|
if hash != root {
|
2023-05-09 02:11:04 -05:00
|
|
|
addedKeys = append(addedKeys, result.Path)
|
|
|
|
addedHashes = append(addedHashes, crypto.Keccak256Hash(result.Data))
|
2015-12-28 07:20:37 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
// Fetch the next batch to retrieve
|
2022-07-15 06:55:51 -05:00
|
|
|
paths, nodes, _ = sched.Missing(1)
|
|
|
|
elements = elements[:0]
|
|
|
|
for i := 0; i < len(paths); i++ {
|
|
|
|
elements = append(elements, trieElement{
|
|
|
|
path: paths[i],
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(paths[i])),
|
|
|
|
})
|
|
|
|
}
|
2015-12-28 07:20:37 -06:00
|
|
|
}
|
|
|
|
// Sanity check that removing any node from the database is detected
|
2023-05-09 02:11:04 -05:00
|
|
|
for i, path := range addedKeys {
|
|
|
|
owner, inner := ResolvePath([]byte(path))
|
|
|
|
nodeHash := addedHashes[i]
|
|
|
|
value := rawdb.ReadTrieNode(diskdb, owner, inner, nodeHash, scheme)
|
|
|
|
rawdb.DeleteTrieNode(diskdb, owner, inner, nodeHash, scheme)
|
|
|
|
if err := checkTrieConsistency(diskdb, srcDb.Scheme(), root); err == nil {
|
|
|
|
t.Fatalf("trie inconsistency not caught, missing: %x", path)
|
|
|
|
}
|
|
|
|
rawdb.WriteTrieNode(diskdb, owner, inner, nodeHash, value, scheme)
|
2015-12-28 07:20:37 -06:00
|
|
|
}
|
|
|
|
}
|
2020-08-28 02:50:37 -05:00
|
|
|
|
|
|
|
// Tests that trie nodes get scheduled lexicographically when having the same
|
|
|
|
// depth.
|
|
|
|
func TestSyncOrdering(t *testing.T) {
|
2023-05-09 02:11:04 -05:00
|
|
|
testSyncOrdering(t, rawdb.HashScheme)
|
2023-08-01 07:17:32 -05:00
|
|
|
testSyncOrdering(t, rawdb.PathScheme)
|
2023-05-09 02:11:04 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
func testSyncOrdering(t *testing.T, scheme string) {
|
2020-08-28 02:50:37 -05:00
|
|
|
// Create a random trie to copy
|
2023-05-09 02:11:04 -05:00
|
|
|
_, srcDb, srcTrie, srcData := makeTestTrie(scheme)
|
2020-08-28 02:50:37 -05:00
|
|
|
|
|
|
|
// Create a destination trie and sync with the scheduler, tracking the requests
|
2022-11-28 07:31:28 -06:00
|
|
|
diskdb := rawdb.NewMemoryDatabase()
|
|
|
|
sched := NewSync(srcTrie.Hash(), diskdb, nil, srcDb.Scheme())
|
2020-08-28 02:50:37 -05:00
|
|
|
|
2022-07-15 06:55:51 -05:00
|
|
|
// The code requests are ignored here since there is no code
|
|
|
|
// at the testing trie.
|
|
|
|
var (
|
|
|
|
reqs []SyncPath
|
|
|
|
elements []trieElement
|
|
|
|
)
|
|
|
|
paths, nodes, _ := sched.Missing(1)
|
|
|
|
for i := 0; i < len(paths); i++ {
|
|
|
|
elements = append(elements, trieElement{
|
|
|
|
path: paths[i],
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(paths[i])),
|
|
|
|
})
|
|
|
|
reqs = append(reqs, NewSyncPath([]byte(paths[i])))
|
|
|
|
}
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
reader, err := srcDb.Reader(srcTrie.Hash())
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("State is not available %x", srcTrie.Hash())
|
|
|
|
}
|
2022-07-15 06:55:51 -05:00
|
|
|
for len(elements) > 0 {
|
|
|
|
results := make([]NodeSyncResult, len(elements))
|
|
|
|
for i, element := range elements {
|
2023-05-09 02:11:04 -05:00
|
|
|
owner, inner := ResolvePath([]byte(element.path))
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
data, err := reader.Node(owner, inner, element.hash)
|
2020-08-28 02:50:37 -05:00
|
|
|
if err != nil {
|
2022-07-15 06:55:51 -05:00
|
|
|
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
|
2020-08-28 02:50:37 -05:00
|
|
|
}
|
2022-07-15 06:55:51 -05:00
|
|
|
results[i] = NodeSyncResult{element.path, data}
|
2020-08-28 02:50:37 -05:00
|
|
|
}
|
|
|
|
for _, result := range results {
|
2022-07-15 06:55:51 -05:00
|
|
|
if err := sched.ProcessNode(result); err != nil {
|
2020-08-28 02:50:37 -05:00
|
|
|
t.Fatalf("failed to process result %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
batch := diskdb.NewBatch()
|
|
|
|
if err := sched.Commit(batch); err != nil {
|
|
|
|
t.Fatalf("failed to commit data: %v", err)
|
|
|
|
}
|
|
|
|
batch.Write()
|
|
|
|
|
2022-07-15 06:55:51 -05:00
|
|
|
paths, nodes, _ = sched.Missing(1)
|
|
|
|
elements = elements[:0]
|
|
|
|
for i := 0; i < len(paths); i++ {
|
|
|
|
elements = append(elements, trieElement{
|
|
|
|
path: paths[i],
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(paths[i])),
|
|
|
|
})
|
|
|
|
reqs = append(reqs, NewSyncPath([]byte(paths[i])))
|
|
|
|
}
|
2020-08-28 02:50:37 -05:00
|
|
|
}
|
|
|
|
// Cross check that the two tries are in sync
|
2023-05-09 02:11:04 -05:00
|
|
|
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), srcData)
|
2020-08-28 02:50:37 -05:00
|
|
|
|
|
|
|
// Check that the trie nodes have been requested path-ordered
|
|
|
|
for i := 0; i < len(reqs)-1; i++ {
|
|
|
|
if len(reqs[i]) > 1 || len(reqs[i+1]) > 1 {
|
|
|
|
// In the case of the trie tests, there's no storage so the tuples
|
|
|
|
// must always be single items. 2-tuples should be tested in state.
|
|
|
|
t.Errorf("Invalid request tuples: len(%v) or len(%v) > 1", reqs[i], reqs[i+1])
|
|
|
|
}
|
|
|
|
if bytes.Compare(compactToHex(reqs[i][0]), compactToHex(reqs[i+1][0])) > 0 {
|
|
|
|
t.Errorf("Invalid request order: %v before %v", compactToHex(reqs[i][0]), compactToHex(reqs[i+1][0]))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2023-05-09 02:11:04 -05:00
|
|
|
|
|
|
|
func syncWith(t *testing.T, root common.Hash, db ethdb.Database, srcDb *Database) {
|
|
|
|
// Create a destination trie and sync with the scheduler
|
|
|
|
sched := NewSync(root, db, nil, srcDb.Scheme())
|
|
|
|
|
|
|
|
// The code requests are ignored here since there is no code
|
|
|
|
// at the testing trie.
|
|
|
|
paths, nodes, _ := sched.Missing(1)
|
|
|
|
var elements []trieElement
|
|
|
|
for i := 0; i < len(paths); i++ {
|
|
|
|
elements = append(elements, trieElement{
|
|
|
|
path: paths[i],
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(paths[i])),
|
|
|
|
})
|
|
|
|
}
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
reader, err := srcDb.Reader(root)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("State is not available %x", root)
|
|
|
|
}
|
2023-05-09 02:11:04 -05:00
|
|
|
for len(elements) > 0 {
|
|
|
|
results := make([]NodeSyncResult, len(elements))
|
|
|
|
for i, element := range elements {
|
|
|
|
owner, inner := ResolvePath([]byte(element.path))
|
cmd, core/state, eth, tests, trie: improve state reader (#27428)
The state availability is checked during the creation of a state reader.
- In hash-based database, if the specified root node does not exist on disk disk, then
the state reader won't be created and an error will be returned.
- In path-based database, if the specified state layer is not available, then the
state reader won't be created and an error will be returned.
This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error.
2023-06-20 14:31:45 -05:00
|
|
|
data, err := reader.Node(owner, inner, element.hash)
|
2023-05-09 02:11:04 -05:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("failed to retrieve node data for hash %x: %v", element.hash, err)
|
|
|
|
}
|
|
|
|
results[i] = NodeSyncResult{element.path, data}
|
|
|
|
}
|
|
|
|
for index, result := range results {
|
|
|
|
if err := sched.ProcessNode(result); err != nil {
|
|
|
|
t.Fatalf("failed to process result[%d][%v] data %v %v", index, []byte(result.Path), result.Data, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
batch := db.NewBatch()
|
|
|
|
if err := sched.Commit(batch); err != nil {
|
|
|
|
t.Fatalf("failed to commit data: %v", err)
|
|
|
|
}
|
|
|
|
batch.Write()
|
|
|
|
|
|
|
|
paths, nodes, _ = sched.Missing(1)
|
|
|
|
elements = elements[:0]
|
|
|
|
for i := 0; i < len(paths); i++ {
|
|
|
|
elements = append(elements, trieElement{
|
|
|
|
path: paths[i],
|
|
|
|
hash: nodes[i],
|
|
|
|
syncPath: NewSyncPath([]byte(paths[i])),
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Tests that the syncing target is keeping moving which may overwrite the stale
|
|
|
|
// states synced in the last cycle.
|
|
|
|
func TestSyncMovingTarget(t *testing.T) {
|
|
|
|
testSyncMovingTarget(t, rawdb.HashScheme)
|
2023-08-01 07:17:32 -05:00
|
|
|
testSyncMovingTarget(t, rawdb.PathScheme)
|
2023-05-09 02:11:04 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
func testSyncMovingTarget(t *testing.T, scheme string) {
|
|
|
|
// Create a random trie to copy
|
|
|
|
_, srcDb, srcTrie, srcData := makeTestTrie(scheme)
|
|
|
|
|
|
|
|
// Create a destination trie and sync with the scheduler
|
|
|
|
diskdb := rawdb.NewMemoryDatabase()
|
|
|
|
syncWith(t, srcTrie.Hash(), diskdb, srcDb)
|
|
|
|
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), srcData)
|
|
|
|
|
|
|
|
// Push more modifications into the src trie, to see if dest trie can still
|
|
|
|
// sync with it(overwrite stale states)
|
|
|
|
var (
|
|
|
|
preRoot = srcTrie.Hash()
|
|
|
|
diff = make(map[string][]byte)
|
|
|
|
)
|
|
|
|
for i := byte(0); i < 10; i++ {
|
|
|
|
key, val := randBytes(32), randBytes(32)
|
|
|
|
srcTrie.MustUpdate(key, val)
|
|
|
|
diff[string(key)] = val
|
|
|
|
}
|
2023-06-27 07:36:38 -05:00
|
|
|
root, nodes, _ := srcTrie.Commit(false)
|
2023-07-24 05:22:09 -05:00
|
|
|
if err := srcDb.Update(root, preRoot, 0, trienode.NewWithNodeSet(nodes), nil); err != nil {
|
2023-05-09 02:11:04 -05:00
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
if err := srcDb.Commit(root, false); err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
preRoot = root
|
|
|
|
srcTrie, _ = NewStateTrie(TrieID(root), srcDb)
|
|
|
|
|
|
|
|
syncWith(t, srcTrie.Hash(), diskdb, srcDb)
|
|
|
|
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), diff)
|
|
|
|
|
|
|
|
// Revert added modifications from the src trie, to see if dest trie can still
|
|
|
|
// sync with it(overwrite reverted states)
|
|
|
|
var reverted = make(map[string][]byte)
|
|
|
|
for k := range diff {
|
|
|
|
srcTrie.MustDelete([]byte(k))
|
|
|
|
reverted[k] = nil
|
|
|
|
}
|
|
|
|
for k := range srcData {
|
|
|
|
val := randBytes(32)
|
|
|
|
srcTrie.MustUpdate([]byte(k), val)
|
|
|
|
reverted[k] = val
|
|
|
|
}
|
2023-06-27 07:36:38 -05:00
|
|
|
root, nodes, _ = srcTrie.Commit(false)
|
2023-07-24 05:22:09 -05:00
|
|
|
if err := srcDb.Update(root, preRoot, 0, trienode.NewWithNodeSet(nodes), nil); err != nil {
|
2023-05-09 02:11:04 -05:00
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
if err := srcDb.Commit(root, false); err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
srcTrie, _ = NewStateTrie(TrieID(root), srcDb)
|
|
|
|
|
|
|
|
syncWith(t, srcTrie.Hash(), diskdb, srcDb)
|
|
|
|
checkTrieContents(t, diskdb, srcDb.Scheme(), srcTrie.Hash().Bytes(), reverted)
|
|
|
|
}
|