2022-08-08 04:08:36 -05:00
|
|
|
// Copyright 2022 The go-ethereum Authors
|
|
|
|
// This file is part of the go-ethereum library.
|
|
|
|
//
|
|
|
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
|
|
|
// it under the terms of the GNU Lesser General Public License as published by
|
|
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
|
|
// (at your option) any later version.
|
|
|
|
//
|
|
|
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
// GNU Lesser General Public License for more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU Lesser General Public License
|
|
|
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
package rawdb
|
|
|
|
|
2024-04-30 04:33:22 -05:00
|
|
|
import (
|
|
|
|
"path/filepath"
|
|
|
|
|
|
|
|
"github.com/ethereum/go-ethereum/ethdb"
|
|
|
|
)
|
2023-08-01 07:17:32 -05:00
|
|
|
|
2022-08-08 04:08:36 -05:00
|
|
|
// The list of table names of chain freezer.
|
|
|
|
const (
|
2023-02-21 04:17:34 -06:00
|
|
|
// ChainFreezerHeaderTable indicates the name of the freezer header table.
|
|
|
|
ChainFreezerHeaderTable = "headers"
|
2022-08-08 04:08:36 -05:00
|
|
|
|
2023-02-21 04:17:34 -06:00
|
|
|
// ChainFreezerHashTable indicates the name of the freezer canonical hash table.
|
|
|
|
ChainFreezerHashTable = "hashes"
|
2022-08-08 04:08:36 -05:00
|
|
|
|
2023-02-21 04:17:34 -06:00
|
|
|
// ChainFreezerBodiesTable indicates the name of the freezer block body table.
|
|
|
|
ChainFreezerBodiesTable = "bodies"
|
2022-08-08 04:08:36 -05:00
|
|
|
|
2023-02-21 04:17:34 -06:00
|
|
|
// ChainFreezerReceiptTable indicates the name of the freezer receipts table.
|
|
|
|
ChainFreezerReceiptTable = "receipts"
|
2022-08-08 04:08:36 -05:00
|
|
|
)
|
|
|
|
|
|
|
|
// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables.
|
|
|
|
// Hashes and difficulties don't compress well.
|
|
|
|
var chainFreezerNoSnappy = map[string]bool{
|
all: nuke total difficulty (#30744)
The total difficulty is the sum of all block difficulties from genesis
to a certain block. This value was used in PoW for deciding which chain
is heavier, and thus which chain to select. Since PoS has a different
fork selection algorithm, all blocks since the merge have a difficulty
of 0, and all total difficulties are the same for the past 2 years.
Whilst the TDs are mostly useless nowadays, there was never really a
reason to mess around removing them since they are so tiny. This
reasoning changes when we go down the path of pruned chain history. In
order to reconstruct any TD, we **must** retrieve all the headers from
chain head to genesis and then iterate all the difficulties to compute
the TD.
In a world where we completely prune past chain segments (bodies,
receipts, headers), it is not possible to reconstruct the TD at all. In
a world where we still keep chain headers and prune only the rest,
reconstructing it possible as long as we process (or download) the chain
forward from genesis, but trying to snap sync the head first and
backfill later hits the same issue, the TD becomes impossible to
calculate until genesis is backfilled.
All in all, the TD is a messy out-of-state, out-of-consensus computed
field that is overall useless nowadays, but code relying on it forces
the client into certain modes of operation and prevents other modes or
other optimizations. This PR completely nukes out the TD from the node.
It doesn't compute it, it doesn't operate on it, it's as if it didn't
even exist.
Caveats:
- Whenever we have APIs that return TD (devp2p handshake, tracer, etc.)
we return a TD of 0.
- For era files, we recompute the TD during export time (fairly quick)
to retain the format content.
- It is not possible to "verify" the merge point (i.e. with TD gone, TTD
is useless). Since we're not verifying PoW any more, just blindly trust
it, not verifying but blindly trusting the many year old merge point
seems just the same trust model.
- Our tests still need to be able to generate pre and post merge blocks,
so they need a new way to split the merge without TTD. The PR introduces
a settable ttdBlock field on the consensus object which is used by tests
as the block where originally the TTD happened. This is not needed for
live nodes, we never want to generate old blocks.
- One merge transition consensus test was disabled. With a
non-operational TD, testing how the client reacts to TTD is useless, it
cannot react.
Questions:
- Should we also drop total terminal difficulty from the genesis json?
It's a number we cannot react on any more, so maybe it would be cleaner
to get rid of even more concepts.
---------
Co-authored-by: Gary Rong <garyrong0905@gmail.com>
2025-01-28 11:55:41 -06:00
|
|
|
ChainFreezerHeaderTable: false,
|
|
|
|
ChainFreezerHashTable: true,
|
|
|
|
ChainFreezerBodiesTable: false,
|
|
|
|
ChainFreezerReceiptTable: false,
|
2022-08-08 04:08:36 -05:00
|
|
|
}
|
|
|
|
|
2023-08-01 07:17:32 -05:00
|
|
|
const (
|
|
|
|
// stateHistoryTableSize defines the maximum size of freezer data files.
|
|
|
|
stateHistoryTableSize = 2 * 1000 * 1000 * 1000
|
|
|
|
|
|
|
|
// stateHistoryAccountIndex indicates the name of the freezer state history table.
|
|
|
|
stateHistoryMeta = "history.meta"
|
|
|
|
stateHistoryAccountIndex = "account.index"
|
|
|
|
stateHistoryStorageIndex = "storage.index"
|
|
|
|
stateHistoryAccountData = "account.data"
|
|
|
|
stateHistoryStorageData = "storage.data"
|
|
|
|
)
|
|
|
|
|
core/rawdb: introduce flush offset in freezer (#30392)
This is a follow-up PR to #29792 to get rid of the data file sync.
**This is a non-backward compatible change, which increments the
database version from 8 to 9**.
We introduce a flushOffset for each freezer table, which tracks the position
of the most recently fsync’d item in the index file. When this offset moves
forward, it indicates that all index entries below it, along with their corresponding
data items, have been properly persisted to disk. The offset can also be moved
backward when truncating from either the head or tail of the file.
Previously, the data file required an explicit fsync after every mutation, which
was highly inefficient. With the introduction of the flush offset, the synchronization
strategy becomes more flexible, allowing the freezer to sync every 30 seconds
instead.
The data items above the flush offset are regarded volatile and callers must ensure
they are recoverable after the unclean shutdown, or explicitly sync the freezer
before any proceeding operations.
---------
Co-authored-by: Felix Lange <fjl@twurst.com>
2025-02-04 04:45:45 -06:00
|
|
|
// stateFreezerNoSnappy configures whether compression is disabled for the state freezer.
|
all: activate pbss as experimental feature (#26274)
* all: activate pbss
* core/rawdb: fix compilation error
* cma, core, eth, les, trie: address comments
* cmd, core, eth, trie: polish code
* core, cmd, eth: address comments
* cmd, core, eth, les, light, tests: address comment
* cmd/utils: shorten log message
* trie/triedb/pathdb: limit node buffer size to 1gb
* cmd/utils: fix opening non-existing db
* cmd/utils: rename flag name
* cmd, core: group chain history flags and fix tests
* core, eth, trie: fix memory leak in snapshot generation
* cmd, eth, internal: deprecate flags
* all: enable state tests for pathdb, fixes
* cmd, core: polish code
* trie/triedb/pathdb: limit the node buffer size to 256mb
---------
Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Péter Szilágyi <peterke@gmail.com>
2023-08-10 14:21:36 -05:00
|
|
|
var stateFreezerNoSnappy = map[string]bool{
|
2023-08-01 07:17:32 -05:00
|
|
|
stateHistoryMeta: true,
|
|
|
|
stateHistoryAccountIndex: false,
|
|
|
|
stateHistoryStorageIndex: false,
|
|
|
|
stateHistoryAccountData: false,
|
|
|
|
stateHistoryStorageData: false,
|
|
|
|
}
|
|
|
|
|
2022-08-08 04:08:36 -05:00
|
|
|
// The list of identifiers of ancient stores.
|
|
|
|
var (
|
2024-07-16 08:17:58 -05:00
|
|
|
ChainFreezerName = "chain" // the folder name of chain segment ancient store.
|
|
|
|
MerkleStateFreezerName = "state" // the folder name of state history ancient store.
|
|
|
|
VerkleStateFreezerName = "state_verkle" // the folder name of state history ancient store.
|
2022-08-08 04:08:36 -05:00
|
|
|
)
|
|
|
|
|
|
|
|
// freezers the collections of all builtin freezers.
|
2024-07-16 08:17:58 -05:00
|
|
|
var freezers = []string{ChainFreezerName, MerkleStateFreezerName, VerkleStateFreezerName}
|
2023-08-01 07:17:32 -05:00
|
|
|
|
2024-04-30 04:33:22 -05:00
|
|
|
// NewStateFreezer initializes the ancient store for state history.
|
|
|
|
//
|
|
|
|
// - if the empty directory is given, initializes the pure in-memory
|
|
|
|
// state freezer (e.g. dev mode).
|
|
|
|
// - if non-empty directory is given, initializes the regular file-based
|
|
|
|
// state freezer.
|
2024-07-16 08:17:58 -05:00
|
|
|
func NewStateFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.ResettableAncientStore, error) {
|
2024-04-30 04:33:22 -05:00
|
|
|
if ancientDir == "" {
|
|
|
|
return NewMemoryFreezer(readOnly, stateFreezerNoSnappy), nil
|
|
|
|
}
|
2024-07-16 08:17:58 -05:00
|
|
|
var name string
|
|
|
|
if verkle {
|
|
|
|
name = filepath.Join(ancientDir, VerkleStateFreezerName)
|
|
|
|
} else {
|
|
|
|
name = filepath.Join(ancientDir, MerkleStateFreezerName)
|
|
|
|
}
|
|
|
|
return newResettableFreezer(name, "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy)
|
2023-08-01 07:17:32 -05:00
|
|
|
}
|