2022-03-10 02:37:23 -06:00
|
|
|
// Copyright 2022 The go-ethereum Authors
|
|
|
|
// This file is part of the go-ethereum library.
|
|
|
|
//
|
|
|
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
|
|
|
// it under the terms of the GNU Lesser General Public License as published by
|
|
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
|
|
// (at your option) any later version.
|
|
|
|
//
|
|
|
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
// GNU Lesser General Public License for more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU Lesser General Public License
|
2022-05-24 13:39:40 -05:00
|
|
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
2022-03-10 02:37:23 -06:00
|
|
|
|
|
|
|
package rawdb
|
|
|
|
|
|
|
|
import (
|
|
|
|
"os"
|
|
|
|
"testing"
|
core/rawdb: introduce flush offset in freezer (#30392)
This is a follow-up PR to #29792 to get rid of the data file sync.
**This is a non-backward compatible change, which increments the
database version from 8 to 9**.
We introduce a flushOffset for each freezer table, which tracks the position
of the most recently fsync’d item in the index file. When this offset moves
forward, it indicates that all index entries below it, along with their corresponding
data items, have been properly persisted to disk. The offset can also be moved
backward when truncating from either the head or tail of the file.
Previously, the data file required an explicit fsync after every mutation, which
was highly inefficient. With the introduction of the flush offset, the synchronization
strategy becomes more flexible, allowing the freezer to sync every 30 seconds
instead.
The data items above the flush offset are regarded volatile and callers must ensure
they are recoverable after the unclean shutdown, or explicitly sync the freezer
before any proceeding operations.
---------
Co-authored-by: Felix Lange <fjl@twurst.com>
2025-02-04 04:45:45 -06:00
|
|
|
|
|
|
|
"github.com/ethereum/go-ethereum/rlp"
|
2022-03-10 02:37:23 -06:00
|
|
|
)
|
|
|
|
|
|
|
|
func TestReadWriteFreezerTableMeta(t *testing.T) {
|
2024-07-15 08:26:58 -05:00
|
|
|
f, err := os.CreateTemp(t.TempDir(), "*")
|
2022-03-10 02:37:23 -06:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("Failed to create file %v", err)
|
|
|
|
}
|
2024-07-15 08:26:58 -05:00
|
|
|
defer f.Close()
|
core/rawdb: introduce flush offset in freezer (#30392)
This is a follow-up PR to #29792 to get rid of the data file sync.
**This is a non-backward compatible change, which increments the
database version from 8 to 9**.
We introduce a flushOffset for each freezer table, which tracks the position
of the most recently fsync’d item in the index file. When this offset moves
forward, it indicates that all index entries below it, along with their corresponding
data items, have been properly persisted to disk. The offset can also be moved
backward when truncating from either the head or tail of the file.
Previously, the data file required an explicit fsync after every mutation, which
was highly inefficient. With the introduction of the flush offset, the synchronization
strategy becomes more flexible, allowing the freezer to sync every 30 seconds
instead.
The data items above the flush offset are regarded volatile and callers must ensure
they are recoverable after the unclean shutdown, or explicitly sync the freezer
before any proceeding operations.
---------
Co-authored-by: Felix Lange <fjl@twurst.com>
2025-02-04 04:45:45 -06:00
|
|
|
|
|
|
|
meta, err := newMetadata(f)
|
2022-03-10 02:37:23 -06:00
|
|
|
if err != nil {
|
core/rawdb: introduce flush offset in freezer (#30392)
This is a follow-up PR to #29792 to get rid of the data file sync.
**This is a non-backward compatible change, which increments the
database version from 8 to 9**.
We introduce a flushOffset for each freezer table, which tracks the position
of the most recently fsync’d item in the index file. When this offset moves
forward, it indicates that all index entries below it, along with their corresponding
data items, have been properly persisted to disk. The offset can also be moved
backward when truncating from either the head or tail of the file.
Previously, the data file required an explicit fsync after every mutation, which
was highly inefficient. With the introduction of the flush offset, the synchronization
strategy becomes more flexible, allowing the freezer to sync every 30 seconds
instead.
The data items above the flush offset are regarded volatile and callers must ensure
they are recoverable after the unclean shutdown, or explicitly sync the freezer
before any proceeding operations.
---------
Co-authored-by: Felix Lange <fjl@twurst.com>
2025-02-04 04:45:45 -06:00
|
|
|
t.Fatalf("Failed to new metadata %v", err)
|
2022-03-10 02:37:23 -06:00
|
|
|
}
|
core/rawdb: introduce flush offset in freezer (#30392)
This is a follow-up PR to #29792 to get rid of the data file sync.
**This is a non-backward compatible change, which increments the
database version from 8 to 9**.
We introduce a flushOffset for each freezer table, which tracks the position
of the most recently fsync’d item in the index file. When this offset moves
forward, it indicates that all index entries below it, along with their corresponding
data items, have been properly persisted to disk. The offset can also be moved
backward when truncating from either the head or tail of the file.
Previously, the data file required an explicit fsync after every mutation, which
was highly inefficient. With the introduction of the flush offset, the synchronization
strategy becomes more flexible, allowing the freezer to sync every 30 seconds
instead.
The data items above the flush offset are regarded volatile and callers must ensure
they are recoverable after the unclean shutdown, or explicitly sync the freezer
before any proceeding operations.
---------
Co-authored-by: Felix Lange <fjl@twurst.com>
2025-02-04 04:45:45 -06:00
|
|
|
meta.setVirtualTail(100, false)
|
|
|
|
|
|
|
|
meta, err = newMetadata(f)
|
2022-03-10 02:37:23 -06:00
|
|
|
if err != nil {
|
core/rawdb: introduce flush offset in freezer (#30392)
This is a follow-up PR to #29792 to get rid of the data file sync.
**This is a non-backward compatible change, which increments the
database version from 8 to 9**.
We introduce a flushOffset for each freezer table, which tracks the position
of the most recently fsync’d item in the index file. When this offset moves
forward, it indicates that all index entries below it, along with their corresponding
data items, have been properly persisted to disk. The offset can also be moved
backward when truncating from either the head or tail of the file.
Previously, the data file required an explicit fsync after every mutation, which
was highly inefficient. With the introduction of the flush offset, the synchronization
strategy becomes more flexible, allowing the freezer to sync every 30 seconds
instead.
The data items above the flush offset are regarded volatile and callers must ensure
they are recoverable after the unclean shutdown, or explicitly sync the freezer
before any proceeding operations.
---------
Co-authored-by: Felix Lange <fjl@twurst.com>
2025-02-04 04:45:45 -06:00
|
|
|
t.Fatalf("Failed to reload metadata %v", err)
|
2022-03-10 02:37:23 -06:00
|
|
|
}
|
core/rawdb: introduce flush offset in freezer (#30392)
This is a follow-up PR to #29792 to get rid of the data file sync.
**This is a non-backward compatible change, which increments the
database version from 8 to 9**.
We introduce a flushOffset for each freezer table, which tracks the position
of the most recently fsync’d item in the index file. When this offset moves
forward, it indicates that all index entries below it, along with their corresponding
data items, have been properly persisted to disk. The offset can also be moved
backward when truncating from either the head or tail of the file.
Previously, the data file required an explicit fsync after every mutation, which
was highly inefficient. With the introduction of the flush offset, the synchronization
strategy becomes more flexible, allowing the freezer to sync every 30 seconds
instead.
The data items above the flush offset are regarded volatile and callers must ensure
they are recoverable after the unclean shutdown, or explicitly sync the freezer
before any proceeding operations.
---------
Co-authored-by: Felix Lange <fjl@twurst.com>
2025-02-04 04:45:45 -06:00
|
|
|
if meta.version != freezerTableV2 {
|
2022-03-10 02:37:23 -06:00
|
|
|
t.Fatalf("Unexpected version field")
|
|
|
|
}
|
core/rawdb: introduce flush offset in freezer (#30392)
This is a follow-up PR to #29792 to get rid of the data file sync.
**This is a non-backward compatible change, which increments the
database version from 8 to 9**.
We introduce a flushOffset for each freezer table, which tracks the position
of the most recently fsync’d item in the index file. When this offset moves
forward, it indicates that all index entries below it, along with their corresponding
data items, have been properly persisted to disk. The offset can also be moved
backward when truncating from either the head or tail of the file.
Previously, the data file required an explicit fsync after every mutation, which
was highly inefficient. With the introduction of the flush offset, the synchronization
strategy becomes more flexible, allowing the freezer to sync every 30 seconds
instead.
The data items above the flush offset are regarded volatile and callers must ensure
they are recoverable after the unclean shutdown, or explicitly sync the freezer
before any proceeding operations.
---------
Co-authored-by: Felix Lange <fjl@twurst.com>
2025-02-04 04:45:45 -06:00
|
|
|
if meta.virtualTail != uint64(100) {
|
2022-03-10 02:37:23 -06:00
|
|
|
t.Fatalf("Unexpected virtual tail field")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
core/rawdb: introduce flush offset in freezer (#30392)
This is a follow-up PR to #29792 to get rid of the data file sync.
**This is a non-backward compatible change, which increments the
database version from 8 to 9**.
We introduce a flushOffset for each freezer table, which tracks the position
of the most recently fsync’d item in the index file. When this offset moves
forward, it indicates that all index entries below it, along with their corresponding
data items, have been properly persisted to disk. The offset can also be moved
backward when truncating from either the head or tail of the file.
Previously, the data file required an explicit fsync after every mutation, which
was highly inefficient. With the introduction of the flush offset, the synchronization
strategy becomes more flexible, allowing the freezer to sync every 30 seconds
instead.
The data items above the flush offset are regarded volatile and callers must ensure
they are recoverable after the unclean shutdown, or explicitly sync the freezer
before any proceeding operations.
---------
Co-authored-by: Felix Lange <fjl@twurst.com>
2025-02-04 04:45:45 -06:00
|
|
|
func TestUpgradeMetadata(t *testing.T) {
|
2024-07-15 08:26:58 -05:00
|
|
|
f, err := os.CreateTemp(t.TempDir(), "*")
|
2022-03-10 02:37:23 -06:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("Failed to create file %v", err)
|
|
|
|
}
|
2024-07-15 08:26:58 -05:00
|
|
|
defer f.Close()
|
core/rawdb: introduce flush offset in freezer (#30392)
This is a follow-up PR to #29792 to get rid of the data file sync.
**This is a non-backward compatible change, which increments the
database version from 8 to 9**.
We introduce a flushOffset for each freezer table, which tracks the position
of the most recently fsync’d item in the index file. When this offset moves
forward, it indicates that all index entries below it, along with their corresponding
data items, have been properly persisted to disk. The offset can also be moved
backward when truncating from either the head or tail of the file.
Previously, the data file required an explicit fsync after every mutation, which
was highly inefficient. With the introduction of the flush offset, the synchronization
strategy becomes more flexible, allowing the freezer to sync every 30 seconds
instead.
The data items above the flush offset are regarded volatile and callers must ensure
they are recoverable after the unclean shutdown, or explicitly sync the freezer
before any proceeding operations.
---------
Co-authored-by: Felix Lange <fjl@twurst.com>
2025-02-04 04:45:45 -06:00
|
|
|
|
|
|
|
// Write legacy metadata into file
|
|
|
|
type obj struct {
|
|
|
|
Version uint16
|
|
|
|
Tail uint64
|
|
|
|
}
|
|
|
|
var o obj
|
|
|
|
o.Version = freezerTableV1
|
|
|
|
o.Tail = 100
|
|
|
|
|
|
|
|
if err := rlp.Encode(f, &o); err != nil {
|
|
|
|
t.Fatalf("Failed to encode %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Reload the metadata, a silent upgrade is expected
|
|
|
|
meta, err := newMetadata(f)
|
2022-03-10 02:37:23 -06:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("Failed to read metadata %v", err)
|
|
|
|
}
|
core/rawdb: introduce flush offset in freezer (#30392)
This is a follow-up PR to #29792 to get rid of the data file sync.
**This is a non-backward compatible change, which increments the
database version from 8 to 9**.
We introduce a flushOffset for each freezer table, which tracks the position
of the most recently fsync’d item in the index file. When this offset moves
forward, it indicates that all index entries below it, along with their corresponding
data items, have been properly persisted to disk. The offset can also be moved
backward when truncating from either the head or tail of the file.
Previously, the data file required an explicit fsync after every mutation, which
was highly inefficient. With the introduction of the flush offset, the synchronization
strategy becomes more flexible, allowing the freezer to sync every 30 seconds
instead.
The data items above the flush offset are regarded volatile and callers must ensure
they are recoverable after the unclean shutdown, or explicitly sync the freezer
before any proceeding operations.
---------
Co-authored-by: Felix Lange <fjl@twurst.com>
2025-02-04 04:45:45 -06:00
|
|
|
if meta.version != freezerTableV1 {
|
|
|
|
t.Fatal("Unexpected version field")
|
2022-03-10 02:37:23 -06:00
|
|
|
}
|
core/rawdb: introduce flush offset in freezer (#30392)
This is a follow-up PR to #29792 to get rid of the data file sync.
**This is a non-backward compatible change, which increments the
database version from 8 to 9**.
We introduce a flushOffset for each freezer table, which tracks the position
of the most recently fsync’d item in the index file. When this offset moves
forward, it indicates that all index entries below it, along with their corresponding
data items, have been properly persisted to disk. The offset can also be moved
backward when truncating from either the head or tail of the file.
Previously, the data file required an explicit fsync after every mutation, which
was highly inefficient. With the introduction of the flush offset, the synchronization
strategy becomes more flexible, allowing the freezer to sync every 30 seconds
instead.
The data items above the flush offset are regarded volatile and callers must ensure
they are recoverable after the unclean shutdown, or explicitly sync the freezer
before any proceeding operations.
---------
Co-authored-by: Felix Lange <fjl@twurst.com>
2025-02-04 04:45:45 -06:00
|
|
|
if meta.virtualTail != uint64(100) {
|
|
|
|
t.Fatal("Unexpected virtual tail field")
|
|
|
|
}
|
|
|
|
if meta.flushOffset != 0 {
|
|
|
|
t.Fatal("Unexpected flush offset field")
|
|
|
|
}
|
|
|
|
|
|
|
|
meta.setFlushOffset(100, true)
|
|
|
|
|
|
|
|
meta, err = newMetadata(f)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("Failed to read metadata %v", err)
|
|
|
|
}
|
|
|
|
if meta.version != freezerTableV2 {
|
|
|
|
t.Fatal("Unexpected version field")
|
|
|
|
}
|
|
|
|
if meta.virtualTail != uint64(100) {
|
|
|
|
t.Fatal("Unexpected virtual tail field")
|
|
|
|
}
|
|
|
|
if meta.flushOffset != 100 {
|
|
|
|
t.Fatal("Unexpected flush offset field")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestInvalidMetadata(t *testing.T) {
|
|
|
|
f, err := os.CreateTemp(t.TempDir(), "*")
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("Failed to create file %v", err)
|
|
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
|
|
|
|
// Write invalid legacy metadata into file
|
|
|
|
type obj struct {
|
|
|
|
Version uint16
|
|
|
|
Tail uint64
|
|
|
|
}
|
|
|
|
var o obj
|
|
|
|
o.Version = freezerTableV2 // -> invalid version tag
|
|
|
|
o.Tail = 100
|
|
|
|
|
|
|
|
if err := rlp.Encode(f, &o); err != nil {
|
|
|
|
t.Fatalf("Failed to encode %v", err)
|
|
|
|
}
|
|
|
|
_, err = newMetadata(f)
|
|
|
|
if err == nil {
|
|
|
|
t.Fatal("Unexpected success")
|
2022-03-10 02:37:23 -06:00
|
|
|
}
|
|
|
|
}
|