interesting
/
go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

freezer: add readonly flag to table

This commit is contained in:
Sina Mahmoodi 2021-12-15 15:18:27 +03:30
parent b1e72f7ea9
commit 63c9ce9620
3 changed files with 34 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/rawdb/freezer.go
View File

@ -133,7 +133,7 @@ func newFreezer(datadir string, namespace string, readonly bool, maxTableSize ui
// Create the tables. // Create the tables.
for name, disableSnappy := range tables { for name, disableSnappy := range tables {
table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, disableSnappy) table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, disableSnappy, readonly)
if err != nil { if err != nil {
for _, table := range freezer.tables { for _, table := range freezer.tables {
table.Close() table.Close()

8
core/rawdb/freezer_table.go
View File

@ -94,7 +94,8 @@ type freezerTable struct {
// so take advantage of that (https://golang.org/pkg/sync/atomic/#pkg-note-BUG). // so take advantage of that (https://golang.org/pkg/sync/atomic/#pkg-note-BUG).
items uint64 // Number of items stored in the table (including items removed from tail) items uint64 // Number of items stored in the table (including items removed from tail)
noCompression bool // if true, disables snappy compression. Note: does not work retroactively noCompression bool // if true, disables snappy compression. Note: does not work retroactively
readonly bool
maxFileSize uint32 // Max file size for data-files maxFileSize uint32 // Max file size for data-files
name string name string
path string path string
@ -120,7 +121,7 @@ type freezerTable struct {
// NewFreezerTable opens the given path as a freezer table. // NewFreezerTable opens the given path as a freezer table.
func NewFreezerTable(path, name string, disableSnappy bool) (*freezerTable, error) { func NewFreezerTable(path, name string, disableSnappy bool) (*freezerTable, error) {
return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy) return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, false)
} }
// openFreezerFileForAppend opens a freezer table file and seeks to the end // openFreezerFileForAppend opens a freezer table file and seeks to the end
@ -164,7 +165,7 @@ func truncateFreezerFile(file *os.File, size int64) error {
// newTable opens a freezer table, creating the data and index files if they are // newTable opens a freezer table, creating the data and index files if they are
// non existent. Both files are truncated to the shortest common length to ensure // non existent. Both files are truncated to the shortest common length to ensure
// they don't go out of sync. // they don't go out of sync.
func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeGauge metrics.Gauge, maxFilesize uint32, noCompression bool) (*freezerTable, error) { func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeGauge metrics.Gauge, maxFilesize uint32, noCompression, readonly bool) (*freezerTable, error) {
// Ensure the containing directory exists and open the indexEntry file // Ensure the containing directory exists and open the indexEntry file
if err := os.MkdirAll(path, 0755); err != nil { if err := os.MkdirAll(path, 0755); err != nil {
return nil, err return nil, err
@ -192,6 +193,7 @@ func newTable(path string, name string, readMeter metrics.Meter, writeMeter metr
path: path, path: path,
logger: log.New("database", path, "table", name), logger: log.New("database", path, "table", name),
noCompression: noCompression, noCompression: noCompression,
readonly: readonly,
maxFileSize: maxFilesize, maxFileSize: maxFilesize,
} }
if err := tab.repair(); err != nil { if err := tab.repair(); err != nil {

56
core/rawdb/freezer_table_test.go
View File

@ -40,7 +40,7 @@ func TestFreezerBasics(t *testing.T) {
// set cutoff at 50 bytes // set cutoff at 50 bytes
f, err := newTable(os.TempDir(), f, err := newTable(os.TempDir(),
fmt.Sprintf("unittest-%d", rand.Uint64()), fmt.Sprintf("unittest-%d", rand.Uint64()),
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true) metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -85,7 +85,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
f *freezerTable f *freezerTable
err error err error
) )
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -99,7 +99,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
require.NoError(t, batch.commit()) require.NoError(t, batch.commit())
f.Close() f.Close()
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -116,7 +116,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
t.Fatalf("test %d, got \n%x != \n%x", y, got, exp) t.Fatalf("test %d, got \n%x != \n%x", y, got, exp)
} }
f.Close() f.Close()
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -131,7 +131,7 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
// Fill table // Fill table
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -160,7 +160,7 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
// Now open it again // Now open it again
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -183,7 +183,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
// Fill a table and close it // Fill a table and close it
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -209,7 +209,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
// Now open it again // Now open it again
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -232,7 +232,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
// And if we open it, we should now be able to read all of them (new values) // And if we open it, we should now be able to read all of them (new values)
{ {
f, _ := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, _ := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
for y := 1; y < 255; y++ { for y := 1; y < 255; y++ {
exp := getChunk(15, ^y) exp := getChunk(15, ^y)
got, err := f.Retrieve(uint64(y)) got, err := f.Retrieve(uint64(y))
@ -254,7 +254,7 @@ func TestSnappyDetection(t *testing.T) {
// Open with snappy // Open with snappy
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -265,7 +265,7 @@ func TestSnappyDetection(t *testing.T) {
// Open without snappy // Open without snappy
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, false) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, false, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -277,7 +277,7 @@ func TestSnappyDetection(t *testing.T) {
// Open with snappy // Open with snappy
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -309,7 +309,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
// Fill a table and close it // Fill a table and close it
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -345,7 +345,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
// 45, 45, 15 // 45, 45, 15
// with 3+3+1 items // with 3+3+1 items
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -366,7 +366,7 @@ func TestFreezerTruncate(t *testing.T) {
// Fill table // Fill table
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -382,7 +382,7 @@ func TestFreezerTruncate(t *testing.T) {
// Reopen, truncate // Reopen, truncate
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -407,7 +407,7 @@ func TestFreezerRepairFirstFile(t *testing.T) {
// Fill table // Fill table
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -440,7 +440,7 @@ func TestFreezerRepairFirstFile(t *testing.T) {
// Reopen // Reopen
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -475,7 +475,7 @@ func TestFreezerReadAndTruncate(t *testing.T) {
// Fill table // Fill table
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -491,7 +491,7 @@ func TestFreezerReadAndTruncate(t *testing.T) {
// Reopen and read all files // Reopen and read all files
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -523,7 +523,7 @@ func TestFreezerOffset(t *testing.T) {
// Fill table // Fill table
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -584,7 +584,7 @@ func TestFreezerOffset(t *testing.T) {
// Now open again // Now open again
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -638,7 +638,7 @@ func TestFreezerOffset(t *testing.T) {
// Check that existing items have been moved to index 1M. // Check that existing items have been moved to index 1M.
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -726,7 +726,7 @@ func TestSequentialRead(t *testing.T) {
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
fname := fmt.Sprintf("batchread-%d", rand.Uint64()) fname := fmt.Sprintf("batchread-%d", rand.Uint64())
{ // Fill table { // Fill table
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -736,7 +736,7 @@ func TestSequentialRead(t *testing.T) {
f.Close() f.Close()
} }
{ // Open it, iterate, verify iteration { // Open it, iterate, verify iteration
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -757,7 +757,7 @@ func TestSequentialRead(t *testing.T) {
} }
{ // Open it, iterate, verify byte limit. The byte limit is less than item { // Open it, iterate, verify byte limit. The byte limit is less than item
// size, so each lookup should only return one item // size, so each lookup should only return one item
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -786,7 +786,7 @@ func TestSequentialReadByteLimit(t *testing.T) {
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
fname := fmt.Sprintf("batchread-2-%d", rand.Uint64()) fname := fmt.Sprintf("batchread-2-%d", rand.Uint64())
{ // Fill table { // Fill table
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -808,7 +808,7 @@ func TestSequentialReadByteLimit(t *testing.T) {
{100, 109, 10}, {100, 109, 10},
} { } {
{ {
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true) f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }