From 3b9808f23ca4eb1621a92aad80de5c89269f17fe Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 13:29:34 +0200 Subject: [PATCH 01/10] cmd/geth, cmd/utils: don't use Ethereum for import, export and upgradedb The blockchain commands don't need the full stack. With this change, p2p, miner, downloader, etc are no longer started for blockchain operations. --- cmd/geth/main.go | 113 +++++++++++++-------------------------------- cmd/utils/cmd.go | 51 +++++++++----------- cmd/utils/flags.go | 44 ++++++++---------- 3 files changed, 73 insertions(+), 135 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 0cbf8e41a2..f849063fae 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -41,6 +41,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" ) @@ -282,17 +283,12 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Conso utils.SolcPathFlag, } app.Before = func(ctx *cli.Context) error { + utils.SetupLogger(ctx) if ctx.GlobalBool(utils.PProfEanbledFlag.Name) { utils.StartPProf(ctx) } return nil } - - // missing: - // flag.StringVar(&ConfigFile, "conf", defaultConfigFile, "config file") - // flag.BoolVar(&DiffTool, "difftool", false, "creates output for diff'ing. Sets LogLevel=0") - // flag.StringVar(&DiffType, "diff", "all", "sets the level of diff output [vm, all]. Has no effect if difftool=false") - } func main() { @@ -516,53 +512,25 @@ func importchain(ctx *cli.Context) { if len(ctx.Args()) != 1 { utils.Fatalf("This command requires an argument.") } - - cfg := utils.MakeEthConfig(ClientIdentifier, Version, ctx) - cfg.SkipBcVersionCheck = true - - ethereum, err := eth.New(cfg) - if err != nil { - utils.Fatalf("%v\n", err) - } - - chainmgr := ethereum.ChainManager() + chain, blockDB, stateDB, extraDB := utils.GetChain(ctx) start := time.Now() - err = utils.ImportChain(chainmgr, ctx.Args().First()) - if err != nil { + if err := utils.ImportChain(chain, ctx.Args().First()); err != nil { utils.Fatalf("Import error: %v\n", err) } - - // force database flush - ethereum.BlockDb().Flush() - ethereum.StateDb().Flush() - ethereum.ExtraDb().Flush() - + flushAll(blockDB, stateDB, extraDB) fmt.Printf("Import done in %v", time.Since(start)) - - return } func exportchain(ctx *cli.Context) { if len(ctx.Args()) != 1 { utils.Fatalf("This command requires an argument.") } - - cfg := utils.MakeEthConfig(ClientIdentifier, nodeNameVersion, ctx) - cfg.SkipBcVersionCheck = true - - ethereum, err := eth.New(cfg) - if err != nil { - utils.Fatalf("%v\n", err) - } - - chainmgr := ethereum.ChainManager() + chain, _, _, _ := utils.GetChain(ctx) start := time.Now() - err = utils.ExportChain(chainmgr, ctx.Args().First()) - if err != nil { + if err := utils.ExportChain(chain, ctx.Args().First()); err != nil { utils.Fatalf("Export error: %v\n", err) } fmt.Printf("Export done in %v", time.Since(start)) - return } func removeDb(ctx *cli.Context) { @@ -585,76 +553,54 @@ func removeDb(ctx *cli.Context) { } func upgradeDb(ctx *cli.Context) { - fmt.Println("Upgrade blockchain DB") + glog.Infoln("Upgrading blockchain database") - cfg := utils.MakeEthConfig(ClientIdentifier, Version, ctx) - cfg.SkipBcVersionCheck = true - - ethereum, err := eth.New(cfg) - if err != nil { - utils.Fatalf("%v\n", err) - } - - v, _ := ethereum.BlockDb().Get([]byte("BlockchainVersion")) + chain, blockDB, stateDB, extraDB := utils.GetChain(ctx) + v, _ := blockDB.Get([]byte("BlockchainVersion")) bcVersion := int(common.NewValue(v).Uint()) - if bcVersion == 0 { bcVersion = core.BlockChainVersion } + // Export the current chain. filename := fmt.Sprintf("blockchain_%d_%s.chain", bcVersion, time.Now().Format("20060102_150405")) exportFile := filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), filename) - - err = utils.ExportChain(ethereum.ChainManager(), exportFile) - if err != nil { + if err := utils.ExportChain(chain, exportFile); err != nil { utils.Fatalf("Unable to export chain for reimport %s\n", err) } - - ethereum.BlockDb().Close() - ethereum.StateDb().Close() - ethereum.ExtraDb().Close() - + flushAll(blockDB, stateDB, extraDB) os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "blockchain")) os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "state")) - ethereum, err = eth.New(cfg) - if err != nil { - utils.Fatalf("%v\n", err) - } - - ethereum.BlockDb().Put([]byte("BlockchainVersion"), common.NewValue(core.BlockChainVersion).Bytes()) - - err = utils.ImportChain(ethereum.ChainManager(), exportFile) + // Import the chain file. + chain, blockDB, stateDB, extraDB = utils.GetChain(ctx) + blockDB.Put([]byte("BlockchainVersion"), common.NewValue(core.BlockChainVersion).Bytes()) + err := utils.ImportChain(chain, exportFile) + flushAll(blockDB, stateDB, extraDB) if err != nil { utils.Fatalf("Import error %v (a backup is made in %s, use the import command to import it)\n", err, exportFile) + } else { + os.Remove(exportFile) + glog.Infoln("Import finished") } - - // force database flush - ethereum.BlockDb().Flush() - ethereum.StateDb().Flush() - ethereum.ExtraDb().Flush() - - os.Remove(exportFile) - - fmt.Println("Import finished") } func dump(ctx *cli.Context) { - chainmgr, _, stateDb := utils.GetChain(ctx) + chain, _, stateDB, _ := utils.GetChain(ctx) for _, arg := range ctx.Args() { var block *types.Block if hashish(arg) { - block = chainmgr.GetBlock(common.HexToHash(arg)) + block = chain.GetBlock(common.HexToHash(arg)) } else { num, _ := strconv.Atoi(arg) - block = chainmgr.GetBlockByNumber(uint64(num)) + block = chain.GetBlockByNumber(uint64(num)) } if block == nil { fmt.Println("{}") utils.Fatalf("block not found") } else { - statedb := state.New(block.Root(), stateDb) - fmt.Printf("%s\n", statedb.Dump()) + state := state.New(block.Root(), stateDB) + fmt.Printf("%s\n", state.Dump()) } } } @@ -707,3 +653,10 @@ func hashish(x string) bool { _, err := strconv.Atoi(x) return err != nil } + +func flushAll(dbs ...common.Database) { + for _, db := range dbs { + db.Flush() + db.Close() + } +} diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 39b4e46da2..550ac1c514 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -167,7 +167,7 @@ func FormatTransactionData(data string) []byte { } func ImportChain(chainmgr *core.ChainManager, fn string) error { - fmt.Printf("importing blockchain '%s'\n", fn) + glog.Infoln("Importing blockchain", fn) fh, err := os.OpenFile(fn, os.O_RDONLY, os.ModePerm) if err != nil { return err @@ -176,43 +176,36 @@ func ImportChain(chainmgr *core.ChainManager, fn string) error { chainmgr.Reset() stream := rlp.NewStream(fh, 0) - var i, n int batchSize := 2500 blocks := make(types.Blocks, batchSize) - - for ; ; i++ { - var b types.Block - if err := stream.Decode(&b); err == io.EOF { - break - } else if err != nil { - return fmt.Errorf("at block %d: %v", i, err) - } - - blocks[n] = &b - n++ - - if n == batchSize { - if _, err := chainmgr.InsertChain(blocks); err != nil { - return fmt.Errorf("invalid block %v", err) + n := 0 + for { + // Load a batch of RLP blocks. + i := 0 + for ; i < batchSize; i++ { + var b types.Block + if err := stream.Decode(&b); err == io.EOF { + break + } else if err != nil { + return fmt.Errorf("at block %d: %v", n, err) } - n = 0 - blocks = make(types.Blocks, batchSize) + blocks[i] = &b + n++ + } + if i == 0 { + break + } + // Import the batch. + if _, err := chainmgr.InsertChain(blocks[:i]); err != nil { + return fmt.Errorf("invalid block %d: %v", n, err) } } - - if n > 0 { - if _, err := chainmgr.InsertChain(blocks[:n]); err != nil { - return fmt.Errorf("invalid block %v", err) - } - } - - fmt.Printf("imported %d blocks\n", i) return nil } func ExportChain(chainmgr *core.ChainManager, fn string) error { - fmt.Printf("exporting blockchain '%s'\n", fn) + glog.Infoln("Exporting blockchain to", fn) fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) if err != nil { return err @@ -221,6 +214,6 @@ func ExportChain(chainmgr *core.ChainManager, fn string) error { if err := chainmgr.Export(fh); err != nil { return err } - fmt.Printf("exported blockchain\n") + glog.Infoln("Exported blockchain to", fn) return nil } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 155110ddce..176a546f1d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -283,20 +283,10 @@ func GetNodeKey(ctx *cli.Context) (key *ecdsa.PrivateKey) { } func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config { - // Set verbosity on glog - glog.SetV(ctx.GlobalInt(VerbosityFlag.Name)) - glog.CopyStandardLogTo("INFO") - // Set the log type - //glog.SetToStderr(ctx.GlobalBool(LogToStdErrFlag.Name)) - glog.SetToStderr(true) - // Set the log dir - glog.SetLogDir(ctx.GlobalString(LogFileFlag.Name)) - customName := ctx.GlobalString(IdentityFlag.Name) if len(customName) > 0 { clientID += "/" + customName } - return ð.Config{ Name: common.MakeName(clientID, version), DataDir: ctx.GlobalString(DataDirFlag.Name), @@ -327,32 +317,34 @@ func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config { } } -func GetChain(ctx *cli.Context) (*core.ChainManager, common.Database, common.Database) { - dataDir := ctx.GlobalString(DataDirFlag.Name) +// SetupLogger configures glog from the logging-related command line flags. +func SetupLogger(ctx *cli.Context) { + glog.SetV(ctx.GlobalInt(VerbosityFlag.Name)) + glog.CopyStandardLogTo("INFO") + glog.SetToStderr(true) + glog.SetLogDir(ctx.GlobalString(LogFileFlag.Name)) +} - blockDb, err := ethdb.NewLDBDatabase(filepath.Join(dataDir, "blockchain")) - if err != nil { +func GetChain(ctx *cli.Context) (chain *core.ChainManager, blockDB, stateDB, extraDB common.Database) { + dd := ctx.GlobalString(DataDirFlag.Name) + var err error + if blockDB, err = ethdb.NewLDBDatabase(filepath.Join(dd, "blockchain")); err != nil { Fatalf("Could not open database: %v", err) } - - stateDb, err := ethdb.NewLDBDatabase(filepath.Join(dataDir, "state")) - if err != nil { + if stateDB, err = ethdb.NewLDBDatabase(filepath.Join(dd, "state")); err != nil { Fatalf("Could not open database: %v", err) } - - extraDb, err := ethdb.NewLDBDatabase(filepath.Join(dataDir, "extra")) - if err != nil { + if extraDB, err = ethdb.NewLDBDatabase(filepath.Join(dd, "extra")); err != nil { Fatalf("Could not open database: %v", err) } eventMux := new(event.TypeMux) pow := ethash.New() - chainManager := core.NewChainManager(blockDb, stateDb, pow, eventMux) - txPool := core.NewTxPool(eventMux, chainManager.State, chainManager.GasLimit) - blockProcessor := core.NewBlockProcessor(stateDb, extraDb, pow, txPool, chainManager, eventMux) - chainManager.SetProcessor(blockProcessor) - - return chainManager, blockDb, stateDb + chain = core.NewChainManager(blockDB, stateDB, pow, eventMux) + txpool := core.NewTxPool(eventMux, chain.State, chain.GasLimit) + proc := core.NewBlockProcessor(stateDB, extraDB, pow, txpool, chain, eventMux) + chain.SetProcessor(proc) + return chain, blockDB, stateDB, extraDB } func GetAccountManager(ctx *cli.Context) *accounts.Manager { From 62671c93c4672a8853c1193286d59f60e7065bbe Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 13:34:14 +0200 Subject: [PATCH 02/10] cmd/mist: use utils.SetupLogger --- cmd/mist/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/mist/main.go b/cmd/mist/main.go index 8365fcec15..e8ae9b5e1c 100644 --- a/cmd/mist/main.go +++ b/cmd/mist/main.go @@ -86,6 +86,10 @@ func init() { utils.BlockchainVersionFlag, utils.NetworkIdFlag, } + app.Before = func(ctx *cli.Context) error { + utils.SetupLogger(ctx) + return nil + } } func main() { From 651030c98d0173db272aaee814c99f0a664d992b Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 13:43:49 +0200 Subject: [PATCH 03/10] cmd/geth: move blockchain commands to chaincmd.go --- cmd/geth/{blocktest.go => blocktestcmd.go} | 2 +- cmd/geth/chaincmd.go | 159 +++++++++++++++++++++ cmd/geth/main.go | 153 +------------------- 3 files changed, 167 insertions(+), 147 deletions(-) rename cmd/geth/{blocktest.go => blocktestcmd.go} (98%) create mode 100644 cmd/geth/chaincmd.go diff --git a/cmd/geth/blocktest.go b/cmd/geth/blocktestcmd.go similarity index 98% rename from cmd/geth/blocktest.go rename to cmd/geth/blocktestcmd.go index 81a64b5f24..f4dcb0286c 100644 --- a/cmd/geth/blocktest.go +++ b/cmd/geth/blocktestcmd.go @@ -12,7 +12,7 @@ import ( "github.com/ethereum/go-ethereum/tests" ) -var blocktestCmd = cli.Command{ +var blocktestCommand = cli.Command{ Action: runBlockTest, Name: "blocktest", Usage: `loads a block test file`, diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go new file mode 100644 index 0000000000..e17d1dc9b0 --- /dev/null +++ b/cmd/geth/chaincmd.go @@ -0,0 +1,159 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/codegangsta/cli" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/logger/glog" +) + +var ( + importCommand = cli.Command{ + Action: importChain, + Name: "import", + Usage: `import a blockchain file`, + } + exportCommand = cli.Command{ + Action: exportChain, + Name: "export", + Usage: `export blockchain into file`, + } + upgradedbCommand = cli.Command{ + Action: upgradeDB, + Name: "upgradedb", + Usage: "upgrade chainblock database", + } + removedbCommand = cli.Command{ + Action: removeDB, + Name: "removedb", + Usage: "Remove blockchain and state databases", + } + dumpCommand = cli.Command{ + Action: dump, + Name: "dump", + Usage: `dump a specific block from storage`, + Description: ` +The arguments are interpreted as block numbers or hashes. +Use "ethereum dump 0" to dump the genesis block. +`, + } +) + +func importChain(ctx *cli.Context) { + if len(ctx.Args()) != 1 { + utils.Fatalf("This command requires an argument.") + } + chain, blockDB, stateDB, extraDB := utils.GetChain(ctx) + start := time.Now() + if err := utils.ImportChain(chain, ctx.Args().First()); err != nil { + utils.Fatalf("Import error: %v\n", err) + } + flushAll(blockDB, stateDB, extraDB) + fmt.Printf("Import done in %v", time.Since(start)) +} + +func exportChain(ctx *cli.Context) { + if len(ctx.Args()) != 1 { + utils.Fatalf("This command requires an argument.") + } + chain, _, _, _ := utils.GetChain(ctx) + start := time.Now() + if err := utils.ExportChain(chain, ctx.Args().First()); err != nil { + utils.Fatalf("Export error: %v\n", err) + } + fmt.Printf("Export done in %v", time.Since(start)) +} + +func removeDB(ctx *cli.Context) { + confirm, err := utils.PromptConfirm("Remove local databases?") + if err != nil { + utils.Fatalf("%v", err) + } + + if confirm { + fmt.Println("Removing chain and state databases...") + start := time.Now() + + os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "blockchain")) + os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "state")) + + fmt.Printf("Removed in %v\n", time.Since(start)) + } else { + fmt.Println("Operation aborted") + } +} + +func upgradeDB(ctx *cli.Context) { + glog.Infoln("Upgrading blockchain database") + + chain, blockDB, stateDB, extraDB := utils.GetChain(ctx) + v, _ := blockDB.Get([]byte("BlockchainVersion")) + bcVersion := int(common.NewValue(v).Uint()) + if bcVersion == 0 { + bcVersion = core.BlockChainVersion + } + + // Export the current chain. + filename := fmt.Sprintf("blockchain_%d_%s.chain", bcVersion, time.Now().Format("20060102_150405")) + exportFile := filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), filename) + if err := utils.ExportChain(chain, exportFile); err != nil { + utils.Fatalf("Unable to export chain for reimport %s\n", err) + } + flushAll(blockDB, stateDB, extraDB) + os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "blockchain")) + os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "state")) + + // Import the chain file. + chain, blockDB, stateDB, extraDB = utils.GetChain(ctx) + blockDB.Put([]byte("BlockchainVersion"), common.NewValue(core.BlockChainVersion).Bytes()) + err := utils.ImportChain(chain, exportFile) + flushAll(blockDB, stateDB, extraDB) + if err != nil { + utils.Fatalf("Import error %v (a backup is made in %s, use the import command to import it)\n", err, exportFile) + } else { + os.Remove(exportFile) + glog.Infoln("Import finished") + } +} + +func dump(ctx *cli.Context) { + chain, _, stateDB, _ := utils.GetChain(ctx) + for _, arg := range ctx.Args() { + var block *types.Block + if hashish(arg) { + block = chain.GetBlock(common.HexToHash(arg)) + } else { + num, _ := strconv.Atoi(arg) + block = chain.GetBlockByNumber(uint64(num)) + } + if block == nil { + fmt.Println("{}") + utils.Fatalf("block not found") + } else { + state := state.New(block.Root(), stateDB) + fmt.Printf("%s\n", state.Dump()) + } + } +} + +// hashish returns true for strings that look like hashes. +func hashish(x string) bool { + _, err := strconv.Atoi(x) + return err != nil +} + +func flushAll(dbs ...common.Database) { + for _, db := range dbs { + db.Flush() + db.Close() + } +} diff --git a/cmd/geth/main.go b/cmd/geth/main.go index f849063fae..9bf0fe6100 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -24,28 +24,23 @@ import ( "fmt" "io" "io/ioutil" + _ "net/http/pprof" "os" "path/filepath" "runtime" "strconv" "strings" - "time" "github.com/codegangsta/cli" "github.com/ethereum/ethash" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/logger" - "github.com/ethereum/go-ethereum/logger/glog" "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" ) -import _ "net/http/pprof" const ( ClientIdentifier = "Geth" @@ -69,7 +64,12 @@ func init() { app.Action = run app.HideVersion = true // we have a command to print the version app.Commands = []cli.Command{ - blocktestCmd, + blocktestCommand, + importCommand, + exportCommand, + upgradedbCommand, + removedbCommand, + dumpCommand, { Action: makedag, Name: "makedag", @@ -194,15 +194,6 @@ nodes. }, }, }, - { - Action: dump, - Name: "dump", - Usage: `dump a specific block from storage`, - Description: ` -The arguments are interpreted as block numbers or hashes. -Use "ethereum dump 0" to dump the genesis block. -`, - }, { Action: console, Name: "console", @@ -222,26 +213,6 @@ The JavaScript VM exposes a node admin interface as well as the Ðapp JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console `, }, - { - Action: importchain, - Name: "import", - Usage: `import a blockchain file`, - }, - { - Action: exportchain, - Name: "export", - Usage: `export blockchain into file`, - }, - { - Action: upgradeDb, - Name: "upgradedb", - Usage: "upgrade chainblock database", - }, - { - Action: removeDb, - Name: "removedb", - Usage: "Remove blockchain and state databases", - }, } app.Flags = []cli.Flag{ utils.IdentityFlag, @@ -508,103 +479,6 @@ func accountImport(ctx *cli.Context) { fmt.Printf("Address: %x\n", acct) } -func importchain(ctx *cli.Context) { - if len(ctx.Args()) != 1 { - utils.Fatalf("This command requires an argument.") - } - chain, blockDB, stateDB, extraDB := utils.GetChain(ctx) - start := time.Now() - if err := utils.ImportChain(chain, ctx.Args().First()); err != nil { - utils.Fatalf("Import error: %v\n", err) - } - flushAll(blockDB, stateDB, extraDB) - fmt.Printf("Import done in %v", time.Since(start)) -} - -func exportchain(ctx *cli.Context) { - if len(ctx.Args()) != 1 { - utils.Fatalf("This command requires an argument.") - } - chain, _, _, _ := utils.GetChain(ctx) - start := time.Now() - if err := utils.ExportChain(chain, ctx.Args().First()); err != nil { - utils.Fatalf("Export error: %v\n", err) - } - fmt.Printf("Export done in %v", time.Since(start)) -} - -func removeDb(ctx *cli.Context) { - confirm, err := utils.PromptConfirm("Remove local databases?") - if err != nil { - utils.Fatalf("%v", err) - } - - if confirm { - fmt.Println("Removing chain and state databases...") - start := time.Now() - - os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "blockchain")) - os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "state")) - - fmt.Printf("Removed in %v\n", time.Since(start)) - } else { - fmt.Println("Operation aborted") - } -} - -func upgradeDb(ctx *cli.Context) { - glog.Infoln("Upgrading blockchain database") - - chain, blockDB, stateDB, extraDB := utils.GetChain(ctx) - v, _ := blockDB.Get([]byte("BlockchainVersion")) - bcVersion := int(common.NewValue(v).Uint()) - if bcVersion == 0 { - bcVersion = core.BlockChainVersion - } - - // Export the current chain. - filename := fmt.Sprintf("blockchain_%d_%s.chain", bcVersion, time.Now().Format("20060102_150405")) - exportFile := filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), filename) - if err := utils.ExportChain(chain, exportFile); err != nil { - utils.Fatalf("Unable to export chain for reimport %s\n", err) - } - flushAll(blockDB, stateDB, extraDB) - os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "blockchain")) - os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "state")) - - // Import the chain file. - chain, blockDB, stateDB, extraDB = utils.GetChain(ctx) - blockDB.Put([]byte("BlockchainVersion"), common.NewValue(core.BlockChainVersion).Bytes()) - err := utils.ImportChain(chain, exportFile) - flushAll(blockDB, stateDB, extraDB) - if err != nil { - utils.Fatalf("Import error %v (a backup is made in %s, use the import command to import it)\n", err, exportFile) - } else { - os.Remove(exportFile) - glog.Infoln("Import finished") - } -} - -func dump(ctx *cli.Context) { - chain, _, stateDB, _ := utils.GetChain(ctx) - for _, arg := range ctx.Args() { - var block *types.Block - if hashish(arg) { - block = chain.GetBlock(common.HexToHash(arg)) - } else { - num, _ := strconv.Atoi(arg) - block = chain.GetBlockByNumber(uint64(num)) - } - if block == nil { - fmt.Println("{}") - utils.Fatalf("block not found") - } else { - state := state.New(block.Root(), stateDB) - fmt.Printf("%s\n", state.Dump()) - } - } -} - func makedag(ctx *cli.Context) { args := ctx.Args() wrongArgs := func() { @@ -647,16 +521,3 @@ func version(c *cli.Context) { fmt.Printf("GOPATH=%s\n", os.Getenv("GOPATH")) fmt.Printf("GOROOT=%s\n", runtime.GOROOT()) } - -// hashish returns true for strings that look like hashes. -func hashish(x string) bool { - _, err := strconv.Atoi(x) - return err != nil -} - -func flushAll(dbs ...common.Database) { - for _, db := range dbs { - db.Flush() - db.Close() - } -} From 8e4512a5e7eecb53f41996776d55fe7acc92330c Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 14:10:02 +0200 Subject: [PATCH 04/10] p2p/nat: bump timeout in TestAutoDiscRace --- p2p/nat/nat_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/nat/nat_test.go b/p2p/nat/nat_test.go index 95c50522e2..b62640b4b3 100644 --- a/p2p/nat/nat_test.go +++ b/p2p/nat/nat_test.go @@ -30,7 +30,7 @@ func TestAutoDiscRace(t *testing.T) { } // Check that they all return the correct result within the deadline. - deadline := time.After(550 * time.Millisecond) + deadline := time.After(2 * time.Second) for i := 0; i < cap(results); i++ { select { case <-deadline: From 74706a0f029fe74ea15e366904d827da03091fef Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 14:50:31 +0200 Subject: [PATCH 05/10] cmd/geth, cmd/utils: rename utils.Get* -> utils.Make* The renaming should make it clearer that these functions create a new instance for every call. @obscuren suggested this renaming a while ago. --- cmd/geth/chaincmd.go | 10 +++++----- cmd/geth/main.go | 8 ++++---- cmd/utils/flags.go | 19 ++++++++++++------- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index e17d1dc9b0..1aed800f66 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -52,7 +52,7 @@ func importChain(ctx *cli.Context) { if len(ctx.Args()) != 1 { utils.Fatalf("This command requires an argument.") } - chain, blockDB, stateDB, extraDB := utils.GetChain(ctx) + chain, blockDB, stateDB, extraDB := utils.MakeChain(ctx) start := time.Now() if err := utils.ImportChain(chain, ctx.Args().First()); err != nil { utils.Fatalf("Import error: %v\n", err) @@ -65,7 +65,7 @@ func exportChain(ctx *cli.Context) { if len(ctx.Args()) != 1 { utils.Fatalf("This command requires an argument.") } - chain, _, _, _ := utils.GetChain(ctx) + chain, _, _, _ := utils.MakeChain(ctx) start := time.Now() if err := utils.ExportChain(chain, ctx.Args().First()); err != nil { utils.Fatalf("Export error: %v\n", err) @@ -95,7 +95,7 @@ func removeDB(ctx *cli.Context) { func upgradeDB(ctx *cli.Context) { glog.Infoln("Upgrading blockchain database") - chain, blockDB, stateDB, extraDB := utils.GetChain(ctx) + chain, blockDB, stateDB, extraDB := utils.MakeChain(ctx) v, _ := blockDB.Get([]byte("BlockchainVersion")) bcVersion := int(common.NewValue(v).Uint()) if bcVersion == 0 { @@ -113,7 +113,7 @@ func upgradeDB(ctx *cli.Context) { os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "state")) // Import the chain file. - chain, blockDB, stateDB, extraDB = utils.GetChain(ctx) + chain, blockDB, stateDB, extraDB = utils.MakeChain(ctx) blockDB.Put([]byte("BlockchainVersion"), common.NewValue(core.BlockChainVersion).Bytes()) err := utils.ImportChain(chain, exportFile) flushAll(blockDB, stateDB, extraDB) @@ -126,7 +126,7 @@ func upgradeDB(ctx *cli.Context) { } func dump(ctx *cli.Context) { - chain, _, stateDB, _ := utils.GetChain(ctx) + chain, _, stateDB, _ := utils.MakeChain(ctx) for _, arg := range ctx.Args() { var block *types.Block if hashish(arg) { diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 9bf0fe6100..eecd7546a8 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -394,7 +394,7 @@ func startEth(ctx *cli.Context, eth *eth.Ethereum) { } func accountList(ctx *cli.Context) { - am := utils.GetAccountManager(ctx) + am := utils.MakeAccountManager(ctx) accts, err := am.Accounts() if err != nil { utils.Fatalf("Could not list accounts: %v", err) @@ -436,7 +436,7 @@ func getPassPhrase(ctx *cli.Context, desc string, confirmation bool) (passphrase } func accountCreate(ctx *cli.Context) { - am := utils.GetAccountManager(ctx) + am := utils.MakeAccountManager(ctx) passphrase := getPassPhrase(ctx, "Your new account is locked with a password. Please give a password. Do not forget this password.", true) acct, err := am.NewAccount(passphrase) if err != nil { @@ -455,7 +455,7 @@ func importWallet(ctx *cli.Context) { utils.Fatalf("Could not read wallet file: %v", err) } - am := utils.GetAccountManager(ctx) + am := utils.MakeAccountManager(ctx) passphrase := getPassPhrase(ctx, "", false) acct, err := am.ImportPreSaleKey(keyJson, passphrase) @@ -470,7 +470,7 @@ func accountImport(ctx *cli.Context) { if len(keyfile) == 0 { utils.Fatalf("keyfile must be given as argument") } - am := utils.GetAccountManager(ctx) + am := utils.MakeAccountManager(ctx) passphrase := getPassPhrase(ctx, "Your new account is locked with a password. Please give a password. Do not forget this password.", true) acct, err := am.Import(keyfile, passphrase) if err != nil { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 176a546f1d..d319055b17 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -256,7 +256,8 @@ var ( } ) -func GetNAT(ctx *cli.Context) nat.Interface { +// MakeNAT creates a port mapper from set command line flags. +func MakeNAT(ctx *cli.Context) nat.Interface { natif, err := nat.Parse(ctx.GlobalString(NATFlag.Name)) if err != nil { Fatalf("Option %s: %v", NATFlag.Name, err) @@ -264,7 +265,8 @@ func GetNAT(ctx *cli.Context) nat.Interface { return natif } -func GetNodeKey(ctx *cli.Context) (key *ecdsa.PrivateKey) { +// MakeNodeKey creates a node key from set command line flags. +func MakeNodeKey(ctx *cli.Context) (key *ecdsa.PrivateKey) { hex, file := ctx.GlobalString(NodeKeyHexFlag.Name), ctx.GlobalString(NodeKeyFileFlag.Name) var err error switch { @@ -282,6 +284,7 @@ func GetNodeKey(ctx *cli.Context) (key *ecdsa.PrivateKey) { return key } +// MakeEthConfig creates ethereum options from set command line flags. func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config { customName := ctx.GlobalString(IdentityFlag.Name) if len(customName) > 0 { @@ -299,15 +302,15 @@ func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config { LogJSON: ctx.GlobalString(LogJSONFlag.Name), Etherbase: ctx.GlobalString(EtherbaseFlag.Name), MinerThreads: ctx.GlobalInt(MinerThreadsFlag.Name), - AccountManager: GetAccountManager(ctx), + AccountManager: MakeAccountManager(ctx), VmDebug: ctx.GlobalBool(VMDebugFlag.Name), MaxPeers: ctx.GlobalInt(MaxPeersFlag.Name), MaxPendingPeers: ctx.GlobalInt(MaxPendingPeersFlag.Name), Port: ctx.GlobalString(ListenPortFlag.Name), - NAT: GetNAT(ctx), + NAT: MakeNAT(ctx), NatSpec: ctx.GlobalBool(NatspecEnabledFlag.Name), Discovery: !ctx.GlobalBool(NoDiscoverFlag.Name), - NodeKey: GetNodeKey(ctx), + NodeKey: MakeNodeKey(ctx), Shh: ctx.GlobalBool(WhisperEnabledFlag.Name), Dial: true, BootNodes: ctx.GlobalString(BootnodesFlag.Name), @@ -325,7 +328,8 @@ func SetupLogger(ctx *cli.Context) { glog.SetLogDir(ctx.GlobalString(LogFileFlag.Name)) } -func GetChain(ctx *cli.Context) (chain *core.ChainManager, blockDB, stateDB, extraDB common.Database) { +// MakeChain creates a chain manager from set command line flags. +func MakeChain(ctx *cli.Context) (chain *core.ChainManager, blockDB, stateDB, extraDB common.Database) { dd := ctx.GlobalString(DataDirFlag.Name) var err error if blockDB, err = ethdb.NewLDBDatabase(filepath.Join(dd, "blockchain")); err != nil { @@ -347,7 +351,8 @@ func GetChain(ctx *cli.Context) (chain *core.ChainManager, blockDB, stateDB, ext return chain, blockDB, stateDB, extraDB } -func GetAccountManager(ctx *cli.Context) *accounts.Manager { +// MakeChain creates an account manager from set command line flags. +func MakeAccountManager(ctx *cli.Context) *accounts.Manager { dataDir := ctx.GlobalString(DataDirFlag.Name) ks := crypto.NewKeyStorePassphrase(filepath.Join(dataDir, "keystore")) return accounts.NewManager(ks) From 705beb4c25f976f6bce40818bd835e235c2babf4 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 15:48:07 +0200 Subject: [PATCH 06/10] cmd/utils: print errors only once if stdout and stderr are the same file --- cmd/utils/cmd.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 550ac1c514..06c240bd4f 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -125,10 +125,17 @@ func initDataDir(Datadir string) { } } -// Fatalf formats a message to standard output and exits the program. +// Fatalf formats a message to standard error and exits the program. +// The message is also printed to standard output if standard error +// is redirected to a different file. func Fatalf(format string, args ...interface{}) { - fmt.Fprintf(os.Stderr, "Fatal: "+format+"\n", args...) - fmt.Fprintf(os.Stdout, "Fatal: "+format+"\n", args...) + w := io.MultiWriter(os.Stdout, os.Stderr) + outf, _ := os.Stdout.Stat() + errf, _ := os.Stderr.Stat() + if outf != nil && errf != nil && os.SameFile(outf, errf) { + w = os.Stderr + } + fmt.Fprintf(w, "Fatal: "+format+"\n", args...) logger.Flush() os.Exit(1) } From 67effb94b6fadac7b207cb5333c91f578326762e Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 16:02:08 +0200 Subject: [PATCH 07/10] cmd/geth, cmd/utils: make chain importing interruptible Interrupting import with Ctrl-C could cause database corruption because the signal wasn't handled. utils.ImportChain now checks for a queued interrupt on every batch. --- cmd/geth/chaincmd.go | 11 ++++++----- cmd/utils/cmd.go | 38 +++++++++++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 1aed800f66..6245c691ba 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -54,10 +54,11 @@ func importChain(ctx *cli.Context) { } chain, blockDB, stateDB, extraDB := utils.MakeChain(ctx) start := time.Now() - if err := utils.ImportChain(chain, ctx.Args().First()); err != nil { - utils.Fatalf("Import error: %v\n", err) - } + err := utils.ImportChain(chain, ctx.Args().First()) flushAll(blockDB, stateDB, extraDB) + if err != nil { + utils.Fatalf("Import error: %v", err) + } fmt.Printf("Import done in %v", time.Since(start)) } @@ -106,7 +107,7 @@ func upgradeDB(ctx *cli.Context) { filename := fmt.Sprintf("blockchain_%d_%s.chain", bcVersion, time.Now().Format("20060102_150405")) exportFile := filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), filename) if err := utils.ExportChain(chain, exportFile); err != nil { - utils.Fatalf("Unable to export chain for reimport %s\n", err) + utils.Fatalf("Unable to export chain for reimport %s", err) } flushAll(blockDB, stateDB, extraDB) os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "blockchain")) @@ -118,7 +119,7 @@ func upgradeDB(ctx *cli.Context) { err := utils.ImportChain(chain, exportFile) flushAll(blockDB, stateDB, extraDB) if err != nil { - utils.Fatalf("Import error %v (a backup is made in %s, use the import command to import it)\n", err, exportFile) + utils.Fatalf("Import error %v (a backup is made in %s, use the import command to import it)", err, exportFile) } else { os.Remove(exportFile) glog.Infoln("Import finished") diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 06c240bd4f..2e2b627df7 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -173,22 +173,47 @@ func FormatTransactionData(data string) []byte { return d } -func ImportChain(chainmgr *core.ChainManager, fn string) error { +func ImportChain(chain *core.ChainManager, fn string) error { + // Watch for Ctrl-C while the import is running. + // If a signal is received, the import will stop at the next batch. + interrupt := make(chan os.Signal, 1) + stop := make(chan struct{}) + signal.Notify(interrupt, os.Interrupt) + defer signal.Stop(interrupt) + defer close(interrupt) + go func() { + if _, ok := <-interrupt; ok { + glog.Info("caught interrupt during import, will stop at next batch") + } + close(stop) + }() + checkInterrupt := func() bool { + select { + case <-stop: + return true + default: + return false + } + } + glog.Infoln("Importing blockchain", fn) - fh, err := os.OpenFile(fn, os.O_RDONLY, os.ModePerm) + fh, err := os.Open(fn) if err != nil { return err } defer fh.Close() - - chainmgr.Reset() stream := rlp.NewStream(fh, 0) + // Remove all existing blocks and start the import. + chain.Reset() batchSize := 2500 blocks := make(types.Blocks, batchSize) n := 0 for { // Load a batch of RLP blocks. + if checkInterrupt() { + return fmt.Errorf("interrupted") + } i := 0 for ; i < batchSize; i++ { var b types.Block @@ -204,7 +229,10 @@ func ImportChain(chainmgr *core.ChainManager, fn string) error { break } // Import the batch. - if _, err := chainmgr.InsertChain(blocks[:i]); err != nil { + if checkInterrupt() { + return fmt.Errorf("interrupted") + } + if _, err := chain.InsertChain(blocks[:i]); err != nil { return fmt.Errorf("invalid block %d: %v", n, err) } } From a8bc2181c94f5d3a9455c4fa526f8722a21ecb04 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 May 2015 17:35:08 +0200 Subject: [PATCH 08/10] cmd/utils: skip batches with known blocks during import This makes block importing restartable. --- cmd/utils/cmd.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 2e2b627df7..2a5e2ec6ad 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -204,12 +204,11 @@ func ImportChain(chain *core.ChainManager, fn string) error { defer fh.Close() stream := rlp.NewStream(fh, 0) - // Remove all existing blocks and start the import. - chain.Reset() + // Run actual the import. batchSize := 2500 blocks := make(types.Blocks, batchSize) n := 0 - for { + for batch := 0; ; batch++ { // Load a batch of RLP blocks. if checkInterrupt() { return fmt.Errorf("interrupted") @@ -232,6 +231,11 @@ func ImportChain(chain *core.ChainManager, fn string) error { if checkInterrupt() { return fmt.Errorf("interrupted") } + if hasAllBlocks(chain, blocks[:i]) { + glog.Infof("skipping batch %d, all blocks present [%x / %x]", + batch, blocks[0].Hash().Bytes()[:4], blocks[i-1].Hash().Bytes()[:4]) + continue + } if _, err := chain.InsertChain(blocks[:i]); err != nil { return fmt.Errorf("invalid block %d: %v", n, err) } @@ -239,6 +243,15 @@ func ImportChain(chain *core.ChainManager, fn string) error { return nil } +func hasAllBlocks(chain *core.ChainManager, bs []*types.Block) bool { + for _, b := range bs { + if !chain.HasBlock(b.Hash()) { + return false + } + } + return true +} + func ExportChain(chainmgr *core.ChainManager, fn string) error { glog.Infoln("Exporting blockchain to", fn) fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) From e1fe75e3b637758f99ddbcaeb01eafa1a0b6455e Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 28 May 2015 01:16:57 +0200 Subject: [PATCH 09/10] cmd/utils: use constant for import batch size --- cmd/utils/cmd.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 2a5e2ec6ad..e5413973de 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -40,6 +40,10 @@ import ( "github.com/peterh/liner" ) +const ( + importBatchSize = 2500 +) + var interruptCallbacks = []func(os.Signal){} // Register interrupt handlers callbacks @@ -205,8 +209,7 @@ func ImportChain(chain *core.ChainManager, fn string) error { stream := rlp.NewStream(fh, 0) // Run actual the import. - batchSize := 2500 - blocks := make(types.Blocks, batchSize) + blocks := make(types.Blocks, importBatchSize) n := 0 for batch := 0; ; batch++ { // Load a batch of RLP blocks. @@ -214,7 +217,7 @@ func ImportChain(chain *core.ChainManager, fn string) error { return fmt.Errorf("interrupted") } i := 0 - for ; i < batchSize; i++ { + for ; i < importBatchSize; i++ { var b types.Block if err := stream.Decode(&b); err == io.EOF { break From e84bbcce3c335b863eb6304ad910047054b68c20 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 28 May 2015 01:20:03 +0200 Subject: [PATCH 10/10] cmd/geth: don't flush databases after import --- cmd/geth/chaincmd.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 6245c691ba..947532f408 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -55,7 +55,7 @@ func importChain(ctx *cli.Context) { chain, blockDB, stateDB, extraDB := utils.MakeChain(ctx) start := time.Now() err := utils.ImportChain(chain, ctx.Args().First()) - flushAll(blockDB, stateDB, extraDB) + closeAll(blockDB, stateDB, extraDB) if err != nil { utils.Fatalf("Import error: %v", err) } @@ -109,7 +109,7 @@ func upgradeDB(ctx *cli.Context) { if err := utils.ExportChain(chain, exportFile); err != nil { utils.Fatalf("Unable to export chain for reimport %s", err) } - flushAll(blockDB, stateDB, extraDB) + closeAll(blockDB, stateDB, extraDB) os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "blockchain")) os.RemoveAll(filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), "state")) @@ -117,7 +117,7 @@ func upgradeDB(ctx *cli.Context) { chain, blockDB, stateDB, extraDB = utils.MakeChain(ctx) blockDB.Put([]byte("BlockchainVersion"), common.NewValue(core.BlockChainVersion).Bytes()) err := utils.ImportChain(chain, exportFile) - flushAll(blockDB, stateDB, extraDB) + closeAll(blockDB, stateDB, extraDB) if err != nil { utils.Fatalf("Import error %v (a backup is made in %s, use the import command to import it)", err, exportFile) } else { @@ -152,9 +152,8 @@ func hashish(x string) bool { return err != nil } -func flushAll(dbs ...common.Database) { +func closeAll(dbs ...common.Database) { for _, db := range dbs { - db.Flush() db.Close() } }