consensus/ethash: implement faster difficulty calculators (#21976)
This PR adds re-written difficulty calculators, which are based on uint256. It also adds a fuzzer + oss-fuzz integration for the new fuzzer. It does differential fuzzing between the new and old calculators. Note: this PR does not actually enable the new calculators.
This commit is contained in:
Martin Holst Swende
GitHub
parent
88c696240d
commit
efe6dd2904
|
|
@ -485,6 +485,11 @@ func calcDifficultyFrontier(time uint64, parent *types.Header) *big.Int {
|
|||
return diff
|
||||
}
|
||||
|
||||
// Exported for fuzzing
|
||||
var FrontierDifficultyCalulator = calcDifficultyFrontier
|
||||
var HomesteadDifficultyCalulator = calcDifficultyHomestead
|
||||
var DynamicDifficultyCalculator = makeDifficultyCalculator
|
||||
|
||||
// VerifySeal implements consensus.Engine, checking whether the given block satisfies
|
||||
// the PoW difficulty requirements.
|
||||
func (ethash *Ethash) VerifySeal(chain consensus.ChainHeaderReader, header *types.Header) error {
|
||||
|
|
|
|||
|
|
@ -17,12 +17,15 @@
|
|||
package ethash
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"math/big"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/common/math"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
|
|
@ -84,3 +87,102 @@ func TestCalcDifficulty(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func randSlice(min, max uint32) []byte {
|
||||
var b = make([]byte, 4)
|
||||
rand.Read(b)
|
||||
a := binary.LittleEndian.Uint32(b)
|
||||
size := min + a%(max-min)
|
||||
out := make([]byte, size)
|
||||
rand.Read(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func TestDifficultyCalculators(t *testing.T) {
|
||||
rand.Seed(2)
|
||||
for i := 0; i < 5000; i++ {
|
||||
// 1 to 300 seconds diff
|
||||
var timeDelta = uint64(1 + rand.Uint32()%3000)
|
||||
diffBig := big.NewInt(0).SetBytes(randSlice(2, 10))
|
||||
if diffBig.Cmp(params.MinimumDifficulty) < 0 {
|
||||
diffBig.Set(params.MinimumDifficulty)
|
||||
}
|
||||
//rand.Read(difficulty)
|
||||
header := &types.Header{
|
||||
Difficulty: diffBig,
|
||||
Number: new(big.Int).SetUint64(rand.Uint64() % 50_000_000),
|
||||
Time: rand.Uint64() - timeDelta,
|
||||
}
|
||||
if rand.Uint32()&1 == 0 {
|
||||
header.UncleHash = types.EmptyUncleHash
|
||||
}
|
||||
bombDelay := new(big.Int).SetUint64(rand.Uint64() % 50_000_000)
|
||||
for i, pair := range []struct {
|
||||
bigFn func(time uint64, parent *types.Header) *big.Int
|
||||
u256Fn func(time uint64, parent *types.Header) *big.Int
|
||||
}{
|
||||
{FrontierDifficultyCalulator, CalcDifficultyFrontierU256},
|
||||
{HomesteadDifficultyCalulator, CalcDifficultyHomesteadU256},
|
||||
{DynamicDifficultyCalculator(bombDelay), MakeDifficultyCalculatorU256(bombDelay)},
|
||||
} {
|
||||
time := header.Time + timeDelta
|
||||
want := pair.bigFn(time, header)
|
||||
have := pair.u256Fn(time, header)
|
||||
if want.BitLen() > 256 {
|
||||
continue
|
||||
}
|
||||
if want.Cmp(have) != 0 {
|
||||
t.Fatalf("pair %d: want %x have %x\nparent.Number: %x\np.Time: %x\nc.Time: %x\nBombdelay: %v\n", i, want, have,
|
||||
header.Number, header.Time, time, bombDelay)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDifficultyCalculator(b *testing.B) {
|
||||
x1 := makeDifficultyCalculator(big.NewInt(1000000))
|
||||
x2 := MakeDifficultyCalculatorU256(big.NewInt(1000000))
|
||||
h := &types.Header{
|
||||
ParentHash: common.Hash{},
|
||||
UncleHash: types.EmptyUncleHash,
|
||||
Difficulty: big.NewInt(0xffffff),
|
||||
Number: big.NewInt(500000),
|
||||
Time: 1000000,
|
||||
}
|
||||
b.Run("big-frontier", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
calcDifficultyFrontier(1000014, h)
|
||||
}
|
||||
})
|
||||
b.Run("u256-frontier", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
CalcDifficultyFrontierU256(1000014, h)
|
||||
}
|
||||
})
|
||||
b.Run("big-homestead", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
calcDifficultyHomestead(1000014, h)
|
||||
}
|
||||
})
|
||||
b.Run("u256-homestead", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
CalcDifficultyHomesteadU256(1000014, h)
|
||||
}
|
||||
})
|
||||
b.Run("big-generic", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
x1(1000014, h)
|
||||
}
|
||||
})
|
||||
b.Run("u256-generic", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
x2(1000014, h)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,193 @@
|
|||
// Copyright 2020 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package ethash
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/holiman/uint256"
|
||||
)
|
||||
|
||||
const (
|
||||
// frontierDurationLimit is for Frontier:
|
||||
// The decision boundary on the blocktime duration used to determine
|
||||
// whether difficulty should go up or down.
|
||||
frontierDurationLimit = 13
|
||||
// minimumDifficulty The minimum that the difficulty may ever be.
|
||||
minimumDifficulty = 131072
|
||||
// expDiffPeriod is the exponential difficulty period
|
||||
expDiffPeriodUint = 100000
|
||||
// difficultyBoundDivisorBitShift is the bound divisor of the difficulty (2048),
|
||||
// This constant is the right-shifts to use for the division.
|
||||
difficultyBoundDivisor = 11
|
||||
)
|
||||
|
||||
// CalcDifficultyFrontierU256 is the difficulty adjustment algorithm. It returns the
|
||||
// difficulty that a new block should have when created at time given the parent
|
||||
// block's time and difficulty. The calculation uses the Frontier rules.
|
||||
func CalcDifficultyFrontierU256(time uint64, parent *types.Header) *big.Int {
|
||||
/*
|
||||
Algorithm
|
||||
block_diff = pdiff + pdiff / 2048 * (1 if time - ptime < 13 else -1) + int(2^((num // 100000) - 2))
|
||||
|
||||
Where:
|
||||
- pdiff = parent.difficulty
|
||||
- ptime = parent.time
|
||||
- time = block.timestamp
|
||||
- num = block.number
|
||||
*/
|
||||
|
||||
pDiff := uint256.NewInt()
|
||||
pDiff.SetFromBig(parent.Difficulty) // pDiff: pdiff
|
||||
adjust := pDiff.Clone()
|
||||
adjust.Rsh(adjust, difficultyBoundDivisor) // adjust: pDiff / 2048
|
||||
|
||||
if time-parent.Time < frontierDurationLimit {
|
||||
pDiff.Add(pDiff, adjust)
|
||||
} else {
|
||||
pDiff.Sub(pDiff, adjust)
|
||||
}
|
||||
if pDiff.LtUint64(minimumDifficulty) {
|
||||
pDiff.SetUint64(minimumDifficulty)
|
||||
}
|
||||
// 'pdiff' now contains:
|
||||
// pdiff + pdiff / 2048 * (1 if time - ptime < 13 else -1)
|
||||
|
||||
if periodCount := (parent.Number.Uint64() + 1) / expDiffPeriodUint; periodCount > 1 {
|
||||
// diff = diff + 2^(periodCount - 2)
|
||||
expDiff := adjust.SetOne()
|
||||
expDiff.Lsh(expDiff, uint(periodCount-2)) // expdiff: 2 ^ (periodCount -2)
|
||||
pDiff.Add(pDiff, expDiff)
|
||||
}
|
||||
return pDiff.ToBig()
|
||||
}
|
||||
|
||||
// CalcDifficultyHomesteadU256 is the difficulty adjustment algorithm. It returns
|
||||
// the difficulty that a new block should have when created at time given the
|
||||
// parent block's time and difficulty. The calculation uses the Homestead rules.
|
||||
func CalcDifficultyHomesteadU256(time uint64, parent *types.Header) *big.Int {
|
||||
/*
|
||||
https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2.md
|
||||
Algorithm:
|
||||
block_diff = pdiff + pdiff / 2048 * max(1 - (time - ptime) / 10, -99) + 2 ^ int((num / 100000) - 2))
|
||||
|
||||
Our modification, to use unsigned ints:
|
||||
block_diff = pdiff - pdiff / 2048 * max((time - ptime) / 10 - 1, 99) + 2 ^ int((num / 100000) - 2))
|
||||
|
||||
Where:
|
||||
- pdiff = parent.difficulty
|
||||
- ptime = parent.time
|
||||
- time = block.timestamp
|
||||
- num = block.number
|
||||
*/
|
||||
|
||||
pDiff := uint256.NewInt()
|
||||
pDiff.SetFromBig(parent.Difficulty) // pDiff: pdiff
|
||||
adjust := pDiff.Clone()
|
||||
adjust.Rsh(adjust, difficultyBoundDivisor) // adjust: pDiff / 2048
|
||||
|
||||
x := (time - parent.Time) / 10 // (time - ptime) / 10)
|
||||
var neg = true
|
||||
if x == 0 {
|
||||
x = 1
|
||||
neg = false
|
||||
} else if x >= 100 {
|
||||
x = 99
|
||||
} else {
|
||||
x = x - 1
|
||||
}
|
||||
z := new(uint256.Int).SetUint64(x)
|
||||
adjust.Mul(adjust, z) // adjust: (pdiff / 2048) * max((time - ptime) / 10 - 1, 99)
|
||||
if neg {
|
||||
pDiff.Sub(pDiff, adjust) // pdiff - pdiff / 2048 * max((time - ptime) / 10 - 1, 99)
|
||||
} else {
|
||||
pDiff.Add(pDiff, adjust) // pdiff + pdiff / 2048 * max((time - ptime) / 10 - 1, 99)
|
||||
}
|
||||
if pDiff.LtUint64(minimumDifficulty) {
|
||||
pDiff.SetUint64(minimumDifficulty)
|
||||
}
|
||||
// for the exponential factor, a.k.a "the bomb"
|
||||
// diff = diff + 2^(periodCount - 2)
|
||||
if periodCount := (1 + parent.Number.Uint64()) / expDiffPeriodUint; periodCount > 1 {
|
||||
expFactor := adjust.Lsh(adjust.SetOne(), uint(periodCount-2))
|
||||
pDiff.Add(pDiff, expFactor)
|
||||
}
|
||||
return pDiff.ToBig()
|
||||
}
|
||||
|
||||
// MakeDifficultyCalculatorU256 creates a difficultyCalculator with the given bomb-delay.
|
||||
// the difficulty is calculated with Byzantium rules, which differs from Homestead in
|
||||
// how uncles affect the calculation
|
||||
func MakeDifficultyCalculatorU256(bombDelay *big.Int) func(time uint64, parent *types.Header) *big.Int {
|
||||
// Note, the calculations below looks at the parent number, which is 1 below
|
||||
// the block number. Thus we remove one from the delay given
|
||||
bombDelayFromParent := bombDelay.Uint64() - 1
|
||||
return func(time uint64, parent *types.Header) *big.Int {
|
||||
/*
|
||||
https://github.com/ethereum/EIPs/issues/100
|
||||
pDiff = parent.difficulty
|
||||
BLOCK_DIFF_FACTOR = 9
|
||||
a = pDiff + (pDiff // BLOCK_DIFF_FACTOR) * adj_factor
|
||||
b = min(parent.difficulty, MIN_DIFF)
|
||||
child_diff = max(a,b )
|
||||
*/
|
||||
x := (time - parent.Time) / 9 // (block_timestamp - parent_timestamp) // 9
|
||||
c := uint64(1) // if parent.unclehash == emptyUncleHashHash
|
||||
if parent.UncleHash != types.EmptyUncleHash {
|
||||
c = 2
|
||||
}
|
||||
xNeg := x >= c
|
||||
if xNeg {
|
||||
// x is now _negative_ adjustment factor
|
||||
x = x - c // - ( (t-p)/p -( 2 or 1) )
|
||||
} else {
|
||||
x = c - x // (2 or 1) - (t-p)/9
|
||||
}
|
||||
if x > 99 {
|
||||
x = 99 // max(x, 99)
|
||||
}
|
||||
// parent_diff + (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99))
|
||||
y := new(uint256.Int)
|
||||
y.SetFromBig(parent.Difficulty) // y: p_diff
|
||||
pDiff := y.Clone() // pdiff: p_diff
|
||||
z := new(uint256.Int).SetUint64(x) //z : +-adj_factor (either pos or negative)
|
||||
y.Rsh(y, difficultyBoundDivisor) // y: p__diff / 2048
|
||||
z.Mul(y, z) // z: (p_diff / 2048 ) * (+- adj_factor)
|
||||
|
||||
if xNeg {
|
||||
y.Sub(pDiff, z) // y: parent_diff + parent_diff/2048 * adjustment_factor
|
||||
} else {
|
||||
y.Add(pDiff, z) // y: parent_diff + parent_diff/2048 * adjustment_factor
|
||||
}
|
||||
// minimum difficulty can ever be (before exponential factor)
|
||||
if y.LtUint64(minimumDifficulty) {
|
||||
y.SetUint64(minimumDifficulty)
|
||||
}
|
||||
// calculate a fake block number for the ice-age delay
|
||||
// Specification: https://eips.ethereum.org/EIPS/eip-1234
|
||||
var pNum = parent.Number.Uint64()
|
||||
if pNum >= bombDelayFromParent {
|
||||
if fakeBlockNumber := pNum - bombDelayFromParent; fakeBlockNumber >= 2*expDiffPeriodUint {
|
||||
z.SetOne()
|
||||
z.Lsh(z, uint(fakeBlockNumber/expDiffPeriodUint-2))
|
||||
y.Add(z, y)
|
||||
}
|
||||
}
|
||||
return y.ToBig()
|
||||
}
|
||||
}
|
||||
|
|
@ -57,6 +57,7 @@ compile_fuzzer tests/fuzzers/txfetcher Fuzz fuzzTxfetcher
|
|||
compile_fuzzer tests/fuzzers/rlp Fuzz fuzzRlp
|
||||
compile_fuzzer tests/fuzzers/trie Fuzz fuzzTrie
|
||||
compile_fuzzer tests/fuzzers/stacktrie Fuzz fuzzStackTrie
|
||||
compile_fuzzer tests/fuzzers/difficulty Fuzz fuzzDifficulty
|
||||
|
||||
compile_fuzzer tests/fuzzers/bls12381 FuzzG1Add fuzz_g1_add
|
||||
compile_fuzzer tests/fuzzers/bls12381 FuzzG1Mul fuzz_g1_mul
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
||||
"github.com/ethereum/go-ethereum/tests/fuzzers/difficulty"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Fprintf(os.Stderr, "Usage: debug <file>")
|
||||
os.Exit(1)
|
||||
}
|
||||
crasher := os.Args[1]
|
||||
data, err := ioutil.ReadFile(crasher)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
difficulty.Fuzz(data)
|
||||
}
|
||||
|
|
@ -0,0 +1,145 @@
|
|||
// Copyright 2020 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package difficulty
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
|
||||
"github.com/ethereum/go-ethereum/consensus/ethash"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
)
|
||||
|
||||
type fuzzer struct {
|
||||
input io.Reader
|
||||
exhausted bool
|
||||
debugging bool
|
||||
}
|
||||
|
||||
func (f *fuzzer) read(size int) []byte {
|
||||
out := make([]byte, size)
|
||||
if _, err := f.input.Read(out); err != nil {
|
||||
f.exhausted = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (f *fuzzer) readSlice(min, max int) []byte {
|
||||
var a uint16
|
||||
binary.Read(f.input, binary.LittleEndian, &a)
|
||||
size := min + int(a)%(max-min)
|
||||
out := make([]byte, size)
|
||||
if _, err := f.input.Read(out); err != nil {
|
||||
f.exhausted = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (f *fuzzer) readUint64(min, max uint64) uint64 {
|
||||
if min == max {
|
||||
return min
|
||||
}
|
||||
var a uint64
|
||||
if err := binary.Read(f.input, binary.LittleEndian, &a); err != nil {
|
||||
f.exhausted = true
|
||||
}
|
||||
a = min + a%(max-min)
|
||||
return a
|
||||
}
|
||||
func (f *fuzzer) readBool() bool {
|
||||
return f.read(1)[0]&0x1 == 0
|
||||
}
|
||||
|
||||
// The function must return
|
||||
// 1 if the fuzzer should increase priority of the
|
||||
// given input during subsequent fuzzing (for example, the input is lexically
|
||||
// correct and was parsed successfully);
|
||||
// -1 if the input must not be added to corpus even if gives new coverage; and
|
||||
// 0 otherwise
|
||||
// other values are reserved for future use.
|
||||
func Fuzz(data []byte) int {
|
||||
f := fuzzer{
|
||||
input: bytes.NewReader(data),
|
||||
exhausted: false,
|
||||
}
|
||||
return f.fuzz()
|
||||
}
|
||||
|
||||
var minDifficulty = big.NewInt(0x2000)
|
||||
|
||||
type calculator func(time uint64, parent *types.Header) *big.Int
|
||||
|
||||
func (f *fuzzer) fuzz() int {
|
||||
// A parent header
|
||||
header := &types.Header{}
|
||||
if f.readBool() {
|
||||
header.UncleHash = types.EmptyUncleHash
|
||||
}
|
||||
// Difficulty can range between 0x2000 (2 bytes) and up to 32 bytes
|
||||
{
|
||||
diff := new(big.Int).SetBytes(f.readSlice(2, 32))
|
||||
if diff.Cmp(minDifficulty) < 0 {
|
||||
diff.Set(minDifficulty)
|
||||
}
|
||||
header.Difficulty = diff
|
||||
}
|
||||
// Number can range between 0 and up to 32 bytes (but not so that the child exceeds it)
|
||||
{
|
||||
// However, if we use astronomic numbers, then the bomb exp karatsuba calculation
|
||||
// in the legacy methods)
|
||||
// times out, so we limit it to fit within reasonable bounds
|
||||
number := new(big.Int).SetBytes(f.readSlice(0, 4)) // 4 bytes: 32 bits: block num max 4 billion
|
||||
header.Number = number
|
||||
}
|
||||
// Both parent and child time must fit within uint64
|
||||
var time uint64
|
||||
{
|
||||
childTime := f.readUint64(1, 0xFFFFFFFFFFFFFFFF)
|
||||
//fmt.Printf("childTime: %x\n",childTime)
|
||||
delta := f.readUint64(1, childTime)
|
||||
//fmt.Printf("delta: %v\n", delta)
|
||||
pTime := childTime - delta
|
||||
header.Time = pTime
|
||||
time = childTime
|
||||
}
|
||||
// Bomb delay will never exceed uint64
|
||||
bombDelay := new(big.Int).SetUint64(f.readUint64(1, 0xFFFFFFFFFFFFFFFe))
|
||||
|
||||
if f.exhausted {
|
||||
return 0
|
||||
}
|
||||
|
||||
for i, pair := range []struct {
|
||||
bigFn calculator
|
||||
u256Fn calculator
|
||||
}{
|
||||
{ethash.FrontierDifficultyCalulator, ethash.CalcDifficultyFrontierU256},
|
||||
{ethash.HomesteadDifficultyCalulator, ethash.CalcDifficultyHomesteadU256},
|
||||
{ethash.DynamicDifficultyCalculator(bombDelay), ethash.MakeDifficultyCalculatorU256(bombDelay)},
|
||||
} {
|
||||
want := pair.bigFn(time, header)
|
||||
have := pair.u256Fn(time, header)
|
||||
if want.Cmp(have) != 0 {
|
||||
panic(fmt.Sprintf("pair %d: want %x have %x\nparent.Number: %x\np.Time: %x\nc.Time: %x\nBombdelay: %v\n", i, want, have,
|
||||
header.Number, header.Time, time, bombDelay))
|
||||
}
|
||||
}
|
||||
return 1
|
||||
}
|
||||
Loading…
Reference in New Issue