Revert "common/prque: generic priority queue (#26290)"

This reverts commit bf1798e04e.
This commit is contained in:
Péter Szilágyi 2023-02-09 13:45:04 +02:00 committed by GitHub
parent da3c974c36
commit 622d5f54f0
20 changed files with 264 additions and 277 deletions

View File

@ -21,7 +21,6 @@ import (
"time" "time"
"github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/common/mclock"
"golang.org/x/exp/constraints"
) )
// LazyQueue is a priority queue data structure where priorities can change over // LazyQueue is a priority queue data structure where priorities can change over
@ -33,31 +32,31 @@ import (
// //
// If the upper estimate is exceeded then Update should be called for that item. // If the upper estimate is exceeded then Update should be called for that item.
// A global Refresh function should also be called periodically. // A global Refresh function should also be called periodically.
type LazyQueue[P constraints.Ordered, V any] struct { type LazyQueue struct {
clock mclock.Clock clock mclock.Clock
// Items are stored in one of two internal queues ordered by estimated max // Items are stored in one of two internal queues ordered by estimated max
// priority until the next and the next-after-next refresh. Update and Refresh // priority until the next and the next-after-next refresh. Update and Refresh
// always places items in queue[1]. // always places items in queue[1].
queue [2]*sstack[P, V] queue [2]*sstack
popQueue *sstack[P, V] popQueue *sstack
period time.Duration period time.Duration
maxUntil mclock.AbsTime maxUntil mclock.AbsTime
indexOffset int indexOffset int
setIndex SetIndexCallback[V] setIndex SetIndexCallback
priority PriorityCallback[P, V] priority PriorityCallback
maxPriority MaxPriorityCallback[P, V] maxPriority MaxPriorityCallback
lastRefresh1, lastRefresh2 mclock.AbsTime lastRefresh1, lastRefresh2 mclock.AbsTime
} }
type ( type (
PriorityCallback[P constraints.Ordered, V any] func(data V) P // actual priority callback PriorityCallback func(data interface{}) int64 // actual priority callback
MaxPriorityCallback[P constraints.Ordered, V any] func(data V, until mclock.AbsTime) P // estimated maximum priority callback MaxPriorityCallback func(data interface{}, until mclock.AbsTime) int64 // estimated maximum priority callback
) )
// NewLazyQueue creates a new lazy queue // NewLazyQueue creates a new lazy queue
func NewLazyQueue[P constraints.Ordered, V any](setIndex SetIndexCallback[V], priority PriorityCallback[P, V], maxPriority MaxPriorityCallback[P, V], clock mclock.Clock, refreshPeriod time.Duration) *LazyQueue[P, V] { func NewLazyQueue(setIndex SetIndexCallback, priority PriorityCallback, maxPriority MaxPriorityCallback, clock mclock.Clock, refreshPeriod time.Duration) *LazyQueue {
q := &LazyQueue[P, V]{ q := &LazyQueue{
popQueue: newSstack[P, V](nil), popQueue: newSstack(nil, false),
setIndex: setIndex, setIndex: setIndex,
priority: priority, priority: priority,
maxPriority: maxPriority, maxPriority: maxPriority,
@ -72,13 +71,13 @@ func NewLazyQueue[P constraints.Ordered, V any](setIndex SetIndexCallback[V], pr
} }
// Reset clears the contents of the queue // Reset clears the contents of the queue
func (q *LazyQueue[P, V]) Reset() { func (q *LazyQueue) Reset() {
q.queue[0] = newSstack[P, V](q.setIndex0) q.queue[0] = newSstack(q.setIndex0, false)
q.queue[1] = newSstack[P, V](q.setIndex1) q.queue[1] = newSstack(q.setIndex1, false)
} }
// Refresh performs queue re-evaluation if necessary // Refresh performs queue re-evaluation if necessary
func (q *LazyQueue[P, V]) Refresh() { func (q *LazyQueue) Refresh() {
now := q.clock.Now() now := q.clock.Now()
for time.Duration(now-q.lastRefresh2) >= q.period*2 { for time.Duration(now-q.lastRefresh2) >= q.period*2 {
q.refresh(now) q.refresh(now)
@ -88,10 +87,10 @@ func (q *LazyQueue[P, V]) Refresh() {
} }
// refresh re-evaluates items in the older queue and swaps the two queues // refresh re-evaluates items in the older queue and swaps the two queues
func (q *LazyQueue[P, V]) refresh(now mclock.AbsTime) { func (q *LazyQueue) refresh(now mclock.AbsTime) {
q.maxUntil = now.Add(q.period) q.maxUntil = now.Add(q.period)
for q.queue[0].Len() != 0 { for q.queue[0].Len() != 0 {
q.Push(heap.Pop(q.queue[0]).(*item[P, V]).value) q.Push(heap.Pop(q.queue[0]).(*item).value)
} }
q.queue[0], q.queue[1] = q.queue[1], q.queue[0] q.queue[0], q.queue[1] = q.queue[1], q.queue[0]
q.indexOffset = 1 - q.indexOffset q.indexOffset = 1 - q.indexOffset
@ -99,22 +98,22 @@ func (q *LazyQueue[P, V]) refresh(now mclock.AbsTime) {
} }
// Push adds an item to the queue // Push adds an item to the queue
func (q *LazyQueue[P, V]) Push(data V) { func (q *LazyQueue) Push(data interface{}) {
heap.Push(q.queue[1], &item[P, V]{data, q.maxPriority(data, q.maxUntil)}) heap.Push(q.queue[1], &item{data, q.maxPriority(data, q.maxUntil)})
} }
// Update updates the upper priority estimate for the item with the given queue index // Update updates the upper priority estimate for the item with the given queue index
func (q *LazyQueue[P, V]) Update(index int) { func (q *LazyQueue) Update(index int) {
q.Push(q.Remove(index)) q.Push(q.Remove(index))
} }
// Pop removes and returns the item with the greatest actual priority // Pop removes and returns the item with the greatest actual priority
func (q *LazyQueue[P, V]) Pop() (V, P) { func (q *LazyQueue) Pop() (interface{}, int64) {
var ( var (
resData V resData interface{}
resPri P resPri int64
) )
q.MultiPop(func(data V, priority P) bool { q.MultiPop(func(data interface{}, priority int64) bool {
resData = data resData = data
resPri = priority resPri = priority
return false return false
@ -124,7 +123,7 @@ func (q *LazyQueue[P, V]) Pop() (V, P) {
// peekIndex returns the index of the internal queue where the item with the // peekIndex returns the index of the internal queue where the item with the
// highest estimated priority is or -1 if both are empty // highest estimated priority is or -1 if both are empty
func (q *LazyQueue[P, V]) peekIndex() int { func (q *LazyQueue) peekIndex() int {
if q.queue[0].Len() != 0 { if q.queue[0].Len() != 0 {
if q.queue[1].Len() != 0 && q.queue[1].blocks[0][0].priority > q.queue[0].blocks[0][0].priority { if q.queue[1].Len() != 0 && q.queue[1].blocks[0][0].priority > q.queue[0].blocks[0][0].priority {
return 1 return 1
@ -140,17 +139,17 @@ func (q *LazyQueue[P, V]) peekIndex() int {
// MultiPop pops multiple items from the queue and is more efficient than calling // MultiPop pops multiple items from the queue and is more efficient than calling
// Pop multiple times. Popped items are passed to the callback. MultiPop returns // Pop multiple times. Popped items are passed to the callback. MultiPop returns
// when the callback returns false or there are no more items to pop. // when the callback returns false or there are no more items to pop.
func (q *LazyQueue[P, V]) MultiPop(callback func(data V, priority P) bool) { func (q *LazyQueue) MultiPop(callback func(data interface{}, priority int64) bool) {
nextIndex := q.peekIndex() nextIndex := q.peekIndex()
for nextIndex != -1 { for nextIndex != -1 {
data := heap.Pop(q.queue[nextIndex]).(*item[P, V]).value data := heap.Pop(q.queue[nextIndex]).(*item).value
heap.Push(q.popQueue, &item[P, V]{data, q.priority(data)}) heap.Push(q.popQueue, &item{data, q.priority(data)})
nextIndex = q.peekIndex() nextIndex = q.peekIndex()
for q.popQueue.Len() != 0 && (nextIndex == -1 || q.queue[nextIndex].blocks[0][0].priority < q.popQueue.blocks[0][0].priority) { for q.popQueue.Len() != 0 && (nextIndex == -1 || q.queue[nextIndex].blocks[0][0].priority < q.popQueue.blocks[0][0].priority) {
i := heap.Pop(q.popQueue).(*item[P, V]) i := heap.Pop(q.popQueue).(*item)
if !callback(i.value, i.priority) { if !callback(i.value, i.priority) {
for q.popQueue.Len() != 0 { for q.popQueue.Len() != 0 {
q.Push(heap.Pop(q.popQueue).(*item[P, V]).value) q.Push(heap.Pop(q.popQueue).(*item).value)
} }
return return
} }
@ -160,28 +159,31 @@ func (q *LazyQueue[P, V]) MultiPop(callback func(data V, priority P) bool) {
} }
// PopItem pops the item from the queue only, dropping the associated priority value. // PopItem pops the item from the queue only, dropping the associated priority value.
func (q *LazyQueue[P, V]) PopItem() V { func (q *LazyQueue) PopItem() interface{} {
i, _ := q.Pop() i, _ := q.Pop()
return i return i
} }
// Remove removes the item with the given index. // Remove removes the item with the given index.
func (q *LazyQueue[P, V]) Remove(index int) V { func (q *LazyQueue) Remove(index int) interface{} {
return heap.Remove(q.queue[index&1^q.indexOffset], index>>1).(*item[P, V]).value if index < 0 {
return nil
}
return heap.Remove(q.queue[index&1^q.indexOffset], index>>1).(*item).value
} }
// Empty checks whether the priority queue is empty. // Empty checks whether the priority queue is empty.
func (q *LazyQueue[P, V]) Empty() bool { func (q *LazyQueue) Empty() bool {
return q.queue[0].Len() == 0 && q.queue[1].Len() == 0 return q.queue[0].Len() == 0 && q.queue[1].Len() == 0
} }
// Size returns the number of items in the priority queue. // Size returns the number of items in the priority queue.
func (q *LazyQueue[P, V]) Size() int { func (q *LazyQueue) Size() int {
return q.queue[0].Len() + q.queue[1].Len() return q.queue[0].Len() + q.queue[1].Len()
} }
// setIndex0 translates internal queue item index to the virtual index space of LazyQueue // setIndex0 translates internal queue item index to the virtual index space of LazyQueue
func (q *LazyQueue[P, V]) setIndex0(data V, index int) { func (q *LazyQueue) setIndex0(data interface{}, index int) {
if index == -1 { if index == -1 {
q.setIndex(data, -1) q.setIndex(data, -1)
} else { } else {
@ -190,6 +192,6 @@ func (q *LazyQueue[P, V]) setIndex0(data V, index int) {
} }
// setIndex1 translates internal queue item index to the virtual index space of LazyQueue // setIndex1 translates internal queue item index to the virtual index space of LazyQueue
func (q *LazyQueue[P, V]) setIndex1(data V, index int) { func (q *LazyQueue) setIndex1(data interface{}, index int) {
q.setIndex(data, index+index+1) q.setIndex(data, index+index+1)
} }

View File

@ -19,59 +19,65 @@ package prque
import ( import (
"container/heap" "container/heap"
"golang.org/x/exp/constraints"
) )
// Priority queue data structure. // Priority queue data structure.
type Prque[P constraints.Ordered, V any] struct { type Prque struct {
cont *sstack[P, V] cont *sstack
} }
// New creates a new priority queue. // New creates a new priority queue.
func New[P constraints.Ordered, V any](setIndex SetIndexCallback[V]) *Prque[P, V] { func New(setIndex SetIndexCallback) *Prque {
return &Prque[P, V]{newSstack[P, V](setIndex)} return &Prque{newSstack(setIndex, false)}
}
// NewWrapAround creates a new priority queue with wrap-around priority handling.
func NewWrapAround(setIndex SetIndexCallback) *Prque {
return &Prque{newSstack(setIndex, true)}
} }
// Pushes a value with a given priority into the queue, expanding if necessary. // Pushes a value with a given priority into the queue, expanding if necessary.
func (p *Prque[P, V]) Push(data V, priority P) { func (p *Prque) Push(data interface{}, priority int64) {
heap.Push(p.cont, &item[P, V]{data, priority}) heap.Push(p.cont, &item{data, priority})
} }
// Peek returns the value with the greatest priority but does not pop it off. // Peek returns the value with the greatest priority but does not pop it off.
func (p *Prque[P, V]) Peek() (V, P) { func (p *Prque) Peek() (interface{}, int64) {
item := p.cont.blocks[0][0] item := p.cont.blocks[0][0]
return item.value, item.priority return item.value, item.priority
} }
// Pops the value with the greatest priority off the stack and returns it. // Pops the value with the greatest priority off the stack and returns it.
// Currently no shrinking is done. // Currently no shrinking is done.
func (p *Prque[P, V]) Pop() (V, P) { func (p *Prque) Pop() (interface{}, int64) {
item := heap.Pop(p.cont).(*item[P, V]) item := heap.Pop(p.cont).(*item)
return item.value, item.priority return item.value, item.priority
} }
// Pops only the item from the queue, dropping the associated priority value. // Pops only the item from the queue, dropping the associated priority value.
func (p *Prque[P, V]) PopItem() V { func (p *Prque) PopItem() interface{} {
return heap.Pop(p.cont).(*item[P, V]).value return heap.Pop(p.cont).(*item).value
} }
// Remove removes the element with the given index. // Remove removes the element with the given index.
func (p *Prque[P, V]) Remove(i int) V { func (p *Prque) Remove(i int) interface{} {
return heap.Remove(p.cont, i).(*item[P, V]).value if i < 0 {
return nil
}
return heap.Remove(p.cont, i)
} }
// Checks whether the priority queue is empty. // Checks whether the priority queue is empty.
func (p *Prque[P, V]) Empty() bool { func (p *Prque) Empty() bool {
return p.cont.Len() == 0 return p.cont.Len() == 0
} }
// Returns the number of element in the priority queue. // Returns the number of element in the priority queue.
func (p *Prque[P, V]) Size() int { func (p *Prque) Size() int {
return p.cont.Len() return p.cont.Len()
} }
// Clears the contents of the priority queue. // Clears the contents of the priority queue.
func (p *Prque[P, V]) Reset() { func (p *Prque) Reset() {
*p = *New[P, V](p.cont.setIndex) *p = *New(p.cont.setIndex)
} }

View File

@ -21,24 +21,22 @@ func TestPrque(t *testing.T) {
for i := 0; i < size; i++ { for i := 0; i < size; i++ {
data[i] = rand.Int() data[i] = rand.Int()
} }
queue := New[int, int](nil) queue := New(nil)
for rep := 0; rep < 2; rep++ { for rep := 0; rep < 2; rep++ {
// Fill a priority queue with the above data // Fill a priority queue with the above data
for i := 0; i < size; i++ { for i := 0; i < size; i++ {
queue.Push(data[i], prio[i]) queue.Push(data[i], int64(prio[i]))
if queue.Size() != i+1 { if queue.Size() != i+1 {
t.Errorf("queue size mismatch: have %v, want %v.", queue.Size(), i+1) t.Errorf("queue size mismatch: have %v, want %v.", queue.Size(), i+1)
} }
} }
// Create a map the values to the priorities for easier verification // Create a map the values to the priorities for easier verification
dict := make(map[int]int) dict := make(map[int64]int)
for i := 0; i < size; i++ { for i := 0; i < size; i++ {
dict[prio[i]] = data[i] dict[int64(prio[i])] = data[i]
} }
// Pop out the elements in priority order and verify them // Pop out the elements in priority order and verify them
prevPrio := size + 1 prevPrio := int64(size + 1)
for !queue.Empty() { for !queue.Empty() {
val, prio := queue.Pop() val, prio := queue.Pop()
if prio > prevPrio { if prio > prevPrio {
@ -61,23 +59,22 @@ func TestReset(t *testing.T) {
for i := 0; i < size; i++ { for i := 0; i < size; i++ {
data[i] = rand.Int() data[i] = rand.Int()
} }
queue := New[int, int](nil) queue := New(nil)
for rep := 0; rep < 2; rep++ { for rep := 0; rep < 2; rep++ {
// Fill a priority queue with the above data // Fill a priority queue with the above data
for i := 0; i < size; i++ { for i := 0; i < size; i++ {
queue.Push(data[i], prio[i]) queue.Push(data[i], int64(prio[i]))
if queue.Size() != i+1 { if queue.Size() != i+1 {
t.Errorf("queue size mismatch: have %v, want %v.", queue.Size(), i+1) t.Errorf("queue size mismatch: have %v, want %v.", queue.Size(), i+1)
} }
} }
// Create a map the values to the priorities for easier verification // Create a map the values to the priorities for easier verification
dict := make(map[int]int) dict := make(map[int64]int)
for i := 0; i < size; i++ { for i := 0; i < size; i++ {
dict[prio[i]] = data[i] dict[int64(prio[i])] = data[i]
} }
// Pop out half the elements in priority order and verify them // Pop out half the elements in priority order and verify them
prevPrio := size + 1 prevPrio := int64(size + 1)
for i := 0; i < size/2; i++ { for i := 0; i < size/2; i++ {
val, prio := queue.Pop() val, prio := queue.Pop()
if prio > prevPrio { if prio > prevPrio {
@ -107,7 +104,7 @@ func BenchmarkPush(b *testing.B) {
} }
// Execute the benchmark // Execute the benchmark
b.ResetTimer() b.ResetTimer()
queue := New[int64, int](nil) queue := New(nil)
for i := 0; i < len(data); i++ { for i := 0; i < len(data); i++ {
queue.Push(data[i], prio[i]) queue.Push(data[i], prio[i])
} }
@ -121,7 +118,7 @@ func BenchmarkPop(b *testing.B) {
data[i] = rand.Int() data[i] = rand.Int()
prio[i] = rand.Int63() prio[i] = rand.Int63()
} }
queue := New[int64, int](nil) queue := New(nil)
for i := 0; i < len(data); i++ { for i := 0; i < len(data); i++ {
queue.Push(data[i], prio[i]) queue.Push(data[i], prio[i])
} }

View File

@ -10,50 +10,53 @@
package prque package prque
import "golang.org/x/exp/constraints"
// The size of a block of data // The size of a block of data
const blockSize = 4096 const blockSize = 4096
// A prioritized item in the sorted stack. // A prioritized item in the sorted stack.
type item[P constraints.Ordered, V any] struct { //
value V // Note: priorities can "wrap around" the int64 range, a comes before b if (a.priority - b.priority) > 0.
priority P // The difference between the lowest and highest priorities in the queue at any point should be less than 2^63.
type item struct {
value interface{}
priority int64
} }
// SetIndexCallback is called when the element is moved to a new index. // SetIndexCallback is called when the element is moved to a new index.
// Providing SetIndexCallback is optional, it is needed only if the application needs // Providing SetIndexCallback is optional, it is needed only if the application needs
// to delete elements other than the top one. // to delete elements other than the top one.
type SetIndexCallback[V any] func(data V, index int) type SetIndexCallback func(data interface{}, index int)
// Internal sortable stack data structure. Implements the Push and Pop ops for // Internal sortable stack data structure. Implements the Push and Pop ops for
// the stack (heap) functionality and the Len, Less and Swap methods for the // the stack (heap) functionality and the Len, Less and Swap methods for the
// sortability requirements of the heaps. // sortability requirements of the heaps.
type sstack[P constraints.Ordered, V any] struct { type sstack struct {
setIndex SetIndexCallback[V] setIndex SetIndexCallback
size int size int
capacity int capacity int
offset int offset int
wrapAround bool
blocks [][]*item[P, V] blocks [][]*item
active []*item[P, V] active []*item
} }
// Creates a new, empty stack. // Creates a new, empty stack.
func newSstack[P constraints.Ordered, V any](setIndex SetIndexCallback[V]) *sstack[P, V] { func newSstack(setIndex SetIndexCallback, wrapAround bool) *sstack {
result := new(sstack[P, V]) result := new(sstack)
result.setIndex = setIndex result.setIndex = setIndex
result.active = make([]*item[P, V], blockSize) result.active = make([]*item, blockSize)
result.blocks = [][]*item[P, V]{result.active} result.blocks = [][]*item{result.active}
result.capacity = blockSize result.capacity = blockSize
result.wrapAround = wrapAround
return result return result
} }
// Pushes a value onto the stack, expanding it if necessary. Required by // Pushes a value onto the stack, expanding it if necessary. Required by
// heap.Interface. // heap.Interface.
func (s *sstack[P, V]) Push(data any) { func (s *sstack) Push(data interface{}) {
if s.size == s.capacity { if s.size == s.capacity {
s.active = make([]*item[P, V], blockSize) s.active = make([]*item, blockSize)
s.blocks = append(s.blocks, s.active) s.blocks = append(s.blocks, s.active)
s.capacity += blockSize s.capacity += blockSize
s.offset = 0 s.offset = 0
@ -62,16 +65,16 @@ func (s *sstack[P, V]) Push(data any) {
s.offset = 0 s.offset = 0
} }
if s.setIndex != nil { if s.setIndex != nil {
s.setIndex(data.(*item[P, V]).value, s.size) s.setIndex(data.(*item).value, s.size)
} }
s.active[s.offset] = data.(*item[P, V]) s.active[s.offset] = data.(*item)
s.offset++ s.offset++
s.size++ s.size++
} }
// Pops a value off the stack and returns it. Currently no shrinking is done. // Pops a value off the stack and returns it. Currently no shrinking is done.
// Required by heap.Interface. // Required by heap.Interface.
func (s *sstack[P, V]) Pop() (res any) { func (s *sstack) Pop() (res interface{}) {
s.size-- s.size--
s.offset-- s.offset--
if s.offset < 0 { if s.offset < 0 {
@ -80,24 +83,28 @@ func (s *sstack[P, V]) Pop() (res any) {
} }
res, s.active[s.offset] = s.active[s.offset], nil res, s.active[s.offset] = s.active[s.offset], nil
if s.setIndex != nil { if s.setIndex != nil {
s.setIndex(res.(*item[P, V]).value, -1) s.setIndex(res.(*item).value, -1)
} }
return return
} }
// Returns the length of the stack. Required by sort.Interface. // Returns the length of the stack. Required by sort.Interface.
func (s *sstack[P, V]) Len() int { func (s *sstack) Len() int {
return s.size return s.size
} }
// Compares the priority of two elements of the stack (higher is first). // Compares the priority of two elements of the stack (higher is first).
// Required by sort.Interface. // Required by sort.Interface.
func (s *sstack[P, V]) Less(i, j int) bool { func (s *sstack) Less(i, j int) bool {
return s.blocks[i/blockSize][i%blockSize].priority > s.blocks[j/blockSize][j%blockSize].priority a, b := s.blocks[i/blockSize][i%blockSize].priority, s.blocks[j/blockSize][j%blockSize].priority
if s.wrapAround {
return a-b > 0
}
return a > b
} }
// Swaps two elements in the stack. Required by sort.Interface. // Swaps two elements in the stack. Required by sort.Interface.
func (s *sstack[P, V]) Swap(i, j int) { func (s *sstack) Swap(i, j int) {
ib, io, jb, jo := i/blockSize, i%blockSize, j/blockSize, j%blockSize ib, io, jb, jo := i/blockSize, i%blockSize, j/blockSize, j%blockSize
a, b := s.blocks[jb][jo], s.blocks[ib][io] a, b := s.blocks[jb][jo], s.blocks[ib][io]
if s.setIndex != nil { if s.setIndex != nil {
@ -108,6 +115,6 @@ func (s *sstack[P, V]) Swap(i, j int) {
} }
// Resets the stack, effectively clearing its contents. // Resets the stack, effectively clearing its contents.
func (s *sstack[P, V]) Reset() { func (s *sstack) Reset() {
*s = *newSstack[P, V](s.setIndex) *s = *newSstack(s.setIndex, false)
} }

View File

@ -17,23 +17,23 @@ import (
func TestSstack(t *testing.T) { func TestSstack(t *testing.T) {
// Create some initial data // Create some initial data
size := 16 * blockSize size := 16 * blockSize
data := make([]*item[int64, int], size) data := make([]*item, size)
for i := 0; i < size; i++ { for i := 0; i < size; i++ {
data[i] = &item[int64, int]{rand.Int(), rand.Int63()} data[i] = &item{rand.Int(), rand.Int63()}
} }
stack := newSstack[int64, int](nil) stack := newSstack(nil, false)
for rep := 0; rep < 2; rep++ { for rep := 0; rep < 2; rep++ {
// Push all the data into the stack, pop out every second // Push all the data into the stack, pop out every second
secs := []*item[int64, int]{} secs := []*item{}
for i := 0; i < size; i++ { for i := 0; i < size; i++ {
stack.Push(data[i]) stack.Push(data[i])
if i%2 == 0 { if i%2 == 0 {
secs = append(secs, stack.Pop().(*item[int64, int])) secs = append(secs, stack.Pop().(*item))
} }
} }
rest := []*item[int64, int]{} rest := []*item{}
for stack.Len() > 0 { for stack.Len() > 0 {
rest = append(rest, stack.Pop().(*item[int64, int])) rest = append(rest, stack.Pop().(*item))
} }
// Make sure the contents of the resulting slices are ok // Make sure the contents of the resulting slices are ok
for i := 0; i < size; i++ { for i := 0; i < size; i++ {
@ -50,12 +50,12 @@ func TestSstack(t *testing.T) {
func TestSstackSort(t *testing.T) { func TestSstackSort(t *testing.T) {
// Create some initial data // Create some initial data
size := 16 * blockSize size := 16 * blockSize
data := make([]*item[int64, int], size) data := make([]*item, size)
for i := 0; i < size; i++ { for i := 0; i < size; i++ {
data[i] = &item[int64, int]{rand.Int(), int64(i)} data[i] = &item{rand.Int(), int64(i)}
} }
// Push all the data into the stack // Push all the data into the stack
stack := newSstack[int64, int](nil) stack := newSstack(nil, false)
for _, val := range data { for _, val := range data {
stack.Push(val) stack.Push(val)
} }
@ -72,18 +72,18 @@ func TestSstackSort(t *testing.T) {
func TestSstackReset(t *testing.T) { func TestSstackReset(t *testing.T) {
// Create some initial data // Create some initial data
size := 16 * blockSize size := 16 * blockSize
data := make([]*item[int64, int], size) data := make([]*item, size)
for i := 0; i < size; i++ { for i := 0; i < size; i++ {
data[i] = &item[int64, int]{rand.Int(), rand.Int63()} data[i] = &item{rand.Int(), rand.Int63()}
} }
stack := newSstack[int64, int](nil) stack := newSstack(nil, false)
for rep := 0; rep < 2; rep++ { for rep := 0; rep < 2; rep++ {
// Push all the data into the stack, pop out every second // Push all the data into the stack, pop out every second
secs := []*item[int64, int]{} secs := []*item{}
for i := 0; i < size; i++ { for i := 0; i < size; i++ {
stack.Push(data[i]) stack.Push(data[i])
if i%2 == 0 { if i%2 == 0 {
secs = append(secs, stack.Pop().(*item[int64, int])) secs = append(secs, stack.Pop().(*item))
} }
} }
// Reset and verify both pulled and stack contents // Reset and verify both pulled and stack contents

View File

@ -169,14 +169,14 @@ type BlockChain struct {
chainConfig *params.ChainConfig // Chain & network configuration chainConfig *params.ChainConfig // Chain & network configuration
cacheConfig *CacheConfig // Cache configuration for pruning cacheConfig *CacheConfig // Cache configuration for pruning
db ethdb.Database // Low level persistent database to store final content in db ethdb.Database // Low level persistent database to store final content in
snaps *snapshot.Tree // Snapshot tree for fast trie leaf access snaps *snapshot.Tree // Snapshot tree for fast trie leaf access
triegc *prque.Prque[int64, common.Hash] // Priority queue mapping block numbers to tries to gc triegc *prque.Prque // Priority queue mapping block numbers to tries to gc
gcproc time.Duration // Accumulates canonical block processing for trie dumping gcproc time.Duration // Accumulates canonical block processing for trie dumping
lastWrite uint64 // Last block when the state was flushed lastWrite uint64 // Last block when the state was flushed
flushInterval int64 // Time interval (processing time) after which to flush a state flushInterval int64 // Time interval (processing time) after which to flush a state
triedb *trie.Database // The database handler for maintaining trie nodes. triedb *trie.Database // The database handler for maintaining trie nodes.
stateCache state.Database // State database to reuse between imports (contains state cache) stateCache state.Database // State database to reuse between imports (contains state cache)
// txLookupLimit is the maximum number of blocks from head whose tx indices // txLookupLimit is the maximum number of blocks from head whose tx indices
// are reserved: // are reserved:
@ -261,7 +261,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
db: db, db: db,
triedb: triedb, triedb: triedb,
flushInterval: int64(cacheConfig.TrieTimeLimit), flushInterval: int64(cacheConfig.TrieTimeLimit),
triegc: prque.New[int64, common.Hash](nil), triegc: prque.New(nil),
quit: make(chan struct{}), quit: make(chan struct{}),
chainmu: syncx.NewClosableMutex(), chainmu: syncx.NewClosableMutex(),
bodyCache: lru.NewCache[common.Hash, *types.Body](bodyCacheLimit), bodyCache: lru.NewCache[common.Hash, *types.Body](bodyCacheLimit),
@ -957,7 +957,7 @@ func (bc *BlockChain) Stop() {
} }
} }
for !bc.triegc.Empty() { for !bc.triegc.Empty() {
triedb.Dereference(bc.triegc.PopItem()) triedb.Dereference(bc.triegc.PopItem().(common.Hash))
} }
if size, _ := triedb.Size(); size != 0 { if size, _ := triedb.Size(); size != 0 {
log.Error("Dangling trie nodes after full cleanup") log.Error("Dangling trie nodes after full cleanup")
@ -1391,7 +1391,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
bc.triegc.Push(root, number) bc.triegc.Push(root, number)
break break
} }
bc.triedb.Dereference(root) bc.triedb.Dereference(root.(common.Hash))
} }
return nil return nil
} }

View File

@ -191,7 +191,7 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan
// in to be [to-1]. Therefore, setting lastNum to means that the // in to be [to-1]. Therefore, setting lastNum to means that the
// prqueue gap-evaluation will work correctly // prqueue gap-evaluation will work correctly
lastNum = to lastNum = to
queue = prque.New[int64, *blockTxHashes](nil) queue = prque.New(nil)
// for stats reporting // for stats reporting
blocks, txs = 0, 0 blocks, txs = 0, 0
) )
@ -210,7 +210,7 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan
break break
} }
// Next block available, pop it off and index it // Next block available, pop it off and index it
delivery := queue.PopItem() delivery := queue.PopItem().(*blockTxHashes)
lastNum = delivery.number lastNum = delivery.number
WriteTxLookupEntries(batch, delivery.number, delivery.hashes) WriteTxLookupEntries(batch, delivery.number, delivery.hashes)
blocks++ blocks++
@ -282,7 +282,7 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch
// we expect the first number to come in to be [from]. Therefore, setting // we expect the first number to come in to be [from]. Therefore, setting
// nextNum to from means that the prqueue gap-evaluation will work correctly // nextNum to from means that the prqueue gap-evaluation will work correctly
nextNum = from nextNum = from
queue = prque.New[int64, *blockTxHashes](nil) queue = prque.New(nil)
// for stats reporting // for stats reporting
blocks, txs = 0, 0 blocks, txs = 0, 0
) )
@ -299,7 +299,7 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch
if hook != nil && !hook(nextNum) { if hook != nil && !hook(nextNum) {
break break
} }
delivery := queue.PopItem() delivery := queue.PopItem().(*blockTxHashes)
nextNum = delivery.number + 1 nextNum = delivery.number + 1
DeleteTxLookupEntries(batch, delivery.hashes) DeleteTxLookupEntries(batch, delivery.hashes)
txs += len(delivery.hashes) txs += len(delivery.hashes)

View File

@ -1395,7 +1395,7 @@ func (pool *TxPool) truncatePending() {
pendingBeforeCap := pending pendingBeforeCap := pending
// Assemble a spam order to penalize large transactors first // Assemble a spam order to penalize large transactors first
spammers := prque.New[int64, common.Address](nil) spammers := prque.New(nil)
for addr, list := range pool.pending { for addr, list := range pool.pending {
// Only evict transactions from high rollers // Only evict transactions from high rollers
if !pool.locals.contains(addr) && uint64(list.Len()) > pool.config.AccountSlots { if !pool.locals.contains(addr) && uint64(list.Len()) > pool.config.AccountSlots {
@ -1407,12 +1407,12 @@ func (pool *TxPool) truncatePending() {
for pending > pool.config.GlobalSlots && !spammers.Empty() { for pending > pool.config.GlobalSlots && !spammers.Empty() {
// Retrieve the next offender if not local address // Retrieve the next offender if not local address
offender, _ := spammers.Pop() offender, _ := spammers.Pop()
offenders = append(offenders, offender) offenders = append(offenders, offender.(common.Address))
// Equalize balances until all the same or below threshold // Equalize balances until all the same or below threshold
if len(offenders) > 1 { if len(offenders) > 1 {
// Calculate the equalization threshold for all current offenders // Calculate the equalization threshold for all current offenders
threshold := pool.pending[offender].Len() threshold := pool.pending[offender.(common.Address)].Len()
// Iteratively reduce all offenders until below limit or threshold reached // Iteratively reduce all offenders until below limit or threshold reached
for pending > pool.config.GlobalSlots && pool.pending[offenders[len(offenders)-2]].Len() > threshold { for pending > pool.config.GlobalSlots && pool.pending[offenders[len(offenders)-2]].Len() > threshold {

View File

@ -91,8 +91,8 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error {
} }
}() }()
ordering := make(map[*eth.Request]int) ordering := make(map[*eth.Request]int)
timeouts := prque.New[int64, *eth.Request](func(data *eth.Request, index int) { timeouts := prque.New(func(data interface{}, index int) {
ordering[data] = index ordering[data.(*eth.Request)] = index
}) })
timeout := time.NewTimer(0) timeout := time.NewTimer(0)
@ -268,12 +268,14 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error {
// below is purely for to catch programming errors, given the correct // below is purely for to catch programming errors, given the correct
// code, there's no possible order of events that should result in a // code, there's no possible order of events that should result in a
// timeout firing for a non-existent event. // timeout firing for a non-existent event.
req, exp := timeouts.Peek() item, exp := timeouts.Peek()
if now, at := time.Now(), time.Unix(0, -exp); now.Before(at) { if now, at := time.Now(), time.Unix(0, -exp); now.Before(at) {
log.Error("Timeout triggered but not reached", "left", at.Sub(now)) log.Error("Timeout triggered but not reached", "left", at.Sub(now))
timeout.Reset(at.Sub(now)) timeout.Reset(at.Sub(now))
continue continue
} }
req := item.(*eth.Request)
// Stop tracking the timed out request from a timing perspective, // Stop tracking the timed out request from a timing perspective,
// cancel it, so it's not considered in-flight anymore, but keep // cancel it, so it's not considered in-flight anymore, but keep
// the peer marked busy to prevent assigning a second request and // the peer marked busy to prevent assigning a second request and

View File

@ -115,7 +115,7 @@ type queue struct {
// Headers are "special", they download in batches, supported by a skeleton chain // Headers are "special", they download in batches, supported by a skeleton chain
headerHead common.Hash // Hash of the last queued header to verify order headerHead common.Hash // Hash of the last queued header to verify order
headerTaskPool map[uint64]*types.Header // Pending header retrieval tasks, mapping starting indexes to skeleton headers headerTaskPool map[uint64]*types.Header // Pending header retrieval tasks, mapping starting indexes to skeleton headers
headerTaskQueue *prque.Prque[int64, uint64] // Priority queue of the skeleton indexes to fetch the filling headers for headerTaskQueue *prque.Prque // Priority queue of the skeleton indexes to fetch the filling headers for
headerPeerMiss map[string]map[uint64]struct{} // Set of per-peer header batches known to be unavailable headerPeerMiss map[string]map[uint64]struct{} // Set of per-peer header batches known to be unavailable
headerPendPool map[string]*fetchRequest // Currently pending header retrieval operations headerPendPool map[string]*fetchRequest // Currently pending header retrieval operations
headerResults []*types.Header // Result cache accumulating the completed headers headerResults []*types.Header // Result cache accumulating the completed headers
@ -125,15 +125,15 @@ type queue struct {
headerContCh chan bool // Channel to notify when header download finishes headerContCh chan bool // Channel to notify when header download finishes
// All data retrievals below are based on an already assembles header chain // All data retrievals below are based on an already assembles header chain
blockTaskPool map[common.Hash]*types.Header // Pending block (body) retrieval tasks, mapping hashes to headers blockTaskPool map[common.Hash]*types.Header // Pending block (body) retrieval tasks, mapping hashes to headers
blockTaskQueue *prque.Prque[int64, *types.Header] // Priority queue of the headers to fetch the blocks (bodies) for blockTaskQueue *prque.Prque // Priority queue of the headers to fetch the blocks (bodies) for
blockPendPool map[string]*fetchRequest // Currently pending block (body) retrieval operations blockPendPool map[string]*fetchRequest // Currently pending block (body) retrieval operations
blockWakeCh chan bool // Channel to notify the block fetcher of new tasks blockWakeCh chan bool // Channel to notify the block fetcher of new tasks
receiptTaskPool map[common.Hash]*types.Header // Pending receipt retrieval tasks, mapping hashes to headers receiptTaskPool map[common.Hash]*types.Header // Pending receipt retrieval tasks, mapping hashes to headers
receiptTaskQueue *prque.Prque[int64, *types.Header] // Priority queue of the headers to fetch the receipts for receiptTaskQueue *prque.Prque // Priority queue of the headers to fetch the receipts for
receiptPendPool map[string]*fetchRequest // Currently pending receipt retrieval operations receiptPendPool map[string]*fetchRequest // Currently pending receipt retrieval operations
receiptWakeCh chan bool // Channel to notify when receipt fetcher of new tasks receiptWakeCh chan bool // Channel to notify when receipt fetcher of new tasks
resultCache *resultStore // Downloaded but not yet delivered fetch results resultCache *resultStore // Downloaded but not yet delivered fetch results
resultSize common.StorageSize // Approximate size of a block (exponential moving average) resultSize common.StorageSize // Approximate size of a block (exponential moving average)
@ -150,9 +150,9 @@ func newQueue(blockCacheLimit int, thresholdInitialSize int) *queue {
lock := new(sync.RWMutex) lock := new(sync.RWMutex)
q := &queue{ q := &queue{
headerContCh: make(chan bool, 1), headerContCh: make(chan bool, 1),
blockTaskQueue: prque.New[int64, *types.Header](nil), blockTaskQueue: prque.New(nil),
blockWakeCh: make(chan bool, 1), blockWakeCh: make(chan bool, 1),
receiptTaskQueue: prque.New[int64, *types.Header](nil), receiptTaskQueue: prque.New(nil),
receiptWakeCh: make(chan bool, 1), receiptWakeCh: make(chan bool, 1),
active: sync.NewCond(lock), active: sync.NewCond(lock),
lock: lock, lock: lock,
@ -258,7 +258,7 @@ func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) {
} }
// Schedule all the header retrieval tasks for the skeleton assembly // Schedule all the header retrieval tasks for the skeleton assembly
q.headerTaskPool = make(map[uint64]*types.Header) q.headerTaskPool = make(map[uint64]*types.Header)
q.headerTaskQueue = prque.New[int64, uint64](nil) q.headerTaskQueue = prque.New(nil)
q.headerPeerMiss = make(map[string]map[uint64]struct{}) // Reset availability to correct invalid chains q.headerPeerMiss = make(map[string]map[uint64]struct{}) // Reset availability to correct invalid chains
q.headerResults = make([]*types.Header, len(skeleton)*MaxHeaderFetch) q.headerResults = make([]*types.Header, len(skeleton)*MaxHeaderFetch)
q.headerHashes = make([]common.Hash, len(skeleton)*MaxHeaderFetch) q.headerHashes = make([]common.Hash, len(skeleton)*MaxHeaderFetch)
@ -428,12 +428,12 @@ func (q *queue) ReserveHeaders(p *peerConnection, count int) *fetchRequest {
for send == 0 && !q.headerTaskQueue.Empty() { for send == 0 && !q.headerTaskQueue.Empty() {
from, _ := q.headerTaskQueue.Pop() from, _ := q.headerTaskQueue.Pop()
if q.headerPeerMiss[p.id] != nil { if q.headerPeerMiss[p.id] != nil {
if _, ok := q.headerPeerMiss[p.id][from]; ok { if _, ok := q.headerPeerMiss[p.id][from.(uint64)]; ok {
skip = append(skip, from) skip = append(skip, from.(uint64))
continue continue
} }
} }
send = from send = from.(uint64)
} }
// Merge all the skipped batches back // Merge all the skipped batches back
for _, from := range skip { for _, from := range skip {
@ -485,7 +485,7 @@ func (q *queue) ReserveReceipts(p *peerConnection, count int) (*fetchRequest, bo
// item - the fetchRequest // item - the fetchRequest
// progress - whether any progress was made // progress - whether any progress was made
// throttle - if the caller should throttle for a while // throttle - if the caller should throttle for a while
func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque[int64, *types.Header], func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque,
pendPool map[string]*fetchRequest, kind uint) (*fetchRequest, bool, bool) { pendPool map[string]*fetchRequest, kind uint) (*fetchRequest, bool, bool) {
// Short circuit if the pool has been depleted, or if the peer's already // Short circuit if the pool has been depleted, or if the peer's already
// downloading something (sanity check not to corrupt state) // downloading something (sanity check not to corrupt state)
@ -503,8 +503,8 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common
for proc := 0; len(send) < count && !taskQueue.Empty(); proc++ { for proc := 0; len(send) < count && !taskQueue.Empty(); proc++ {
// the task queue will pop items in order, so the highest prio block // the task queue will pop items in order, so the highest prio block
// is also the lowest block number. // is also the lowest block number.
header, _ := taskQueue.Peek() h, _ := taskQueue.Peek()
header := h.(*types.Header)
// we can ask the resultcache if this header is within the // we can ask the resultcache if this header is within the
// "prioritized" segment of blocks. If it is not, we need to throttle // "prioritized" segment of blocks. If it is not, we need to throttle
@ -627,14 +627,12 @@ func (q *queue) ExpireReceipts(peer string) int {
} }
// expire is the generic check that moves a specific expired task from a pending // expire is the generic check that moves a specific expired task from a pending
// pool back into a task pool. The syntax on the passed taskQueue is a bit weird // pool back into a task pool.
// as we would need a generic expire method to handle both types, but that is not
// supported at the moment at least (Go 1.19).
// //
// Note, this method expects the queue lock to be already held. The reason the // Note, this method expects the queue lock to be already held. The reason the
// lock is not obtained in here is that the parameters already need to access // lock is not obtained in here is that the parameters already need to access
// the queue, so they already need a lock anyway. // the queue, so they already need a lock anyway.
func (q *queue) expire(peer string, pendPool map[string]*fetchRequest, taskQueue interface{}) int { func (q *queue) expire(peer string, pendPool map[string]*fetchRequest, taskQueue *prque.Prque) int {
// Retrieve the request being expired and log an error if it's non-existent, // Retrieve the request being expired and log an error if it's non-existent,
// as there's no order of events that should lead to such expirations. // as there's no order of events that should lead to such expirations.
req := pendPool[peer] req := pendPool[peer]
@ -646,10 +644,10 @@ func (q *queue) expire(peer string, pendPool map[string]*fetchRequest, taskQueue
// Return any non-satisfied requests to the pool // Return any non-satisfied requests to the pool
if req.From > 0 { if req.From > 0 {
taskQueue.(*prque.Prque[int64, uint64]).Push(req.From, -int64(req.From)) taskQueue.Push(req.From, -int64(req.From))
} }
for _, header := range req.Headers { for _, header := range req.Headers {
taskQueue.(*prque.Prque[int64, *types.Header]).Push(header, -int64(header.Number.Uint64())) taskQueue.Push(header, -int64(header.Number.Uint64()))
} }
return len(req.Headers) return len(req.Headers)
} }
@ -826,7 +824,7 @@ func (q *queue) DeliverReceipts(id string, receiptList [][]*types.Receipt, recei
// reason this lock is not obtained in here is because the parameters already need // reason this lock is not obtained in here is because the parameters already need
// to access the queue, so they already need a lock anyway. // to access the queue, so they already need a lock anyway.
func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header,
taskQueue *prque.Prque[int64, *types.Header], pendPool map[string]*fetchRequest, taskQueue *prque.Prque, pendPool map[string]*fetchRequest,
reqTimer metrics.Timer, resInMeter metrics.Meter, resDropMeter metrics.Meter, reqTimer metrics.Timer, resInMeter metrics.Meter, resDropMeter metrics.Meter,
results int, validate func(index int, header *types.Header) error, results int, validate func(index int, header *types.Header) error,
reconstruct func(index int, result *fetchResult)) (int, error) { reconstruct func(index int, result *fetchResult)) (int, error) {

View File

@ -175,9 +175,9 @@ type BlockFetcher struct {
completing map[common.Hash]*blockAnnounce // Blocks with headers, currently body-completing completing map[common.Hash]*blockAnnounce // Blocks with headers, currently body-completing
// Block cache // Block cache
queue *prque.Prque[int64, *blockOrHeaderInject] // Queue containing the import operations (block number sorted) queue *prque.Prque // Queue containing the import operations (block number sorted)
queues map[string]int // Per peer block counts to prevent memory exhaustion queues map[string]int // Per peer block counts to prevent memory exhaustion
queued map[common.Hash]*blockOrHeaderInject // Set of already queued blocks (to dedup imports) queued map[common.Hash]*blockOrHeaderInject // Set of already queued blocks (to dedup imports)
// Callbacks // Callbacks
getHeader HeaderRetrievalFn // Retrieves a header from the local chain getHeader HeaderRetrievalFn // Retrieves a header from the local chain
@ -212,7 +212,7 @@ func NewBlockFetcher(light bool, getHeader HeaderRetrievalFn, getBlock blockRetr
fetching: make(map[common.Hash]*blockAnnounce), fetching: make(map[common.Hash]*blockAnnounce),
fetched: make(map[common.Hash][]*blockAnnounce), fetched: make(map[common.Hash][]*blockAnnounce),
completing: make(map[common.Hash]*blockAnnounce), completing: make(map[common.Hash]*blockAnnounce),
queue: prque.New[int64, *blockOrHeaderInject](nil), queue: prque.New(nil),
queues: make(map[string]int), queues: make(map[string]int),
queued: make(map[common.Hash]*blockOrHeaderInject), queued: make(map[common.Hash]*blockOrHeaderInject),
getHeader: getHeader, getHeader: getHeader,
@ -351,7 +351,7 @@ func (f *BlockFetcher) loop() {
// Import any queued blocks that could potentially fit // Import any queued blocks that could potentially fit
height := f.chainHeight() height := f.chainHeight()
for !f.queue.Empty() { for !f.queue.Empty() {
op := f.queue.PopItem() op := f.queue.PopItem().(*blockOrHeaderInject)
hash := op.hash() hash := op.hash()
if f.queueChangeHook != nil { if f.queueChangeHook != nil {
f.queueChangeHook(hash, false) f.queueChangeHook(hash, false)

6
go.mod
View File

@ -59,12 +59,11 @@ require (
github.com/tyler-smith/go-bip39 v1.1.0 github.com/tyler-smith/go-bip39 v1.1.0
github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa
golang.org/x/crypto v0.1.0 golang.org/x/crypto v0.1.0
golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
golang.org/x/sys v0.3.0 golang.org/x/sys v0.3.0
golang.org/x/text v0.4.0 golang.org/x/text v0.4.0
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
golang.org/x/tools v0.2.0 golang.org/x/tools v0.1.12
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce
) )
@ -117,7 +116,8 @@ require (
github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect
github.com/tklauser/numcpus v0.2.2 // indirect github.com/tklauser/numcpus v0.2.2 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
golang.org/x/mod v0.6.0 // indirect golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/net v0.1.0 // indirect golang.org/x/net v0.1.0 // indirect
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
google.golang.org/protobuf v1.27.1 // indirect google.golang.org/protobuf v1.27.1 // indirect

14
go.sum
View File

@ -298,7 +298,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa h1:Q75Upo5UN4JbPFURXZ8nLKYUvF85dyFRop/vQ0Rv+64= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa h1:Q75Upo5UN4JbPFURXZ8nLKYUvF85dyFRop/vQ0Rv+64=
@ -657,8 +657,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9 h1:yZNXmy+j/JpX19vZkVktWqAo7Gny4PBWYYK3zskGpx4= golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5 h1:rxKZ2gOnYxjfmakvUUqh9Gyb6KXfrj7JWTxORTYqb0E=
golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@ -680,8 +680,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -872,8 +872,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

View File

@ -115,7 +115,7 @@ type queue struct {
// Headers are "special", they download in batches, supported by a skeleton chain // Headers are "special", they download in batches, supported by a skeleton chain
headerHead common.Hash // Hash of the last queued header to verify order headerHead common.Hash // Hash of the last queued header to verify order
headerTaskPool map[uint64]*types.Header // Pending header retrieval tasks, mapping starting indexes to skeleton headers headerTaskPool map[uint64]*types.Header // Pending header retrieval tasks, mapping starting indexes to skeleton headers
headerTaskQueue *prque.Prque[int64, uint64] // Priority queue of the skeleton indexes to fetch the filling headers for headerTaskQueue *prque.Prque // Priority queue of the skeleton indexes to fetch the filling headers for
headerPeerMiss map[string]map[uint64]struct{} // Set of per-peer header batches known to be unavailable headerPeerMiss map[string]map[uint64]struct{} // Set of per-peer header batches known to be unavailable
headerPendPool map[string]*fetchRequest // Currently pending header retrieval operations headerPendPool map[string]*fetchRequest // Currently pending header retrieval operations
headerResults []*types.Header // Result cache accumulating the completed headers headerResults []*types.Header // Result cache accumulating the completed headers
@ -124,13 +124,13 @@ type queue struct {
headerContCh chan bool // Channel to notify when header download finishes headerContCh chan bool // Channel to notify when header download finishes
// All data retrievals below are based on an already assembles header chain // All data retrievals below are based on an already assembles header chain
blockTaskPool map[common.Hash]*types.Header // Pending block (body) retrieval tasks, mapping hashes to headers blockTaskPool map[common.Hash]*types.Header // Pending block (body) retrieval tasks, mapping hashes to headers
blockTaskQueue *prque.Prque[int64, *types.Header] // Priority queue of the headers to fetch the blocks (bodies) for blockTaskQueue *prque.Prque // Priority queue of the headers to fetch the blocks (bodies) for
blockPendPool map[string]*fetchRequest // Currently pending block (body) retrieval operations blockPendPool map[string]*fetchRequest // Currently pending block (body) retrieval operations
receiptTaskPool map[common.Hash]*types.Header // Pending receipt retrieval tasks, mapping hashes to headers receiptTaskPool map[common.Hash]*types.Header // Pending receipt retrieval tasks, mapping hashes to headers
receiptTaskQueue *prque.Prque[int64, *types.Header] // Priority queue of the headers to fetch the receipts for receiptTaskQueue *prque.Prque // Priority queue of the headers to fetch the receipts for
receiptPendPool map[string]*fetchRequest // Currently pending receipt retrieval operations receiptPendPool map[string]*fetchRequest // Currently pending receipt retrieval operations
resultCache *resultStore // Downloaded but not yet delivered fetch results resultCache *resultStore // Downloaded but not yet delivered fetch results
resultSize common.StorageSize // Approximate size of a block (exponential moving average) resultSize common.StorageSize // Approximate size of a block (exponential moving average)
@ -147,8 +147,8 @@ func newQueue(blockCacheLimit int, thresholdInitialSize int) *queue {
lock := new(sync.RWMutex) lock := new(sync.RWMutex)
q := &queue{ q := &queue{
headerContCh: make(chan bool), headerContCh: make(chan bool),
blockTaskQueue: prque.New[int64, *types.Header](nil), blockTaskQueue: prque.New(nil),
receiptTaskQueue: prque.New[int64, *types.Header](nil), receiptTaskQueue: prque.New(nil),
active: sync.NewCond(lock), active: sync.NewCond(lock),
lock: lock, lock: lock,
} }
@ -262,7 +262,7 @@ func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) {
} }
// Schedule all the header retrieval tasks for the skeleton assembly // Schedule all the header retrieval tasks for the skeleton assembly
q.headerTaskPool = make(map[uint64]*types.Header) q.headerTaskPool = make(map[uint64]*types.Header)
q.headerTaskQueue = prque.New[int64, uint64](nil) q.headerTaskQueue = prque.New(nil)
q.headerPeerMiss = make(map[string]map[uint64]struct{}) // Reset availability to correct invalid chains q.headerPeerMiss = make(map[string]map[uint64]struct{}) // Reset availability to correct invalid chains
q.headerResults = make([]*types.Header, len(skeleton)*MaxHeaderFetch) q.headerResults = make([]*types.Header, len(skeleton)*MaxHeaderFetch)
q.headerProced = 0 q.headerProced = 0
@ -424,12 +424,12 @@ func (q *queue) ReserveHeaders(p *peerConnection, count int) *fetchRequest {
for send == 0 && !q.headerTaskQueue.Empty() { for send == 0 && !q.headerTaskQueue.Empty() {
from, _ := q.headerTaskQueue.Pop() from, _ := q.headerTaskQueue.Pop()
if q.headerPeerMiss[p.id] != nil { if q.headerPeerMiss[p.id] != nil {
if _, ok := q.headerPeerMiss[p.id][from]; ok { if _, ok := q.headerPeerMiss[p.id][from.(uint64)]; ok {
skip = append(skip, from) skip = append(skip, from.(uint64))
continue continue
} }
} }
send = from send = from.(uint64)
} }
// Merge all the skipped batches back // Merge all the skipped batches back
for _, from := range skip { for _, from := range skip {
@ -481,7 +481,7 @@ func (q *queue) ReserveReceipts(p *peerConnection, count int) (*fetchRequest, bo
// item - the fetchRequest // item - the fetchRequest
// progress - whether any progress was made // progress - whether any progress was made
// throttle - if the caller should throttle for a while // throttle - if the caller should throttle for a while
func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque[int64, *types.Header], func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque,
pendPool map[string]*fetchRequest, kind uint) (*fetchRequest, bool, bool) { pendPool map[string]*fetchRequest, kind uint) (*fetchRequest, bool, bool) {
// Short circuit if the pool has been depleted, or if the peer's already // Short circuit if the pool has been depleted, or if the peer's already
// downloading something (sanity check not to corrupt state) // downloading something (sanity check not to corrupt state)
@ -499,8 +499,8 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common
for proc := 0; len(send) < count && !taskQueue.Empty(); proc++ { for proc := 0; len(send) < count && !taskQueue.Empty(); proc++ {
// the task queue will pop items in order, so the highest prio block // the task queue will pop items in order, so the highest prio block
// is also the lowest block number. // is also the lowest block number.
header, _ := taskQueue.Peek() h, _ := taskQueue.Peek()
header := h.(*types.Header)
// we can ask the resultcache if this header is within the // we can ask the resultcache if this header is within the
// "prioritized" segment of blocks. If it is not, we need to throttle // "prioritized" segment of blocks. If it is not, we need to throttle
@ -591,12 +591,12 @@ func (q *queue) CancelReceipts(request *fetchRequest) {
} }
// Cancel aborts a fetch request, returning all pending hashes to the task queue. // Cancel aborts a fetch request, returning all pending hashes to the task queue.
func (q *queue) cancel(request *fetchRequest, taskQueue interface{}, pendPool map[string]*fetchRequest) { func (q *queue) cancel(request *fetchRequest, taskQueue *prque.Prque, pendPool map[string]*fetchRequest) {
if request.From > 0 { if request.From > 0 {
taskQueue.(*prque.Prque[int64, uint64]).Push(request.From, -int64(request.From)) taskQueue.Push(request.From, -int64(request.From))
} }
for _, header := range request.Headers { for _, header := range request.Headers {
taskQueue.(*prque.Prque[int64, *types.Header]).Push(header, -int64(header.Number.Uint64())) taskQueue.Push(header, -int64(header.Number.Uint64()))
} }
delete(pendPool, request.Peer.id) delete(pendPool, request.Peer.id)
} }
@ -655,7 +655,7 @@ func (q *queue) ExpireReceipts(timeout time.Duration) map[string]int {
// Note, this method expects the queue lock to be already held. The // Note, this method expects the queue lock to be already held. The
// reason the lock is not obtained in here is because the parameters already need // reason the lock is not obtained in here is because the parameters already need
// to access the queue, so they already need a lock anyway. // to access the queue, so they already need a lock anyway.
func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, taskQueue interface{}, timeoutMeter metrics.Meter) map[string]int { func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, taskQueue *prque.Prque, timeoutMeter metrics.Meter) map[string]int {
// Iterate over the expired requests and return each to the queue // Iterate over the expired requests and return each to the queue
expiries := make(map[string]int) expiries := make(map[string]int)
for id, request := range pendPool { for id, request := range pendPool {
@ -665,10 +665,10 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest,
// Return any non satisfied requests to the pool // Return any non satisfied requests to the pool
if request.From > 0 { if request.From > 0 {
taskQueue.(*prque.Prque[int64, uint64]).Push(request.From, -int64(request.From)) taskQueue.Push(request.From, -int64(request.From))
} }
for _, header := range request.Headers { for _, header := range request.Headers {
taskQueue.(*prque.Prque[int64, *types.Header]).Push(header, -int64(header.Number.Uint64())) taskQueue.Push(header, -int64(header.Number.Uint64()))
} }
// Add the peer to the expiry report along the number of failed requests // Add the peer to the expiry report along the number of failed requests
expiries[id] = len(request.Headers) expiries[id] = len(request.Headers)
@ -831,7 +831,7 @@ func (q *queue) DeliverReceipts(id string, receiptList [][]*types.Receipt) (int,
// reason this lock is not obtained in here is because the parameters already need // reason this lock is not obtained in here is because the parameters already need
// to access the queue, so they already need a lock anyway. // to access the queue, so they already need a lock anyway.
func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header,
taskQueue *prque.Prque[int64, *types.Header], pendPool map[string]*fetchRequest, reqTimer metrics.Timer, taskQueue *prque.Prque, pendPool map[string]*fetchRequest, reqTimer metrics.Timer,
results int, validate func(index int, header *types.Header) error, results int, validate func(index int, header *types.Header) error,
reconstruct func(index int, result *fetchResult)) (int, error) { reconstruct func(index int, result *fetchResult)) (int, error) {
// Short circuit if the data was never requested // Short circuit if the data was never requested

View File

@ -177,9 +177,9 @@ type BlockFetcher struct {
completing map[common.Hash]*blockAnnounce // Blocks with headers, currently body-completing completing map[common.Hash]*blockAnnounce // Blocks with headers, currently body-completing
// Block cache // Block cache
queue *prque.Prque[int64, *blockOrHeaderInject] // Queue containing the import operations (block number sorted) queue *prque.Prque // Queue containing the import operations (block number sorted)
queues map[string]int // Per peer block counts to prevent memory exhaustion queues map[string]int // Per peer block counts to prevent memory exhaustion
queued map[common.Hash]*blockOrHeaderInject // Set of already queued blocks (to dedup imports) queued map[common.Hash]*blockOrHeaderInject // Set of already queued blocks (to dedup imports)
// Callbacks // Callbacks
getHeader HeaderRetrievalFn // Retrieves a header from the local chain getHeader HeaderRetrievalFn // Retrieves a header from the local chain
@ -214,7 +214,7 @@ func NewBlockFetcher(light bool, getHeader HeaderRetrievalFn, getBlock blockRetr
fetching: make(map[common.Hash]*blockAnnounce), fetching: make(map[common.Hash]*blockAnnounce),
fetched: make(map[common.Hash][]*blockAnnounce), fetched: make(map[common.Hash][]*blockAnnounce),
completing: make(map[common.Hash]*blockAnnounce), completing: make(map[common.Hash]*blockAnnounce),
queue: prque.New[int64, *blockOrHeaderInject](nil), queue: prque.New(nil),
queues: make(map[string]int), queues: make(map[string]int),
queued: make(map[common.Hash]*blockOrHeaderInject), queued: make(map[common.Hash]*blockOrHeaderInject),
getHeader: getHeader, getHeader: getHeader,
@ -353,7 +353,7 @@ func (f *BlockFetcher) loop() {
// Import any queued blocks that could potentially fit // Import any queued blocks that could potentially fit
height := f.chainHeight() height := f.chainHeight()
for !f.queue.Empty() { for !f.queue.Empty() {
op := f.queue.PopItem() op := f.queue.PopItem().(*blockOrHeaderInject)
hash := op.hash() hash := op.hash()
if f.queueChangeHook != nil { if f.queueChangeHook != nil {
f.queueChangeHook(hash, false) f.queueChangeHook(hash, false)

View File

@ -75,11 +75,10 @@ type ClientManager struct {
// (totalRecharge / sumRecharge)*FixedPointMultiplier or 0 if sumRecharge==0 // (totalRecharge / sumRecharge)*FixedPointMultiplier or 0 if sumRecharge==0
rcLastUpdate mclock.AbsTime // last time the recharge integrator was updated rcLastUpdate mclock.AbsTime // last time the recharge integrator was updated
rcLastIntValue int64 // last updated value of the recharge integrator rcLastIntValue int64 // last updated value of the recharge integrator
priorityOffset int64 // offset for prque priority values ensures that all priorities stay in the int64 range
// recharge queue is a priority queue with currently recharging client nodes // recharge queue is a priority queue with currently recharging client nodes
// as elements. The priority value is rcFullIntValue which allows to quickly // as elements. The priority value is rcFullIntValue which allows to quickly
// determine which client will first finish recharge. // determine which client will first finish recharge.
rcQueue *prque.Prque[int64, *ClientNode] rcQueue *prque.Prque
} }
// NewClientManager returns a new client manager. // NewClientManager returns a new client manager.
@ -108,7 +107,7 @@ type ClientManager struct {
func NewClientManager(curve PieceWiseLinear, clock mclock.Clock) *ClientManager { func NewClientManager(curve PieceWiseLinear, clock mclock.Clock) *ClientManager {
cm := &ClientManager{ cm := &ClientManager{
clock: clock, clock: clock,
rcQueue: prque.New[int64, *ClientNode](func(a *ClientNode, i int) { a.queueIndex = i }), rcQueue: prque.NewWrapAround(func(a interface{}, i int) { a.(*ClientNode).queueIndex = i }),
capLastUpdate: clock.Now(), capLastUpdate: clock.Now(),
stop: make(chan chan struct{}), stop: make(chan chan struct{}),
} }
@ -289,13 +288,13 @@ func (cm *ClientManager) updateRecharge(now mclock.AbsTime) {
} }
dt := now - lastUpdate dt := now - lastUpdate
// fetch the client that finishes first // fetch the client that finishes first
rcqNode := cm.rcQueue.PopItem() // if sumRecharge > 0 then the queue cannot be empty rcqNode := cm.rcQueue.PopItem().(*ClientNode) // if sumRecharge > 0 then the queue cannot be empty
// check whether it has already finished // check whether it has already finished
dtNext := mclock.AbsTime(float64(rcqNode.rcFullIntValue-cm.rcLastIntValue) / bonusRatio) dtNext := mclock.AbsTime(float64(rcqNode.rcFullIntValue-cm.rcLastIntValue) / bonusRatio)
if dt < dtNext { if dt < dtNext {
// not finished yet, put it back, update integrator according // not finished yet, put it back, update integrator according
// to current bonusRatio and return // to current bonusRatio and return
cm.addToQueue(rcqNode) cm.rcQueue.Push(rcqNode, -rcqNode.rcFullIntValue)
cm.rcLastIntValue += int64(bonusRatio * float64(dt)) cm.rcLastIntValue += int64(bonusRatio * float64(dt))
return return
} }
@ -309,20 +308,6 @@ func (cm *ClientManager) updateRecharge(now mclock.AbsTime) {
} }
} }
func (cm *ClientManager) addToQueue(node *ClientNode) {
if cm.priorityOffset-node.rcFullIntValue < -0x4000000000000000 {
cm.priorityOffset += 0x4000000000000000
// recreate priority queue with new offset to avoid overflow; should happen very rarely
newRcQueue := prque.New[int64, *ClientNode](func(a *ClientNode, i int) { a.queueIndex = i })
for cm.rcQueue.Size() > 0 {
n := cm.rcQueue.PopItem()
newRcQueue.Push(n, cm.priorityOffset-n.rcFullIntValue)
}
cm.rcQueue = newRcQueue
}
cm.rcQueue.Push(node, cm.priorityOffset-node.rcFullIntValue)
}
// updateNodeRc updates a node's corrBufValue and adds an external correction value. // updateNodeRc updates a node's corrBufValue and adds an external correction value.
// It also adds or removes the rcQueue entry and updates ServerParams and sumRecharge if necessary. // It also adds or removes the rcQueue entry and updates ServerParams and sumRecharge if necessary.
func (cm *ClientManager) updateNodeRc(node *ClientNode, bvc int64, params *ServerParams, now mclock.AbsTime) { func (cm *ClientManager) updateNodeRc(node *ClientNode, bvc int64, params *ServerParams, now mclock.AbsTime) {
@ -359,7 +344,7 @@ func (cm *ClientManager) updateNodeRc(node *ClientNode, bvc int64, params *Serve
} }
node.rcLastIntValue = cm.rcLastIntValue node.rcLastIntValue = cm.rcLastIntValue
node.rcFullIntValue = cm.rcLastIntValue + (int64(node.params.BufLimit)-node.corrBufValue)*FixedPointMultiplier/int64(node.params.MinRecharge) node.rcFullIntValue = cm.rcLastIntValue + (int64(node.params.BufLimit)-node.corrBufValue)*FixedPointMultiplier/int64(node.params.MinRecharge)
cm.addToQueue(node) cm.rcQueue.Push(node, -node.rcFullIntValue)
} }
} }

View File

@ -17,7 +17,6 @@
package flowcontrol package flowcontrol
import ( import (
"math"
"math/rand" "math/rand"
"testing" "testing"
"time" "time"
@ -45,17 +44,16 @@ const (
// maximum permitted rate. The max capacity nodes are changed multiple times during // maximum permitted rate. The max capacity nodes are changed multiple times during
// a single test. // a single test.
func TestConstantTotalCapacity(t *testing.T) { func TestConstantTotalCapacity(t *testing.T) {
testConstantTotalCapacity(t, 10, 1, 0, false) testConstantTotalCapacity(t, 10, 1, 0)
testConstantTotalCapacity(t, 10, 1, 1, false) testConstantTotalCapacity(t, 10, 1, 1)
testConstantTotalCapacity(t, 30, 1, 0, false) testConstantTotalCapacity(t, 30, 1, 0)
testConstantTotalCapacity(t, 30, 2, 3, false) testConstantTotalCapacity(t, 30, 2, 3)
testConstantTotalCapacity(t, 100, 1, 0, false) testConstantTotalCapacity(t, 100, 1, 0)
testConstantTotalCapacity(t, 100, 3, 5, false) testConstantTotalCapacity(t, 100, 3, 5)
testConstantTotalCapacity(t, 100, 5, 10, false) testConstantTotalCapacity(t, 100, 5, 10)
testConstantTotalCapacity(t, 100, 3, 5, true)
} }
func testConstantTotalCapacity(t *testing.T, nodeCount, maxCapacityNodes, randomSend int, priorityOverflow bool) { func testConstantTotalCapacity(t *testing.T, nodeCount, maxCapacityNodes, randomSend int) {
clock := &mclock.Simulated{} clock := &mclock.Simulated{}
nodes := make([]*testNode, nodeCount) nodes := make([]*testNode, nodeCount)
var totalCapacity uint64 var totalCapacity uint64
@ -64,10 +62,6 @@ func testConstantTotalCapacity(t *testing.T, nodeCount, maxCapacityNodes, random
totalCapacity += nodes[i].capacity totalCapacity += nodes[i].capacity
} }
m := NewClientManager(PieceWiseLinear{{0, totalCapacity}}, clock) m := NewClientManager(PieceWiseLinear{{0, totalCapacity}}, clock)
if priorityOverflow {
// provoke a situation where rcLastUpdate overflow needs to be handled
m.rcLastIntValue = math.MaxInt64 - 10000000000
}
for _, n := range nodes { for _, n := range nodes {
n.bufLimit = n.capacity * 6000 n.bufLimit = n.capacity * 6000
n.node = NewClientNode(m, ServerParams{BufLimit: n.bufLimit, MinRecharge: n.capacity}) n.node = NewClientNode(m, ServerParams{BufLimit: n.bufLimit, MinRecharge: n.capacity})

View File

@ -38,10 +38,10 @@ type servingQueue struct {
setThreadsCh chan int setThreadsCh chan int
wg sync.WaitGroup wg sync.WaitGroup
threadCount int // number of currently running threads threadCount int // number of currently running threads
queue *prque.Prque[int64, *servingTask] // priority queue for waiting or suspended tasks queue *prque.Prque // priority queue for waiting or suspended tasks
best *servingTask // the highest priority task (not included in the queue) best *servingTask // the highest priority task (not included in the queue)
suspendBias int64 // priority bias against suspending an already running task suspendBias int64 // priority bias against suspending an already running task
} }
// servingTask represents a request serving task. Tasks can be implemented to // servingTask represents a request serving task. Tasks can be implemented to
@ -123,7 +123,7 @@ func (t *servingTask) waitOrStop() bool {
// newServingQueue returns a new servingQueue // newServingQueue returns a new servingQueue
func newServingQueue(suspendBias int64, utilTarget float64) *servingQueue { func newServingQueue(suspendBias int64, utilTarget float64) *servingQueue {
sq := &servingQueue{ sq := &servingQueue{
queue: prque.New[int64, *servingTask](nil), queue: prque.NewWrapAround(nil),
suspendBias: suspendBias, suspendBias: suspendBias,
queueAddCh: make(chan *servingTask, 100), queueAddCh: make(chan *servingTask, 100),
queueBestCh: make(chan *servingTask), queueBestCh: make(chan *servingTask),
@ -214,7 +214,7 @@ func (sq *servingQueue) freezePeers() {
} }
sq.best = nil sq.best = nil
for sq.queue.Size() > 0 { for sq.queue.Size() > 0 {
task := sq.queue.PopItem() task := sq.queue.PopItem().(*servingTask)
tasks := peerMap[task.peer] tasks := peerMap[task.peer]
if tasks == nil { if tasks == nil {
bufValue, bufLimit := task.peer.fcClient.BufferStatus() bufValue, bufLimit := task.peer.fcClient.BufferStatus()
@ -251,7 +251,7 @@ func (sq *servingQueue) freezePeers() {
} }
} }
if sq.queue.Size() > 0 { if sq.queue.Size() > 0 {
sq.best = sq.queue.PopItem() sq.best = sq.queue.PopItem().(*servingTask)
} }
} }
@ -310,7 +310,7 @@ func (sq *servingQueue) queueLoop() {
if sq.queue.Size() == 0 { if sq.queue.Size() == 0 {
sq.best = nil sq.best = nil
} else { } else {
sq.best = sq.queue.PopItem() sq.best, _ = sq.queue.PopItem().(*servingTask)
} }
case <-sq.quit: case <-sq.quit:
return return

View File

@ -77,8 +77,8 @@ type priorityPool struct {
// temporary state if tempState is not empty // temporary state if tempState is not empty
tempState []*ppNodeInfo tempState []*ppNodeInfo
activeCount, activeCap uint64 activeCount, activeCap uint64
activeQueue *prque.LazyQueue[int64, *ppNodeInfo] activeQueue *prque.LazyQueue
inactiveQueue *prque.Prque[int64, *ppNodeInfo] inactiveQueue *prque.Prque
} }
// ppNodeInfo is the internal node descriptor of priorityPool // ppNodeInfo is the internal node descriptor of priorityPool
@ -104,7 +104,7 @@ func newPriorityPool(ns *nodestate.NodeStateMachine, setup *serverSetup, clock m
setup: setup, setup: setup,
ns: ns, ns: ns,
clock: clock, clock: clock,
inactiveQueue: prque.New[int64, *ppNodeInfo](inactiveSetIndex), inactiveQueue: prque.New(inactiveSetIndex),
minCap: minCap, minCap: minCap,
activeBias: activeBias, activeBias: activeBias,
capacityStepDiv: capacityStepDiv, capacityStepDiv: capacityStepDiv,
@ -183,7 +183,8 @@ func (pp *priorityPool) requestCapacity(node *enode.Node, minTarget, maxTarget u
} }
pp.setTempCapacity(c, maxTarget) pp.setTempCapacity(c, maxTarget)
c.minTarget = minTarget c.minTarget = minTarget
pp.removeFromQueues(c) pp.activeQueue.Remove(c.activeIndex)
pp.inactiveQueue.Remove(c.inactiveIndex)
pp.activeQueue.Push(c) pp.activeQueue.Push(c)
pp.enforceLimits() pp.enforceLimits()
updates := pp.finalizeChanges(c.tempCapacity >= minTarget && c.tempCapacity <= maxTarget && c.tempCapacity != c.capacity) updates := pp.finalizeChanges(c.tempCapacity >= minTarget && c.tempCapacity <= maxTarget && c.tempCapacity != c.capacity)
@ -249,13 +250,13 @@ func (pp *priorityPool) Limits() (uint64, uint64) {
} }
// inactiveSetIndex callback updates ppNodeInfo item index in inactiveQueue // inactiveSetIndex callback updates ppNodeInfo item index in inactiveQueue
func inactiveSetIndex(a *ppNodeInfo, index int) { func inactiveSetIndex(a interface{}, index int) {
a.inactiveIndex = index a.(*ppNodeInfo).inactiveIndex = index
} }
// activeSetIndex callback updates ppNodeInfo item index in activeQueue // activeSetIndex callback updates ppNodeInfo item index in activeQueue
func activeSetIndex(a *ppNodeInfo, index int) { func activeSetIndex(a interface{}, index int) {
a.activeIndex = index a.(*ppNodeInfo).activeIndex = index
} }
// invertPriority inverts a priority value. The active queue uses inverted priorities // invertPriority inverts a priority value. The active queue uses inverted priorities
@ -268,7 +269,8 @@ func invertPriority(p int64) int64 {
} }
// activePriority callback returns actual priority of ppNodeInfo item in activeQueue // activePriority callback returns actual priority of ppNodeInfo item in activeQueue
func activePriority(c *ppNodeInfo) int64 { func activePriority(a interface{}) int64 {
c := a.(*ppNodeInfo)
if c.bias == 0 { if c.bias == 0 {
return invertPriority(c.nodePriority.priority(c.tempCapacity)) return invertPriority(c.nodePriority.priority(c.tempCapacity))
} else { } else {
@ -277,7 +279,8 @@ func activePriority(c *ppNodeInfo) int64 {
} }
// activeMaxPriority callback returns estimated maximum priority of ppNodeInfo item in activeQueue // activeMaxPriority callback returns estimated maximum priority of ppNodeInfo item in activeQueue
func (pp *priorityPool) activeMaxPriority(c *ppNodeInfo, until mclock.AbsTime) int64 { func (pp *priorityPool) activeMaxPriority(a interface{}, until mclock.AbsTime) int64 {
c := a.(*ppNodeInfo)
future := time.Duration(until - pp.clock.Now()) future := time.Duration(until - pp.clock.Now())
if future < 0 { if future < 0 {
future = 0 future = 0
@ -290,16 +293,6 @@ func (pp *priorityPool) inactivePriority(p *ppNodeInfo) int64 {
return p.nodePriority.priority(pp.minCap) return p.nodePriority.priority(pp.minCap)
} }
// removeFromQueues removes the node from the active/inactive queues
func (pp *priorityPool) removeFromQueues(c *ppNodeInfo) {
if c.activeIndex >= 0 {
pp.activeQueue.Remove(c.activeIndex)
}
if c.inactiveIndex >= 0 {
pp.inactiveQueue.Remove(c.inactiveIndex)
}
}
// connectNode is called when a new node has been added to the pool (inactiveFlag set) // connectNode is called when a new node has been added to the pool (inactiveFlag set)
// Note: this function should run inside a NodeStateMachine operation // Note: this function should run inside a NodeStateMachine operation
func (pp *priorityPool) connectNode(c *ppNodeInfo) { func (pp *priorityPool) connectNode(c *ppNodeInfo) {
@ -327,7 +320,8 @@ func (pp *priorityPool) disconnectNode(c *ppNodeInfo) {
return return
} }
c.connected = false c.connected = false
pp.removeFromQueues(c) pp.activeQueue.Remove(c.activeIndex)
pp.inactiveQueue.Remove(c.inactiveIndex)
var updates []capUpdate var updates []capUpdate
if c.capacity != 0 { if c.capacity != 0 {
@ -417,11 +411,11 @@ func (pp *priorityPool) enforceLimits() (*ppNodeInfo, int64) {
return nil, math.MinInt64 return nil, math.MinInt64
} }
var ( var (
lastNode *ppNodeInfo c *ppNodeInfo
maxActivePriority int64 maxActivePriority int64
) )
pp.activeQueue.MultiPop(func(c *ppNodeInfo, priority int64) bool { pp.activeQueue.MultiPop(func(data interface{}, priority int64) bool {
lastNode = c c = data.(*ppNodeInfo)
pp.setTempState(c) pp.setTempState(c)
maxActivePriority = priority maxActivePriority = priority
if c.tempCapacity == c.minTarget || pp.activeCount > pp.maxCount { if c.tempCapacity == c.minTarget || pp.activeCount > pp.maxCount {
@ -439,7 +433,7 @@ func (pp *priorityPool) enforceLimits() (*ppNodeInfo, int64) {
} }
return pp.activeCap > pp.maxCap || pp.activeCount > pp.maxCount return pp.activeCap > pp.maxCap || pp.activeCount > pp.maxCount
}) })
return lastNode, invertPriority(maxActivePriority) return c, invertPriority(maxActivePriority)
} }
// finalizeChanges either commits or reverts temporary changes. The necessary capacity // finalizeChanges either commits or reverts temporary changes. The necessary capacity
@ -448,7 +442,8 @@ func (pp *priorityPool) enforceLimits() (*ppNodeInfo, int64) {
func (pp *priorityPool) finalizeChanges(commit bool) (updates []capUpdate) { func (pp *priorityPool) finalizeChanges(commit bool) (updates []capUpdate) {
for _, c := range pp.tempState { for _, c := range pp.tempState {
// always remove and push back in order to update biased priority // always remove and push back in order to update biased priority
pp.removeFromQueues(c) pp.activeQueue.Remove(c.activeIndex)
pp.inactiveQueue.Remove(c.inactiveIndex)
oldCapacity := c.capacity oldCapacity := c.capacity
if commit { if commit {
c.capacity = c.tempCapacity c.capacity = c.tempCapacity
@ -501,7 +496,7 @@ func (pp *priorityPool) updateFlags(updates []capUpdate) {
// tryActivate tries to activate inactive nodes if possible // tryActivate tries to activate inactive nodes if possible
func (pp *priorityPool) tryActivate(commit bool) []capUpdate { func (pp *priorityPool) tryActivate(commit bool) []capUpdate {
for pp.inactiveQueue.Size() > 0 { for pp.inactiveQueue.Size() > 0 {
c := pp.inactiveQueue.PopItem() c := pp.inactiveQueue.PopItem().(*ppNodeInfo)
pp.setTempState(c) pp.setTempState(c)
pp.setTempBias(c, pp.activeBias) pp.setTempBias(c, pp.activeBias)
pp.setTempCapacity(c, pp.minCap) pp.setTempCapacity(c, pp.minCap)
@ -529,7 +524,8 @@ func (pp *priorityPool) updatePriority(node *enode.Node) {
pp.lock.Unlock() pp.lock.Unlock()
return return
} }
pp.removeFromQueues(c) pp.activeQueue.Remove(c.activeIndex)
pp.inactiveQueue.Remove(c.inactiveIndex)
if c.capacity != 0 { if c.capacity != 0 {
pp.activeQueue.Push(c) pp.activeQueue.Push(c)
} else { } else {

View File

@ -160,7 +160,7 @@ type Sync struct {
membatch *syncMemBatch // Memory buffer to avoid frequent database writes membatch *syncMemBatch // Memory buffer to avoid frequent database writes
nodeReqs map[string]*nodeRequest // Pending requests pertaining to a trie node path nodeReqs map[string]*nodeRequest // Pending requests pertaining to a trie node path
codeReqs map[common.Hash]*codeRequest // Pending requests pertaining to a code hash codeReqs map[common.Hash]*codeRequest // Pending requests pertaining to a code hash
queue *prque.Prque[int64, any] // Priority queue with the pending requests queue *prque.Prque // Priority queue with the pending requests
fetches map[int]int // Number of active fetches per trie node depth fetches map[int]int // Number of active fetches per trie node depth
} }
@ -172,7 +172,7 @@ func NewSync(root common.Hash, database ethdb.KeyValueReader, callback LeafCallb
membatch: newSyncMemBatch(), membatch: newSyncMemBatch(),
nodeReqs: make(map[string]*nodeRequest), nodeReqs: make(map[string]*nodeRequest),
codeReqs: make(map[common.Hash]*codeRequest), codeReqs: make(map[common.Hash]*codeRequest),
queue: prque.New[int64, any](nil), // Ugh, can contain both string and hash, whyyy queue: prque.New(nil),
fetches: make(map[int]int), fetches: make(map[int]int),
} }
ts.AddSubTrie(root, nil, common.Hash{}, nil, callback) ts.AddSubTrie(root, nil, common.Hash{}, nil, callback)