Fixing bugs, but broken.
This commit is contained in:
parent
5c128f64bb
commit
8ec2703f3c
|
@ -170,13 +170,14 @@ func (c *Crawler) Run(resultChan chan<- Result, numWorkers int) {
|
|||
// Don't start any new workers, leave the slot filled.
|
||||
break
|
||||
} else if c.queue.IsEmpty() {
|
||||
<-workerChan
|
||||
|
||||
if numActive == 0 {
|
||||
logger.Infof("Done.")
|
||||
logger.Infof("Done after %d queued items.", c.queue.Count())
|
||||
close(resultChan)
|
||||
return
|
||||
}
|
||||
|
||||
<-workerChan
|
||||
break
|
||||
}
|
||||
|
||||
numActive++
|
||||
|
@ -194,7 +195,7 @@ func (c *Crawler) Run(resultChan chan<- Result, numWorkers int) {
|
|||
<-workerChan
|
||||
|
||||
case <-c.shutdown:
|
||||
logger.Infof("Shutting down.")
|
||||
logger.Infof("Shutting down after %d workers finish.", numActive)
|
||||
isActive = false
|
||||
}
|
||||
}
|
||||
|
|
|
@ -7,6 +7,7 @@ type Queue struct {
|
|||
Output chan string
|
||||
overflow []string
|
||||
filter func(string) *string
|
||||
count int
|
||||
}
|
||||
|
||||
func NewQueue(filter func(string) *string, bufferSize int) *Queue {
|
||||
|
@ -23,12 +24,13 @@ func NewQueue(filter func(string) *string, bufferSize int) *Queue {
|
|||
|
||||
for {
|
||||
select {
|
||||
case input := <-q.Input:
|
||||
case item := <-input:
|
||||
// New input
|
||||
r := q.filter(input)
|
||||
r := q.filter(item)
|
||||
if r != nil {
|
||||
// Store in the overflow
|
||||
q.overflow = append(q.overflow, *r)
|
||||
q.count++
|
||||
}
|
||||
case output <- nextItem:
|
||||
// Block until we have more inputs
|
||||
|
@ -55,6 +57,7 @@ func (q *Queue) next() string {
|
|||
r := q.filter(<-q.Input)
|
||||
|
||||
if r != nil {
|
||||
q.count++
|
||||
return *r
|
||||
}
|
||||
}
|
||||
|
@ -64,7 +67,7 @@ func (q *Queue) IsEmpty() bool {
|
|||
// FIXME: This effectively cycles the order of the output buffer. Kinda sad.
|
||||
|
||||
if len(q.overflow) > 0 {
|
||||
return true
|
||||
return false
|
||||
}
|
||||
|
||||
select {
|
||||
|
@ -73,9 +76,9 @@ func (q *Queue) IsEmpty() bool {
|
|||
// Put it back
|
||||
q.Output <- r
|
||||
}()
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -87,3 +90,8 @@ func (q *Queue) Wait() {
|
|||
q.Output <- r
|
||||
}()
|
||||
}
|
||||
|
||||
func (q *Queue) Count() int {
|
||||
// Number of outputs produced.
|
||||
return q.count
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue