Fixing bugs, but broken.

This commit is contained in:
Andrey Petrov 2014-05-15 17:59:01 -07:00
parent 5c128f64bb
commit 8ec2703f3c
3 changed files with 19 additions and 10 deletions

View File

@ -1,5 +1,5 @@
btc-crawl: **.go
go build .
go build ./...
build: btc-crawl

View File

@ -170,13 +170,14 @@ func (c *Crawler) Run(resultChan chan<- Result, numWorkers int) {
// Don't start any new workers, leave the slot filled.
break
} else if c.queue.IsEmpty() {
<-workerChan
if numActive == 0 {
logger.Infof("Done.")
logger.Infof("Done after %d queued items.", c.queue.Count())
close(resultChan)
return
}
<-workerChan
break
}
numActive++
@ -194,7 +195,7 @@ func (c *Crawler) Run(resultChan chan<- Result, numWorkers int) {
<-workerChan
case <-c.shutdown:
logger.Infof("Shutting down.")
logger.Infof("Shutting down after %d workers finish.", numActive)
isActive = false
}
}

View File

@ -7,6 +7,7 @@ type Queue struct {
Output chan string
overflow []string
filter func(string) *string
count int
}
func NewQueue(filter func(string) *string, bufferSize int) *Queue {
@ -23,12 +24,13 @@ func NewQueue(filter func(string) *string, bufferSize int) *Queue {
for {
select {
case input := <-q.Input:
case item := <-input:
// New input
r := q.filter(input)
r := q.filter(item)
if r != nil {
// Store in the overflow
q.overflow = append(q.overflow, *r)
q.count++
}
case output <- nextItem:
// Block until we have more inputs
@ -55,6 +57,7 @@ func (q *Queue) next() string {
r := q.filter(<-q.Input)
if r != nil {
q.count++
return *r
}
}
@ -64,7 +67,7 @@ func (q *Queue) IsEmpty() bool {
// FIXME: This effectively cycles the order of the output buffer. Kinda sad.
if len(q.overflow) > 0 {
return true
return false
}
select {
@ -73,9 +76,9 @@ func (q *Queue) IsEmpty() bool {
// Put it back
q.Output <- r
}()
return true
default:
return false
default:
return true
}
}
@ -87,3 +90,8 @@ func (q *Queue) Wait() {
q.Output <- r
}()
}
func (q *Queue) Count() int {
// Number of outputs produced.
return q.count
}