diff --git a/btc-crawl.go b/btc-crawl.go index d2aea25..206d03a 100644 --- a/btc-crawl.go +++ b/btc-crawl.go @@ -4,6 +4,7 @@ package main import ( "github.com/jessevdk/go-flags" + "time" ) // Taken from: https://github.com/bitcoin/bitcoin/blob/89d72f3d9b6e7ef051ad1439f266b809f348229b/src/chainparams.cpp#L143 @@ -17,10 +18,11 @@ var defaultDnsSeeds = []string{ } type Options struct { - Verbose []bool `short:"v" long:"verbose" description:"Show verbose logging."` - Seed []string `short:"s" long:"seed" description:"Override which seeds to use." default-mask:""` - Concurrency int `short:"c" long:"concurrency" description:"Maximum number of concurrent connections to open." default:"10"` - UserAgent string `short:"A" long:"user-agent" description:"Client name to advertise while crawling. Should be in format of '/name:x.y.z/'." default:"/btc-crawl:0.1.1/"` + Verbose []bool `short:"v" long:"verbose" description:"Show verbose logging."` + Seed []string `short:"s" long:"seed" description:"Override which seeds to use." default-mask:""` + Concurrency int `short:"c" long:"concurrency" description:"Maximum number of concurrent connections to open." default:"10"` + UserAgent string `short:"A" long:"user-agent" description:"Client name to advertise while crawling. Should be in format of '/name:x.y.z/'." default:"/btc-crawl:0.1.1/"` + PeerAge time.Duration `long:"peer-age" description:"Ignore discovered peers older than this." default:"24h"` } func main() { @@ -40,6 +42,6 @@ func main() { } client := NewClient(options.UserAgent) - crawler := NewCrawler(client, seedNodes, options.Concurrency) + crawler := NewCrawler(client, seedNodes, options.Concurrency, options.PeerAge) crawler.Start() } diff --git a/crawler.go b/crawler.go index a099a58..7124091 100644 --- a/crawler.go +++ b/crawler.go @@ -8,24 +8,24 @@ import ( // TODO: Break Client/Peer/Crawler into separate modules. type Crawler struct { - client *Client - count int - seenFilter map[string]bool // TODO: Replace with bloom filter? - results chan []string - workers chan struct{} - queue []string - activeSince time.Duration + client *Client + count int + seenFilter map[string]bool // TODO: Replace with bloom filter? + results chan []string + workers chan struct{} + queue []string + peerAge time.Duration } -func NewCrawler(client *Client, queue []string, numWorkers int) *Crawler { +func NewCrawler(client *Client, queue []string, numWorkers int, peerAge time.Duration) *Crawler { c := Crawler{ - client: client, - count: 0, - seenFilter: map[string]bool{}, - results: make(chan []string), - workers: make(chan struct{}, numWorkers), - queue: []string{}, - activeSince: time.Hour * -24, + client: client, + count: 0, + seenFilter: map[string]bool{}, + results: make(chan []string), + workers: make(chan struct{}, numWorkers), + queue: []string{}, + peerAge: peerAge, } // Prefill the queue @@ -65,7 +65,7 @@ func (c *Crawler) handleAddress(address string) *[]string { firstReceived := -1 tolerateMessages := 3 otherMessages := []string{} - timestampSince := time.Now().Add(c.activeSince) + timestampSince := time.Now().Add(-c.peerAge) for { // We can't really tell when we're done receiving peers, so we stop either @@ -81,6 +81,7 @@ func (c *Crawler) handleAddress(address string) *[]string { case *btcwire.MsgAddr: for _, addr := range tmsg.AddrList { if addr.Timestamp.After(timestampSince) { + // TODO: Move this check to .Start()? r = append(r, NetAddressKey(addr)) } } @@ -154,7 +155,9 @@ func (c *Crawler) Start() { } numWorkers -= 1 - log.Printf("Added %d new peers of %d returned. Total %d known peers via %d connected.", newAdded, len(r), c.count, numGood) + if len(r) > 0 { + log.Printf("Added %d new peers of %d returned. Total %d known peers via %d connected.", newAdded, len(r), c.count, numGood) + } if len(c.queue) == 0 && numWorkers == 0 { log.Printf("Done.") diff --git a/peer.go b/peer.go index 0ceb6d0..a8e08b3 100644 --- a/peer.go +++ b/peer.go @@ -37,6 +37,7 @@ func (p *Peer) Connect() error { func (p *Peer) Disconnect() { p.conn.Close() + log.Printf("[%s] Closed.", p.address) } func (p *Peer) Handshake() error {