README update.

README.md

@@ -33,11 +33,9 @@ known nodes, but usually only several have timestamps within the last hour.
 
 (In approximate order of priority)
 
-* Apply peer-age filter to results
-* Output using some space-conscious format. Right now the output file grows
-  fairly quickly.
 * Namespace useful sub-packages properly (outside of `main`)
-* Fix race condition warnings.
+* Fix race condition warnings. (Not sure if this is feasible. `golog` and other
+  fundamental pieces seem to trigger warnings possibly erroneously.)
 
 
 ## License

cmd.go

@@ -40,6 +40,34 @@ var logLevels = []log.Level{
 	log.Debug,
 }
 
+type ResultJSON struct {
+	Address   string   `json:"address"`
+	UserAgent string   `json:"useragent"`
+	Peers     []string `json:"peers"`
+}
+
+func NewResultJSON(result *Result, peerAge time.Duration) *ResultJSON {
+	// Convert the full result object into the subset which we want to return
+	// as JSON.
+	timestampSince := time.Now().Add(-peerAge)
+	peers := []string{}
+	for _, addr := range result.Peers {
+		if !addr.Timestamp.After(timestampSince) {
+			continue
+		}
+
+		peers = append(peers, NetAddressKey(addr))
+	}
+
+	r := ResultJSON{
+		Address:   result.Node.Address,
+		UserAgent: result.Node.UserAgent,
+		Peers:     peers,
+	}
+
+	return &r
+}
+
 func main() {
 	now := time.Now()
 	options := Options{}
@@ -113,7 +141,8 @@ func main() {
 	// Start processing results
 	count := 0
 	for result := range resultChan {
-		b, err := json.Marshal(result)
+		jsonResult := NewResultJSON(&result, options.PeerAge)
+		b, err := json.Marshal(jsonResult)
 		if err != nil {
 			logger.Warningf("Failed to export JSON, skipping: %v", err)
 		}

crawler.go

@@ -148,9 +148,11 @@ func (c *Crawler) filter(address string) *string {
 
 func (c *Crawler) process(r *Result) *Result {
 	timestampSince := time.Now().Add(-c.PeerAge)
+	numStalePeers := 0
 
 	for _, addr := range r.Peers {
 		if !addr.Timestamp.After(timestampSince) {
+			numStalePeers++
 			continue
 		}
 
@@ -158,7 +160,7 @@ func (c *Crawler) process(r *Result) *Result {
 	}
 
 	if len(r.Peers) > 0 {
-		logger.Infof("[%s] Returned %d peers. Total %d unique peers via %d connected (of %d attempted).", r.Node.Address, len(r.Peers), c.numUnique, c.numConnected, c.numAttempted)
+		logger.Infof("[%s] Returned %d peers (including %d stale). Total %d unique peers via %d connected (of %d attempted).", r.Node.Address, len(r.Peers), numStalePeers, c.numUnique, c.numConnected, c.numAttempted)
 		return r
 	}
 

seed.go

@@ -1,16 +1,17 @@
 package main
 
 import (
-	"github.com/conformal/btcwire"
 	"net"
 	"sync"
+
+	"github.com/conformal/btcwire"
 )
 
 func GetSeedsFromDNS(dnsSeeds []string) []string {
 	wait := sync.WaitGroup{}
 	results := make(chan []net.IP)
 
-	for _, address := range dnsSeeds {
+	for _, seed := range dnsSeeds {
 		wait.Add(1)
 		go func(address string) {
 			defer wait.Done()
@@ -21,7 +22,7 @@ func GetSeedsFromDNS(dnsSeeds []string) []string {
 			}
 			logger.Debugf("Resolved %d seeds from %s.", len(ips), address)
 			results <- ips
-		}(address)
+		}(seed)
 	}
 
 	go func() {