Importing code.

This commit is contained in:
Andrey Petrov 2014-04-18 21:26:47 -07:00
parent d7e87f46ea
commit 3a6b6ec8fb
4 changed files with 343 additions and 0 deletions

3
.gitignore vendored
View File

@ -21,3 +21,6 @@ _testmain.go
*.exe
*.test
# Binary
/btc-crawl

View File

@ -2,3 +2,8 @@ btc-crawl
=========
Bitcoin node network crawler (written in golang).
This is a for-fun project to explore the Bitcoin protocol and network.
Current status: It crawls from hard-coded values and spits a bunch of stuff to
stdout.

287
btc-crawl.go Normal file
View File

@ -0,0 +1,287 @@
package main
import (
"fmt"
"github.com/conformal/btcwire"
"log"
"net"
"time"
)
// TODO: Unhardcode these:
var seedNodes []string = []string{"85.214.251.25:8333", "62.75.216.13:8333"}
var userAgent string = "/btc-crawl:0.0.1"
var lastBlock int32 = 0
// TODO: Break Client/Peer/Crawler into separate modules.
type Client struct {
btcnet btcwire.BitcoinNet // Bitcoin Network
pver uint32 // Protocl Version
userAgent string // User Agent
lastBlock int32
}
func NewDefaultClient() *Client {
return &Client{
btcnet: btcwire.MainNet,
pver: btcwire.ProtocolVersion,
userAgent: userAgent,
lastBlock: lastBlock,
}
}
type Peer struct {
client *Client
address string
conn net.Conn
nonce uint64 // Nonce we're sending to the peer
}
func NewPeer(client *Client, address string) *Peer {
p := Peer{
client: client,
address: address,
}
return &p
}
func (p *Peer) Connect() error {
if p.conn != nil {
return fmt.Errorf("Peer already connected, can't connect again.")
}
conn, err := net.Dial("tcp", p.address)
if err != nil {
return err
}
p.conn = conn
return nil
}
func (p *Peer) Disconnect() {
p.conn.Close()
}
func (p *Peer) Handshake() error {
if p.conn == nil {
return fmt.Errorf("Peer is not connected, can't handshake.")
}
log.Printf("[%s] Starting handshake.", p.address)
nonce, err := btcwire.RandomUint64()
if err != nil {
return err
}
p.nonce = nonce
pver, btcnet := p.client.pver, p.client.btcnet
msgVersion, err := btcwire.NewMsgVersionFromConn(p.conn, p.nonce, p.client.userAgent, 0)
msgVersion.DisableRelayTx = true
if err := btcwire.WriteMessage(p.conn, msgVersion, pver, btcnet); err != nil {
return err
}
// Read the response version.
msg, _, err := btcwire.ReadMessage(p.conn, pver, btcnet)
if err != nil {
return err
}
vmsg, ok := msg.(*btcwire.MsgVersion)
if !ok {
return fmt.Errorf("Did not receive version message: %T", vmsg)
}
// Negotiate protocol version.
if uint32(vmsg.ProtocolVersion) < pver {
pver = uint32(vmsg.ProtocolVersion)
}
log.Printf("[%s] -> Version: %s", p.address, vmsg.UserAgent)
// Normally we'd check if vmsg.Nonce == p.nonce but the crawler does not
// accept external connections so we skip it.
// Send verack.
if err := btcwire.WriteMessage(p.conn, btcwire.NewMsgVerAck(), pver, btcnet); err != nil {
return err
}
return nil
}
type Crawler struct {
client *Client
count int
seenFilter map[string]bool // TODO: Replace with bloom filter?
results chan []string
workers chan struct{}
queue []string
activeSince time.Duration
}
func NewCrawler(client *Client, queue []string, numWorkers int) *Crawler {
c := Crawler{
client: client,
count: 0,
seenFilter: map[string]bool{},
results: make(chan []string),
workers: make(chan struct{}, numWorkers),
queue: []string{},
activeSince: time.Hour * -24,
}
// Prefill the queue
for _, address := range queue {
c.addAddress(address)
}
return &c
}
func (c *Crawler) handleAddress(address string) *[]string {
r := []string{}
client := c.client
peer := NewPeer(client, address)
err := peer.Connect()
if err != nil {
log.Printf("[%s] Connection failed: %v", address, err)
return &r
}
defer peer.Disconnect()
err = peer.Handshake()
if err != nil {
log.Printf("[%s] Handsake failed: %v", address, err)
return &r
}
// Send getaddr.
if err := btcwire.WriteMessage(peer.conn, btcwire.NewMsgGetAddr(), client.pver, client.btcnet); err != nil {
log.Printf("[%s] GetAddr failed: %v", address, err)
return &r
}
// Listen for tx inv messages.
firstReceived := -1
tolerateMessages := 3
otherMessages := []string{}
timestampSince := time.Now().Add(c.activeSince)
for {
// We can't really tell when we're done receiving peers, so we stop either
// when we get a smaller-than-normal set size or when we've received too
// many unrelated messages.
msg, _, err := btcwire.ReadMessage(peer.conn, client.pver, client.btcnet)
if err != nil {
log.Printf("[%s] Failed to read message: %v", address, err)
continue
}
switch tmsg := msg.(type) {
case *btcwire.MsgAddr:
for _, addr := range tmsg.AddrList {
if addr.Timestamp.After(timestampSince) {
r = append(r, NetAddressKey(addr))
}
}
if firstReceived == -1 {
firstReceived = len(tmsg.AddrList)
} else if firstReceived > len(tmsg.AddrList) || firstReceived == 0 {
// Probably done.
return &r
}
default:
otherMessages = append(otherMessages, tmsg.Command())
if len(otherMessages) > tolerateMessages {
log.Printf("[%s] Giving up with %d results after tolerating messages: %v.", address, len(r), otherMessages)
return &r
}
}
}
}
func (c *Crawler) addAddress(address string) bool {
// Returns true if not seen before, otherwise false
state, ok := c.seenFilter[address]
if ok == true && state == true {
return false
}
c.seenFilter[address] = true
c.count += 1
c.queue = append(c.queue, address)
return true
}
func (c *Crawler) Start() (chan struct{}, error) {
done := make(chan struct{}, 1)
numWorkers := 0
numGood := 0
// This is the main "event loop". Feels like there may be a better way to
// manage the number of concurrent workers but I can't think of it right now.
for {
select {
case c.workers <- struct{}{}:
if len(c.queue) == 0 {
// No work yet.
<-c.workers
continue
}
// Pop from the queue
address := c.queue[0]
c.queue = c.queue[1:]
numWorkers += 1
go func() {
log.Printf("[%s] Worker started.", address)
results := *c.handleAddress(address)
c.results <- results
}()
case r := <-c.results:
newAdded := 0
for _, address := range r {
if c.addAddress(address) {
newAdded += 1
}
}
if newAdded > 0 {
numGood += 1
}
numWorkers -= 1
log.Printf("Added %d new peers of %d returned. Total %d known peers via %d connected.", newAdded, len(r), c.count, numGood)
if len(c.queue) == 0 && numWorkers == 0 {
log.Printf("Done.")
done <- struct{}{}
return done, nil
}
<-c.workers
}
}
}
func main() {
// TODO: Parse args.
// TODO: Export to a reasonable format.
// TODO: Use proper logger for logging.
client := NewDefaultClient()
crawler := NewCrawler(client, seedNodes, 10)
done, err := crawler.Start()
if err != nil {
log.Fatal(err)
}
<-done
}

48
ext_btcd.go Normal file
View File

@ -0,0 +1,48 @@
// Things borrowed from https://github.com/conformal/btcd/blob/master/addrmanager.go
// because "github.com/conformal/btcd" wouldn't import for some reason.
package main
import (
"encoding/base32"
"github.com/conformal/btcwire"
"net"
"strconv"
"strings"
)
var onioncatrange = net.IPNet{IP: net.ParseIP("FD87:d87e:eb43::"),
Mask: net.CIDRMask(48, 128)}
func Tor(na *btcwire.NetAddress) bool {
// bitcoind encodes a .onion address as a 16 byte number by decoding the
// address prior to the .onion (i.e. the key hash) base32 into a ten
// byte number. it then stores the first 6 bytes of the address as
// 0xfD, 0x87, 0xD8, 0x7e, 0xeb, 0x43
// this is the same range used by onioncat, part of the
// RFC4193 Unique local IPv6 range.
// In summary the format is:
// { magic 6 bytes, 10 bytes base32 decode of key hash }
return onioncatrange.Contains(na.IP)
}
// ipString returns a string for the ip from the provided NetAddress. If the
// ip is in the range used for tor addresses then it will be transformed into
// the relavent .onion address.
func ipString(na *btcwire.NetAddress) string {
if Tor(na) {
// We know now that na.IP is long enogh.
base32 := base32.StdEncoding.EncodeToString(na.IP[6:])
return strings.ToLower(base32) + ".onion"
} else {
return na.IP.String()
}
}
// NetAddressKey returns a string key in the form of ip:port for IPv4 addresses
// or [ip]:port for IPv6 addresses.
func NetAddressKey(na *btcwire.NetAddress) string {
port := strconv.FormatUint(uint64(na.Port), 10)
addr := net.JoinHostPort(ipString(na), port)
return addr
}