diff --git a/blocks.py b/blocks.py new file mode 100644 index 0000000000..c9e94c2098 --- /dev/null +++ b/blocks.py @@ -0,0 +1,100 @@ +from pybitcointools import * +import rlp +import re +from transactions import Transaction +from trie import Trie +import sys + +class Block(): + def __init__(self,data=None): + + if not data: + return + + if re.match('^[0-9a-fA-F]*$',data): + data = data.decode('hex') + + header, transaction_list, self.uncles = rlp.decode(data) + [ self.number, + self.prevhash, + self.uncles_root, + self.coinbase, + state_root, + self.transactions_root, + self.difficulty, + self.timestamp, + self.nonce, + self.extra ] = header + self.transactions = [Transaction(x) for x in transaction_list] + self.state = Trie('statedb',state_root) + self.reward = 0 + + # Verifications + if self.state.root != '' and self.state.db.get(self.state.root) == '': + raise Exception("State Merkle root not found in database!") + if bin_sha256(rlp.encode(transaction_list)) != self.transactions_root: + raise Exception("Transaction list root hash does not match!") + if bin_sha256(rlp.encode(self.uncles)) != self.uncles_root: + raise Exception("Uncle root hash does not match!") + # TODO: check POW + + def pay_fee(self,address,fee,tominer=True): + # Subtract fee from sender + sender_state = rlp.decode(self.state.get(address)) + if not sender_state or sender_state[1] < fee: + return False + sender_state[1] -= fee + self.state.update(address,sender_state) + # Pay fee to miner + if tominer: + miner_state = rlp.decode(self.state.get(self.coinbase)) or [0,0,0] + miner_state[1] += fee + self.state.update(self.coinbase,miner_state) + return True + + def get_nonce(self,address): + state = rlp.decode(self.state.get(address)) + if not state or state[0] == 0: return False + return state[2] + + def get_balance(self,address): + state = rlp.decode(self.state.get(address)) + return state[1] if state else 0 + + def set_balance(self,address,balance): + state = rlp.decode(self.state.get(address)) or [0,0,0] + state[1] = balance + self.state.update(address,rlp.encode(state)) + + + # Making updates to the object obtained from this method will do nothing. You need + # to call update_contract to finalize the changes. + def get_contract(self,address): + state = rlp.decode(self.state.get(address)) + if not state or state[0] == 0: return False + return Trie('statedb',state[2]) + + def update_contract(self,address,contract): + state = rlp.decode(self.state.get(address)) or [1,0,''] + if state[0] == 0: return False + state[2] = contract.root + self.state.update(address,state) + + # Serialization method; should act as perfect inverse function of the constructor + # assuming no verification failures + def serialize(self): + txlist = [x.serialize() for x in self.transactions] + header = [ self.number, + self.prevhash, + bin_sha256(rlp.encode(self.uncles)), + self.coinbase, + self.state.root, + bin_sha256(rlp.encode(txlist)), + self.difficulty, + self.timestamp, + self.nonce, + self.extra ] + return rlp.encode([header, txlist, self.uncles ]) + + def hash(self): + return bin_sha256(self.serialize()) diff --git a/manager.py b/manager.py new file mode 100644 index 0000000000..dbe0f37cdd --- /dev/null +++ b/manager.py @@ -0,0 +1,89 @@ +import rlp +import leveldb +from blocks import Block +from transactions import Transaction +import processblock +import hashlib +from pybitcointools import * + +txpool = {} + +genesis_header = [ + 0, + '', + bin_sha256(rlp.encode([])), + '', + '', + bin_sha256(rlp.encode([])), + 2**36, + 0, + 0, + '' +] + +genesis = [ genesis_header, [], [] ] + +mainblk = Block(rlp.encode(genesis)) + +db = leveldb.LevelDB("objects") + +def genaddr(seed): + priv = bin_sha256(seed) + addr = bin_sha256(privtopub(priv)[1:])[-20:] + return priv,addr + +# For testing +k1,a1 = genaddr("123") +k2,a2 = genaddr("456") + +def broadcast(obj): + pass + +def receive(obj): + d = rlp.decode(obj) + # Is transaction + if len(d) == 8: + tx = Transaction(obj) + if mainblk.get_balance(tx.sender) < tx.value + tx.fee: return + if mainblk.get_nonce(tx.sender) != tx.nonce: return + txpool[bin_sha256(blk)] = blk + broadcast(blk) + # Is message + elif len(d) == 2: + if d[0] == 'getobj': + try: return db.Get(d[1][0]) + except: + try: return mainblk.state.db.get(d[1][0]) + except: return None + elif d[0] == 'getbalance': + try: return mainblk.state.get_balance(d[1][0]) + except: return None + elif d[0] == 'getcontractroot': + try: return mainblk.state.get_contract(d[1][0]).root + except: return None + elif d[0] == 'getcontractsize': + try: return mainblk.state.get_contract(d[1][0]).get_size() + except: return None + elif d[0] == 'getcontractstate': + try: return mainblk.state.get_contract(d[1][0]).get(d[1][1]) + except: return None + # Is block + elif len(d) == 3: + blk = Block(obj) + p = block.prevhash + try: + parent = Block(db.Get(p)) + except: + return + uncles = block.uncles + for s in uncles: + try: + sib = db.Get(s) + except: + return + processblock.eval(parent,blk.transactions,blk.timestamp,blk.coinbase) + if parent.state.root != blk.state.root: return + if parent.difficulty != blk.difficulty: return + if parent.number != blk.number: return + db.Put(blk.hash(),blk.serialize()) + diff --git a/mining.py b/mining.py new file mode 100644 index 0000000000..be651881d9 --- /dev/null +++ b/mining.py @@ -0,0 +1,56 @@ +import hashlib + +def bin_sha256(x): return hashlib.sha256(x).digest() + +def spread(L): return 16 if L == 9 else 3 + +def nodes(L): return 2**25 if L == 9 else 8**L + +def to_binary(x): return '' if x == 0 else to_binary(int(x / 256)) + chr(x % 256) + +def from_binary(x): return 0 if x == '' else 256 * from_binary(x[:-1]) + ord(x[-1]) + +def mine(root,difficulty,extranonce): + layers = [[] for x in range(9)] + layers[0] = [root] + for L in range(1,10): + prefix = root + to_binary(extranonce) + to_binary(L) + for i in range(nodes(L)): + p = [] + for k in range(spread(L)): + h = bin_sha256(prefix + to_binary(i) + to_binary(k)) + ind = from_binary(h) % nodes(L-1) + p.append(layers[L-1][ind]) + layers[L].append(bin_sha256(''.join(p))) + print "Computed level ",L + prefix = root + to_binary(extranonce) + for i in range(2**26): + p = [] + for k in range(4): + h = bin_sha256(prefix + to_binary(i) + to_binary(k)) + ind = from_binary(h) % nodes(9) + p.append(layers[9][ind]) + h = from_binary(bin_sha256(''.join(p))) + if h * difficulty <= 2**256: + return i + return None + +def verify(root,difficulty,extranonce,nonce): + layers = [{} for x in range(9)] + layers[0] = [root] + def getnode(L,i): + if i not in layers[L]: + p = [] + for k in range(spread(L)): + h = bin_sha256(root + to_binary(extranonce) + to_binary(L) + to_binary(o) + to_binary(k)) + ind = from_binary(h) % nodes(L-1) + p.append(getnode(L-1,ind)) + layers[L][i] = bin_sha256(''.join(p)) + return layers[L][i] + p = [] + for k in range(4): + h = bin_sha256(root + to_binary(extranonce) + to_binary(nonce) + to_binary(k)) + ind = from_binary(h) % nodes(9) + p.append(getnode(9,ind)) + h = from_binary(bin_sha256(''.join(p))) + return h * difficulty <= 2**256 diff --git a/parser.py b/parser.py new file mode 100644 index 0000000000..00823aeaa3 --- /dev/null +++ b/parser.py @@ -0,0 +1,5 @@ +import rlp + +def parse(inp): + if inp[0] == '\x00': + return { "type": "transaction", "data": rlp.parse( diff --git a/processblock.py b/processblock.py new file mode 100644 index 0000000000..a681a5970c --- /dev/null +++ b/processblock.py @@ -0,0 +1,405 @@ +from transactions import Transaction +from blocks import Block +import time +import sys +import rlp +import math + +scriptcode_map = { + 0x00: 'STOP', + 0x01: 'ADD', + 0x02: 'SUB', + 0x03: 'MUL', + 0x04: 'DIV', + 0x05: 'SDIV', + 0x06: 'MOD', + 0x07: 'SMOD', + 0x08: 'EXP', + 0x09: 'NEG', + 0x0a: 'LT', + 0x0b: 'LE', + 0x0c: 'GT', + 0x0d: 'GE', + 0x0e: 'EQ', + 0x0f: 'NOT', + 0x10: 'MYADDRESS', + 0x11: 'TXSENDER', + 0x12: 'TXVALUE', + 0x13: 'TXFEE', + 0x14: 'TXDATAN', + 0x15: 'TXDATA', + 0x16: 'BLK_PREVHASH', + 0x17: 'BLK_COINBASE', + 0x18: 'BLK_TIMESTAMP', + 0x19: 'BLK_NUMBER', + 0x1a: 'BLK_DIFFICULTY', + 0x20: 'SHA256', + 0x21: 'RIPEMD160', + 0x22: 'ECMUL', + 0x23: 'ECADD', + 0x24: 'ECSIGN', + 0x25: 'ECRECOVER', + 0x26: 'ECVALID', + 0x30: 'PUSH', + 0x31: 'POP', + 0x32: 'DUP', + 0x33: 'DUPN', + 0x34: 'SWAP', + 0x35: 'SWAPN', + 0x36: 'LOAD', + 0x37: 'STORE', + 0x40: 'JMP', + 0x41: 'JMPI', + 0x42: 'IND', + 0x50: 'EXTRO', + 0x51: 'BALANCE', + 0x60: 'MKTX', + 0xff: 'SUICIDE' +} + +params = { + 'stepfee': 1, + 'txfee': 100, + 'newcontractfee': 100, + 'memoryfee': 20, + 'datafee': 4, + 'cryptofee': 10, + 'extrofee': 10, + 'blocktime': 60, + 'period_1_reward': 10**18 * 800, + 'period_1_duration': 57600, + 'period_2_reward': 10**18 * 400, + 'period_2_duration': 57600, + 'period_3_reward': 10**18 * 100, + 'period_3_duration': 57600, + 'period_4_reward': 10**18 * 50 +} + +def getfee(block,t): + if t in ['stepfee','txfee','newcontractfee','memoryfee','datafee','cryptofee','extrofee']: + return int(10**24 / int(block.difficulty ** 0.5)) * params[t] + +def process_transactions(block,transactions): + while len(transactions) > 0: + tx = transactions.pop(0) + enc = (tx.value, tx.fee, tx.sender.encode('hex'), tx.to.encode('hex')) + sys.stderr.write("Attempting to send %d plus fee %d from %s to %s\n" % enc) + # Grab data about sender, recipient and miner + sdata = rlp.decode(block.state.get(tx.sender)) or [0,0,0] + tdata = rlp.decode(block.state.get(tx.to)) or [0,0,0] + # Calculate fee + if tx.to == '\x00'*20: + fee = getfee('newcontractfee') + len(tx.data) * getfee('memoryfee') + else: + fee = getfee('txfee') + # Insufficient fee, do nothing + if fee > tx.fee: + sys.stderr.write("Insufficient fee\n") + continue + # Too much data, do nothing + if len(tx.data) > 256: + sys.stderr.write("Too many data items\n") + continue + if not sdata or sdata[1] < tx.value + tx.fee: + sys.stderr.write("Insufficient funds to send fee\n") + continue + elif tx.nonce != sdata[2] and sdata[0] == 0: + sys.stderr.write("Bad nonce\n") + continue + # Try to send the tx + if sdata[0] == 0: sdata[2] += 1 + sdata[1] -= (tx.value + tx.fee) + block.reward += tx.fee + if tx.to != '': + tdata[1] += tx.value + else: + addr = tx.hash()[-20:] + adata = rlp.decode(block.state.get(addr)) + if adata[2] != '': + sys.stderr.write("Contract already exists\n") + continue + block.state.update(addr,rlp.encode([1,tx.value,''])) + contract = block.get_contract(addr) + for i in range(len(tx.data)): + contract.update(encode(i,256,32),tx.data[i]) + block.update_contract(addr) + print sdata, tdata + block.state.update(tx.sender,rlp.encode(sdata)) + block.state.update(tx.to,rlp.encode(tdata)) + # Evaluate contract if applicable + if tdata[0] == 1: + eval_contract(block,transactions,tx) + sys.stderr.write("tx processed\n") + +def eval(block,transactions,timestamp,coinbase): + h = block.hash() + # Process all transactions + process_transactions(block,transactions) + # Pay miner fee + miner_state = rlp.decode(block.state.get(block.coinbase)) or [0,0,0] + block.number += 1 + reward = 0 + if block.number < params['period_1_duration']: + reward = params['period_1_reward'] + elif block.number < params['period_2_duration']: + reward = params['period_2_reward'] + elif block.number < params['period_3_duration']: + reward = params['period_3_reward'] + else: + reward = params['period_4_reward'] + miner_state[1] += reward + block.reward + for uncle in block.uncles: + sib_miner_state = rlp_decode(block.state.get(uncle[3])) + sib_miner_state[1] += reward*7/8 + block.state.update(uncle[3],sib_miner_state) + miner_state[1] += reward/8 + block.state.update(block.coinbase,rlp.encode(miner_state)) + # Check timestamp + if timestamp < block.timestamp or timestamp > int(time.time()) + 3600: + raise Exception("timestamp not in valid range!") + # Update difficulty + if timestamp >= block.timestamp + 42: + block.difficulty += int(block.difficulty / 1024) + else: + block.difficulty -= int(block.difficulty / 1024) + block.prevhash = h + block.coinbase = coinbase + block.transactions = [] + block.uncles = [] + return block + +def eval_contract(block,transaction_list,tx): + sys.stderr.write("evaluating contract\n") + address = tx.to + # Initialize stack + stack = [] + index = 0 + stepcounter = 0 + contract = block.get_contract(address) + if not contract: + return + while 1: + # Convert the data item into a code piece + val_at_index = decode(contract.get(encode(index,256,32)),256) + code = [ int(val_at_index / (256**i)) % 256 for i in range(6) ] + code[0] = scriptcode_map.get(code[0],'INVALID') + sys.stderr.write("Evaluating: "+ str(code)+"\n") + # Invalid code instruction or STOP code stops execution sans fee + if val_at_index >= 256 or code[0] in ['STOP','INVALID']: + sys.stderr.write("stop code, exiting\n") + break + # Calculate fee + minerfee = 0 + nullfee = 0 + stepcounter += 1 + if stepcounter > 16: + minerfee += getfee("stepfee") + c = scriptcode_map[code[0]] + if c in ['STORE','LOAD']: + minerfee += getfee("datafee") + if c in ['EXTRO','BALANCE']: + minerfee += getfee("extrofee") + if c in ['SHA256','RIPEMD-160','ECMUL','ECADD','ECSIGN','ECRECOVER']: + minerfee += getfee("cryptofee") + if c == 'STORE': + existing = block.get_contract_state(address,code[2]) + if reg[code[1]] != 0: nullfee += getfee("memoryfee") + if existing: nullfee -= getfee("memoryfee") + + # If we can't pay the fee, break, otherwise pay it + if block.get_balance(address) < minerfee + nullfee: + sys.stderr.write("insufficient fee, exiting\n") + break + block.set_balance(address,block.get_balance(address) - nullfee - minerfee) + block.reward += minerfee + sys.stderr.write("evaluating operation\n") + exit = False + def stack_pop(n): + if len(stack) < n: + sys.stderr.write("Stack height insufficient, exiting") + exit = True + return [0] * n + o = stack[-n:] + stack = stack[:-n] + return o + # Evaluate operations + if c == 'ADD': + x,y = stack_pop(2) + stack.append((x + y) % 2**256) + elif c == 'MUL': + x,y = stack_pop(2) + stack.append((x * y) % 2**256) + elif c == 'SUB': + x,y = stack_pop(2) + stack.append((x - y) % 2**256) + elif c == 'DIV': + x,y = stack_pop(2) + if y == 0: break + stack.append(int(x / y)) + elif c == 'SDIV': + x,y = stack_pop(2) + if y == 0: break + sign = (1 if x < 2**255 else -1) * (1 if y < 2**255 else -1) + xx = x if x < 2**255 else 2**256 - x + yy = y if y < 2**255 else 2**256 - y + z = int(xx/yy) + stack.append(z if sign == 1 else 2**256 - z) + elif code == 'MOD': + x,y = stack_pop(2) + if y == 0: break + stack.append(x % y) + elif code == 'SMOD': + x,y = stack_pop(2) + if y == 0: break + sign = (1 if x < 2**255 else -1) * (1 if y < 2**255 else -1) + xx = x if x < 2**255 else 2**256 - x + yy = y if y < 2**255 else 2**256 - y + z = xx%yy + stack.append(z if sign == 1 else 2**256 - z) + elif code == 'EXP': + x,y = stack_pop(2) + stack.append(pow(x,y,2**256)) + elif code == 'NEG': + stack.append(2**256 - stack.pop(1)[0]) + elif code == 'LT': + x,y = stack_pop(2) + stack.append(1 if x < y else 0) + elif code == 'LE': + x,y = stack_pop(2) + stack.append(1 if x <= y else 0) + elif code == 'GT': + x,y = stack_pop(2) + stack.append(1 if x > y else 0) + elif code == 'GE': + x,y = stack_pop(2) + stack.append(1 if x >= y else 0) + elif code == 'EQ': + x,y = stack_pop(2) + stack.append(1 if x == y else 0) + elif code == 'NOT': + stack.append(1 if stack.pop(1)[0] == 0 else 0) + elif code == 'MYADDRESS': + stack.append(address) + elif code == 'TXSENDER': + stack.append(decode(tx.sender,256)) + elif code == 'TXVALUE': + stack.append(tx.value) + elif code == 'TXFEE': + stack.append(tx.fee) + elif code == 'TXDATAN': + stack.append(len(tx.data)) + elif code == 'TXDATA': + x, = stack_pop(1) + stack.append(0 if x >= len(tx.data) else tx.data[x]) + elif code == 'BLK_PREVHASH': + stack.append(decode(block.prevhash,256)) + elif code == 'BLK_COINBASE': + stack.append(decode(block.coinbase,160)) + elif code == 'BLK_TIMESTAMP': + stack.append(block.timestamp) + elif code == 'BLK_NUMBER': + stack.append(block.number) + elif code == 'BLK_DIFFICULTY': + stack.append(block.difficulty) + elif code == 'SHA256': + L = stack_pop(1) + hdataitems = stack_pop(math.ceil(L / 32.0)) + hdata = ''.join([encode(x,256,32) for x in hdataitems])[:L] + stack.append(decode(hashlib.sha256(hdata).digest(),256)) + elif code == 'RIPEMD-160': + L = stack_pop(1) + hdataitems = stack_pop(math.ceil(L / 32.0)) + hdata = ''.join([encode(x,256,32) for x in hdataitems])[:L] + stack.append(decode(hashlib.new('ripemd160',hdata).digest(),256)) + elif code == 'ECMUL': + n,x,y = stack_pop(3) + # Point at infinity + if x == 0 and y == 0: + stack.extend([0,0]) + # Point not on curve, coerce to infinity + elif x >= P or y >= P or (x ** 3 + 7 - y ** 2) % P != 0: + stack.extend([0,0]) + # Legitimate point + else: + x2,y2 = base10_multiply((x,y),n) + stack.extend([x2,y2]) + elif code == 'ECADD': + x1,y1,x2,y2 = stack_pop(4) + # Invalid point 1 + if x1 >= P or y1 >= P or (x1 ** 3 + 7 - y1 ** 2) % P != 0: + stack.extend([0,0]) + # Invalid point 2 + elif x2 >= P or y2 >= P or (x2 ** 3 + 7 - y2 ** 2) % P != 0: + stack.extend([0,0]) + # Legitimate points + else: + x3,y3 = base10_add((x1,y1),(x2,y2)) + stack.extend([x3,y3]) + elif code == 'ECSIGN': + k,h = stack_pop(2) + v,r,s = ecdsa_raw_sign(h,k) + stack.extend([v,r,s]) + elif code == 'ECRECOVER': + h,v,r,s = stack_pop(4) + x,y = ecdsa_raw_recover((v,r,s),h) + stack.extend([x,y]) + elif code == 'PUSH': + stack.append(contract.get(encode(index + 1,256,32))) + index += 1 + elif code == 'POP': + stack_pop(1) + elif code == 'DUP': + x, = stack_pop(1) + stack.extend([x,x]) + elif code == 'DUPN': + arr = stack_pop(contract.get(encode(index + 1,256,32))) + arr.append(arr[0]) + stack.extend(arr) + index += 1 + elif code == 'SWAP': + x,y = stack_pop(2) + stack.extend([y,x]) + elif code == 'SWAPN': + arr = stack_pop(contract.get(encode(index + 1,256,32))) + arr.append(arr[0]) + arr.pop(0) + stack.extend(arr) + index += 1 + elif code == 'LOAD': + stack.append(contract.get(encode(stack_pop(1)[0],256,32))) + elif code == 'STORE': + x,y = stack_pop(2) + if exit: break + contract.update(encode(x,256,32),y) + elif code == 'JMP': + index = stack_pop(1)[0] + elif code == 'JMPI': + newpos,c = stack_pop(2) + if c != 0: index = newpos + elif code == 'IND': + stack.append(index) + elif code == 'EXTRO': + ind,addr = stack_pop(2) + stack.push(block.get_contract(encode(addr,256,20)).get(encode(ind,256,32))) + elif code == 'BALANCE': + stack.push(block.get_balance(encode(stack_pop(1)[0],256,20))) + elif code == 'MKTX': + datan,fee,value,to = stack_pop(4) + if exit: + break + elif (value + fee) > block.get_balance(address): + break + else: + data = stack_pop(datan) + tx = Transaction(0,encode(to,256,20),value,fee,data) + tx.sender = address + transaction_list.insert(0,tx) + elif code == 'SUICIDE': + sz = contract.get_size() + negfee = -sz * getfee("memoryfee") + toaddress = encode(stack_pop(1)[0],256,20) + block.pay_fee(toaddress,negfee,False) + contract.root = '' + break + if exit: break + block.update_contract(address,contract) diff --git a/rlp.py b/rlp.py new file mode 100644 index 0000000000..e35f111941 --- /dev/null +++ b/rlp.py @@ -0,0 +1,86 @@ +def binary_length(n): + if n == 0: return 0 + else: return 1 + binary_length(n / 256) + +def to_binary_array(n,L=None): + if L is None: L = binary_length(n) + if n == 0: return [] + else: + x = to_binary_array(n / 256) + x.append(n % 256) + return x + +def to_binary(n,L=None): return ''.join([chr(x) for x in to_binary_array(n,L)]) + +def from_binary(b): + if len(b) == 0: return 0 + else: return from_binary(b[:-1]) * 256 + ord(b[-1]) + +def __decode(s,pos=0): + if not s: + return (None, 0) + else: + fchar = ord(s[pos]) + if fchar < 24: + return (ord(s[pos]), pos+1) + elif fchar < 56: + b = ord(s[pos]) - 23 + return (from_binary(s[pos+1:pos+1+b]), pos+1+b) + elif fchar < 64: + b = ord(s[pos]) - 55 + b2 = from_binary(s[pos+1:pos+1+b]) + return (from_binary(s[pos+1+b:pos+1+b+b2]), pos+1+b+b2) + elif fchar < 120: + b = ord(s[pos]) - 64 + return (s[pos+1:pos+1+b], pos+1+b) + elif fchar < 128: + b = ord(s[pos]) - 119 + b2 = from_binary(s[pos+1:pos+1+b]) + return (s[pos+1+b:pos+1+b+b2], pos+1+b+b2) + elif fchar < 184: + b = ord(s[pos]) - 128 + o, pos = [], pos+1 + for i in range(b): + obj, pos = __decode(s,pos) + o.append(obj) + return (o,pos) + elif fchar < 192: + b = ord(s[pos]) - 183 + b2 = from_binary(s[pos+1:pos+1+b]) + o, pos = [], pos+1+b + for i in range(b): + obj, pos = __decode(s,pos) + o.append(obj) + return (o,pos) + else: + raise Exception("byte not supported: "+fchar) + +def decode(s): return __decode(s)[0] + +def encode(s): + if isinstance(s,(int,long)): + if s < 0: + raise Exception("can't handle negative ints") + elif s >= 0 and s < 24: + return chr(s) + elif s < 2**256: + b = to_binary(s) + return chr(len(b) + 23) + b + else: + b = to_binary(s) + b2 = to_binary(len(b)) + return chr(len(b2) + 55) + b2 + b + elif isinstance(s,(str,unicode)): + if len(s) < 56: + return chr(len(s) + 64) + str(s) + else: + b2 = to_binary(len(s)) + return chr(len(b2) + 119) + b2 + str(s) + elif isinstance(s,list): + if len(s) < 56: + return chr(len(s) + 128) + ''.join([encode(x) for x in s]) + else: + b2 = to_binary(len(s)) + return chr(len(b2) + 183) + b2 + ''.join([encode(x) for x in s]) + else: + raise Exception("Encoding for "+s+" not yet implemented") diff --git a/runtest.py b/runtest.py new file mode 100644 index 0000000000..5ef92f5d57 --- /dev/null +++ b/runtest.py @@ -0,0 +1,81 @@ +import json, sys, os +import rlp, trie +import random + +testdir = sys.argv[1] if len(sys.argv) >= 2 else '../tests' + +rlpdata = json.loads(open(os.path.join(testdir,'rlptest.txt')).read()) +for x,y in rlpdata: + yprime = rlp.encode(x).encode('hex') + if yprime != y: print "RLPEncode Mismatch: ",x,y,yprime + xprime = rlp.decode(y.decode('hex')) + jx, jxprime = json.dumps(x), json.dumps(xprime) + if jx != jxprime: print "RLPDecode Mismatch: ",jx,jxprime,y + +hexencodedata = json.loads(open(os.path.join(testdir,'hexencodetest.txt')).read()) + +for x,y in hexencodedata: + yprime = trie.hexarraykey_to_bin(x).encode('hex') + if yprime != y: print "HexEncode Mismatch: ",x,y,yprime + xprime = trie.bin_to_hexarraykey(y.decode('hex')) + jx,jxprime = json.dumps(x), json.dumps(xprime) + if jx != jxprime: print "HexDecode Mismatch: ",jx,jxprime,y + +triedata = json.loads(open(os.path.join(testdir,'trietest.txt')).read()) + +for x,y in triedata: + t0 = trie.Trie('/tmp/trietest-'+str(random.randrange(1000000000000))) + for k in x: + t0.update(k,x[k]) + if t0.root.encode('hex') != y: + print "Mismatch with adds only" + continue + t = trie.Trie('/tmp/trietest-'+str(random.randrange(1000000000000))) + dummies, reals = [], [] + for k in x: + reals.append([k,x[k]]) + dummies.append(k[:random.randrange(len(k)-1)]) + dummies.append(k+random.choice(dummies)) + dummies.append(k[:random.randrange(len(k)-1)]+random.choice(dummies)) + dummies_to_pop = set([]) + i = 0 + ops = [] + mp = {} + success = [True] + def update(k,v): + t.update(k,v) + if v == '' and k in mp: del mp[k] + else: mp[k] = v + ops.append([k,v,t.root.encode('hex')]) + tn = trie.Trie('/tmp/trietest-'+str(random.randrange(1000000000000))) + for k in mp: + tn.update(k,mp[k]) + if tn.root != t.root: + print "Mismatch: " + for op in ops: print op + success[0] = False + while i < len(reals): + s = random.randrange(3) + if s == 0: + update(reals[i][0],reals[i][1]) + i += 1 + elif s == 1: + k,v = random.choice(dummies), random.choice(dummies) + update(k,v) + dummies_to_pop.add(k) + elif s == 2: + if len(dummies_to_pop) > 0: + k = random.choice(list(dummies_to_pop)) + update(k,'') + dummies_to_pop.remove(k) + if not success[0]: + break + if not success[0]: + continue + i = len(reals) * 2 + while len(dummies_to_pop) > 0: + k = random.choice(list(dummies_to_pop)) + update(k,'') + dummies_to_pop.remove(k) + if not success[0]: + break diff --git a/transactions.py b/transactions.py new file mode 100644 index 0000000000..1cd1d74512 --- /dev/null +++ b/transactions.py @@ -0,0 +1,47 @@ +from pybitcointools import * +import rlp +import re + +class Transaction(): + def __init__(*args): + self = args[0] + if len(args) == 2: + self.parse(args[1]) + else: + self.nonce = args[1] + self.to = args[2] + self.value = args[3] + self.fee = args[4] + self.data = args[5] + + def parse(self,data): + if re.match('^[0-9a-fA-F]*$',data): + data = data.decode('hex') + o = rlp.decode(data) + self.nonce = o[0] + self.to = o[1] + self.value = o[2] + self.fee = o[3] + self.data = o[4] + self.v = o[5] + self.r = o[6] + self.s = o[7] + rawhash = sha256(rlp.encode([self.nonce,self.to,self.value,self.fee,self.data])) + pub = encode_pubkey(ecdsa_raw_recover(rawhash,(self.v,self.r,self.s)),'bin') + self.sender = bin_sha256(pub[1:])[-20:] + return self + + def sign(self,key): + rawhash = sha256(rlp.encode([self.nonce,self.to,self.value,self.fee,self.data])) + self.v,self.r,self.s = ecdsa_raw_sign(rawhash,key) + self.sender = bin_sha256(privtopub(key)[1:])[-20:] + return self + + def serialize(self): + return rlp.encode([self.nonce, self.to, self.value, self.fee, self.data, self.v, self.r, self.s]) + + def hex_serialize(self): + return self.serialize().encode('hex') + + def hash(self): + return bin_sha256(self.serialize()) diff --git a/trie.py b/trie.py new file mode 100644 index 0000000000..2aa9a3d0fd --- /dev/null +++ b/trie.py @@ -0,0 +1,233 @@ +import leveldb +import rlp +from sha3 import sha3_256 + +def sha3(x): return sha3_256(x).digest() + +class DB(): + def __init__(self,dbfile): self.db = leveldb.LevelDB(dbfile) + def get(self,key): + try: return self.db.Get(key) + except: return '' + def put(self,key,value): return self.db.Put(key,value) + def delete(self,key): return self.db.Delete(key) + +def hexarraykey_to_bin(key): + term = 1 if key[-1] == 16 else 0 + if term: key = key[:-1] + oddlen = len(key) % 2 + flags = 2 * term + oddlen + if oddlen: key = [flags] + key + else: key = [flags,0] + key + o = '' + for i in range(0,len(key),2): + o += chr(16 * key[i] + key[i+1]) + return o + +def bin_to_hexarraykey(bindata): + o = ['0123456789abcdef'.find(x) for x in bindata.encode('hex')] + if o[0] >= 2: o.append(16) + if o[0] % 2 == 1: o = o[1:] + else: o = o[2:] + return o + +databases = {} + +class Trie(): + def __init__(self,dbfile,root='',debug=False): + self.root = root + self.debug = debug + if dbfile not in databases: + databases[dbfile] = DB(dbfile) + self.db = databases[dbfile] + + def __get_state(self,node,key): + if self.debug: print 'nk',node.encode('hex'),key + if len(key) == 0 or not node: + return node + curnode = self.lookup(node) + if self.debug: print 'cn', curnode + if not curnode: + raise Exception("node not found in database") + elif len(curnode) == 2: + (k2,v2) = curnode + k2 = bin_to_hexarraykey(k2) + if len(key) >= len(k2) and k2 == key[:len(k2)]: + return self.__get_state(v2,key[len(k2):]) + else: + return '' + elif len(curnode) == 17: + return self.__get_state(curnode[key[0]],key[1:]) + + def __put(self,node,root=False): + rlpnode = rlp.encode(node) + if len(rlpnode) >= 32: + h = sha3(rlpnode) + self.db.put(h,rlpnode) + else: + h = rlpnode if root else node + return h + + def lookup(self,node): + if not isinstance(node,(str,unicode)): return node + elif len(node) == 0: return node + elif len(node) < 32: return rlp.decode(node) + else: return rlp.decode(self.db.get(node)) + + def __update_state(self,node,key,value): + if value != '': return self.__insert_state(node,key,value) + else: return self.__delete_state(node,key) + + def __insert_state(self,node,key,value): + if self.debug: print 'ins', node.encode('hex'), key + if len(key) == 0: + return value + else: + if not node: + newnode = [ hexarraykey_to_bin(key), value ] + return self.__put(newnode) + curnode = self.lookup(node) + if self.debug: print 'icn', curnode + if not curnode: + raise Exception("node not found in database") + if len(curnode) == 2: + (k2, v2) = curnode + k2 = bin_to_hexarraykey(k2) + if key == k2: + newnode = [ hexarraykey_to_bin(key), value ] + return self.__put(newnode) + else: + i = 0 + while key[:i+1] == k2[:i+1] and i < len(k2): i += 1 + if i == len(k2): + newhash3 = self.__insert_state(v2,key[i:],value) + else: + newnode1 = self.__insert_state('',key[i+1:],value) + newnode2 = self.__insert_state('',k2[i+1:],v2) + newnode3 = [ '' ] * 17 + newnode3[key[i]] = newnode1 + newnode3[k2[i]] = newnode2 + newhash3 = self.__put(newnode3) + if i == 0: + return newhash3 + else: + newnode4 = [ hexarraykey_to_bin(key[:i]), newhash3 ] + return self.__put(newnode4) + else: + newnode = [ curnode[i] for i in range(17) ] + newnode[key[0]] = self.__insert_state(curnode[key[0]],key[1:],value) + return self.__put(newnode) + + def __delete_state(self,node,key): + if self.debug: print 'dnk', node.encode('hex'), key + if len(key) == 0 or not node: + return '' + else: + curnode = self.lookup(node) + if not curnode: + raise Exception("node not found in database") + if self.debug: print 'dcn', curnode + if len(curnode) == 2: + (k2, v2) = curnode + k2 = bin_to_hexarraykey(k2) + if key == k2: + return '' + elif key[:len(k2)] == k2: + newhash = self.__delete_state(v2,key[len(k2):]) + childnode = self.lookup(newhash) + if len(childnode) == 2: + newkey = k2 + bin_to_hexarraykey(childnode[0]) + newnode = [ hexarraykey_to_bin(newkey), childnode[1] ] + else: + newnode = [ curnode[0], newhash ] + return self.__put(newnode) + else: return node + else: + newnode = [ curnode[i] for i in range(17) ] + newnode[key[0]] = self.__delete_state(newnode[key[0]],key[1:]) + onlynode = -1 + for i in range(17): + if newnode[i]: + if onlynode == -1: onlynode = i + else: onlynode = -2 + if onlynode == 16: + newnode2 = [ hexarraykey_to_bin([16]), newnode[onlynode] ] + elif onlynode >= 0: + childnode = self.lookup(newnode[onlynode]) + if not childnode: + raise Exception("?????") + if len(childnode) == 17: + newnode2 = [ hexarraykey_to_bin([onlynode]), newnode[onlynode] ] + elif len(childnode) == 2: + newkey = [onlynode] + bin_to_hexarraykey(childnode[0]) + newnode2 = [ hexarraykey_to_bin(newkey), childnode[1] ] + else: + newnode2 = newnode + return self.__put(newnode2) + + def __get_size(self,node): + if not node: return 0 + curnode = self.lookup(node) + if not curnode: + raise Exception("node not found in database") + if len(curnode) == 2: + key = hexarraykey_to_bin(curnode[0]) + if key[-1] == 16: return 1 + else: return self.__get_size(curnode[1]) + elif len(curnode) == 17: + total = 0 + for i in range(16): + total += self.__get_size(curnode[i]) + if curnode[16]: total += 1 + return total + + def __to_dict(self,node): + if not node: return {} + curnode = self.lookup(node) + if not curnode: + raise Exception("node not found in database") + if len(curnode) == 2: + lkey = bin_to_hexarraykey(curnode[0]) + o = {} + if lkey[-1] == 16: + o[curnode[0]] = curnode[1] + else: + d = self.__to_dict(curnode[1]) + for v in d: + subkey = bin_to_hexarraykey(v) + totalkey = hexarraykey_to_bin(lkey+subkey) + o[totalkey] = d[v] + return o + elif len(curnode) == 17: + o = {} + for i in range(16): + d = self.__to_dict(curnode[i]) + for v in d: + subkey = bin_to_hexarraykey(v) + totalkey = hexarraykey_to_bin([i] + subkey) + o[totalkey] = d[v] + if curnode[16]: o[chr(16)] = curnode[16] + return o + else: + raise Exception("bad curnode! "+curnode) + + def to_dict(self,as_hex=False): + d = self.__to_dict(self.root) + o = {} + for v in d: + v2 = ''.join(['0123456789abcdef'[x] for x in bin_to_hexarraykey(v)[:-1]]) + if not as_hex: v2 = v2.decode('hex') + o[v2] = d[v] + return o + + def get(self,key): + key2 = ['0123456789abcdef'.find(x) for x in str(key).encode('hex')] + [16] + return self.__get_state(self.root,key2) + + def get_size(self): return self.__get_size(self.root) + + def update(self,key,value): + if not isinstance(key,(str,unicode)) or not isinstance(value,(str,unicode)): + raise Exception("Key and value must be strings") + key2 = ['0123456789abcdef'.find(x) for x in str(key).encode('hex')] + [16] + self.root = self.__update_state(self.root,key2,str(value)) diff --git a/trietest.py b/trietest.py new file mode 100644 index 0000000000..d427b1e4f9 --- /dev/null +++ b/trietest.py @@ -0,0 +1,25 @@ +from trie import Trie +import random + +def genkey(): + L = random.randrange(30) + if random.randrange(5) == 0: return '' + return ''.join([random.choice('1234579qetyiasdfghjklzxcvbnm') for x in range(L)]) + +t = Trie('/tmp/'+genkey()) + +def trie_test(): + o = {} + for i in range(60): + key, value = genkey(), genkey() + if value: print "setting key: '"+key+"', value: '"+value+"'" + else: print "deleting key: '"+key+"'" + o[key] = value + t.update(key,value) + for k in o.keys(): + v1 = o[k] + v2 = t.get(k) + print v1,v2 + if v1 != v2: raise Exception("incorrect!") + +trie_test()