interesting
/
go-ethereum
mirror of https://github.com/ethereum/go-ethereum.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

godeps: update leveldb and snappy, dump serpent-go

This commit is contained in:
Péter Szilágyi 2015-04-28 12:18:01 +03:00
parent 182d484aa7
commit 7e3b080f85
155 changed files with 7615 additions and 10150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
Godeps/Godeps.json generated
View File

@ -10,11 +10,6 @@
"Comment": "null-12",
"Rev": "7dda39b2e7d5e265014674c5af696ba4186679e9"
},
{
"ImportPath": "code.google.com/p/snappy-go/snappy",
"Comment": "null-15",
"Rev": "12e4b4183793ac4b061921e7980845e750679fd0"
},
{
"ImportPath": "github.com/codegangsta/cli",
"Comment": "1.2.0-95-g9b2bd2b",
@ -25,10 +20,6 @@
"Comment": "v23.1-82-g908aad3",
"Rev": "908aad345c9fbf3ab9bbb94031dc02d0d90df1b8"
},
{
"ImportPath": "github.com/ethereum/serpent-go",
"Rev": "5767a0dbd759d313df3f404dadb7f98d7ab51443"
},
{
"ImportPath": "github.com/howeyc/fsnotify",
"Comment": "v0.9.0-11-g6b1ef89",
@ -46,10 +37,6 @@
"ImportPath": "github.com/kardianos/osext",
"Rev": "ccfcd0245381f0c94c68f50626665eed3c6b726a"
},
{
"ImportPath": "github.com/robertkrimen/otto",
"Rev": "dea31a3d392779af358ec41f77a07fcc7e9d04ba"
},
{
"ImportPath": "github.com/obscuren/qml",
"Rev": "c288002b52e905973b131089a8a7c761d4a2c36a"
@ -67,27 +54,7 @@
"Rev": "907cca0f578a5316fb864ec6992dc3d9730ec58c"
},
{
"ImportPath": "github.com/robertkrimen/otto/ast",
"Rev": "dea31a3d392779af358ec41f77a07fcc7e9d04ba"
},
{
"ImportPath": "github.com/robertkrimen/otto/dbg",
"Rev": "dea31a3d392779af358ec41f77a07fcc7e9d04ba"
},
{
"ImportPath": "github.com/robertkrimen/otto/file",
"Rev": "dea31a3d392779af358ec41f77a07fcc7e9d04ba"
},
{
"ImportPath": "github.com/robertkrimen/otto/parser",
"Rev": "dea31a3d392779af358ec41f77a07fcc7e9d04ba"
},
{
"ImportPath": "github.com/robertkrimen/otto/registry",
"Rev": "dea31a3d392779af358ec41f77a07fcc7e9d04ba"
},
{
"ImportPath": "github.com/robertkrimen/otto/token",
"ImportPath": "github.com/robertkrimen/otto",
"Rev": "dea31a3d392779af358ec41f77a07fcc7e9d04ba"
},
{
@ -96,7 +63,11 @@
},
{
"ImportPath": "github.com/syndtr/goleveldb/leveldb",
"Rev": "832fa7ed4d28545eab80f19e1831fc004305cade"
"Rev": "4875955338b0a434238a31165cb87255ab6e9e4a"
},
{
"ImportPath": "github.com/syndtr/gosnappy/snappy",
"Rev": "156a073208e131d7d2e212cb749feae7c339e846"
},
{
"ImportPath": "golang.org/x/crypto/pbkdf2",

124
Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/decode.go generated vendored
View File

@ -1,124 +0,0 @@
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package snappy
import (
"encoding/binary"
"errors"
)
// ErrCorrupt reports that the input is invalid.
var ErrCorrupt = errors.New("snappy: corrupt input")
// DecodedLen returns the length of the decoded block.
func DecodedLen(src []byte) (int, error) {
v, _, err := decodedLen(src)
return v, err
}
// decodedLen returns the length of the decoded block and the number of bytes
// that the length header occupied.
func decodedLen(src []byte) (blockLen, headerLen int, err error) {
v, n := binary.Uvarint(src)
if n == 0 {
return 0, 0, ErrCorrupt
}
if uint64(int(v)) != v {
return 0, 0, errors.New("snappy: decoded block is too large")
}
return int(v), n, nil
}
// Decode returns the decoded form of src. The returned slice may be a sub-
// slice of dst if dst was large enough to hold the entire decoded block.
// Otherwise, a newly allocated slice will be returned.
// It is valid to pass a nil dst.
func Decode(dst, src []byte) ([]byte, error) {
dLen, s, err := decodedLen(src)
if err != nil {
return nil, err
}
if len(dst) < dLen {
dst = make([]byte, dLen)
}
var d, offset, length int
for s < len(src) {
switch src[s] & 0x03 {
case tagLiteral:
x := uint(src[s] >> 2)
switch {
case x < 60:
s += 1
case x == 60:
s += 2
if s > len(src) {
return nil, ErrCorrupt
}
x = uint(src[s-1])
case x == 61:
s += 3
if s > len(src) {
return nil, ErrCorrupt
}
x = uint(src[s-2]) | uint(src[s-1])<<8
case x == 62:
s += 4
if s > len(src) {
return nil, ErrCorrupt
}
x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16
case x == 63:
s += 5
if s > len(src) {
return nil, ErrCorrupt
}
x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24
}
length = int(x + 1)
if length <= 0 {
return nil, errors.New("snappy: unsupported literal length")
}
if length > len(dst)-d || length > len(src)-s {
return nil, ErrCorrupt
}
copy(dst[d:], src[s:s+length])
d += length
s += length
continue
case tagCopy1:
s += 2
if s > len(src) {
return nil, ErrCorrupt
}
length = 4 + int(src[s-2])>>2&0x7
offset = int(src[s-2])&0xe0<<3 | int(src[s-1])
case tagCopy2:
s += 3
if s > len(src) {
return nil, ErrCorrupt
}
length = 1 + int(src[s-3])>>2
offset = int(src[s-2]) | int(src[s-1])<<8
case tagCopy4:
return nil, errors.New("snappy: unsupported COPY_4 tag")
}
end := d + length
if offset > d || end > len(dst) {
return nil, ErrCorrupt
}
for ; d < end; d++ {
dst[d] = dst[d-offset]
}
}
if d != dLen {
return nil, ErrCorrupt
}
return dst[:d], nil
}

5
Godeps/_workspace/src/github.com/ethereum/serpent-go/.gitignore generated vendored
View File

@ -1,5 +0,0 @@
/tmp
*/**/*un~
*un~
.DS_Store
*/**/.DS_Store

3
Godeps/_workspace/src/github.com/ethereum/serpent-go/.gitmodules generated vendored
View File

@ -1,3 +0,0 @@
[submodule "serp"]
path = serpent
url = https://github.com/ethereum/serpent.git

12
Godeps/_workspace/src/github.com/ethereum/serpent-go/README.md generated vendored
View File

@ -1,12 +0,0 @@
[serpent](https://github.com/ethereum/serpent) go bindings.
## Build instructions
```
go get -d github.com/ethereum/serpent-go
cd $GOPATH/src/github.com/ethereum/serpent-go
git submodule init
git submodule update
```
You're now ready to go :-)

16
Godeps/_workspace/src/github.com/ethereum/serpent-go/all.cpp generated vendored
View File

@ -1,16 +0,0 @@
#include "serpent/bignum.cpp"
#include "serpent/util.cpp"
#include "serpent/tokenize.cpp"
#include "serpent/parser.cpp"
#include "serpent/compiler.cpp"
#include "serpent/funcs.cpp"
#include "serpent/lllparser.cpp"
#include "serpent/rewriter.cpp"
#include "serpent/opcodes.cpp"
#include "serpent/optimize.cpp"
#include "serpent/functions.cpp"
#include "serpent/preprocess.cpp"
#include "serpent/rewriteutils.cpp"
#include "cpp/api.cpp"

26
Godeps/_workspace/src/github.com/ethereum/serpent-go/cpp/api.cpp generated vendored
View File

@ -1,26 +0,0 @@
#include <string>
#include "serpent/lllparser.h"
#include "serpent/bignum.h"
#include "serpent/util.h"
#include "serpent/tokenize.h"
#include "serpent/parser.h"
#include "serpent/compiler.h"
#include "cpp/api.h"
const char *compileGo(char *code, int *err)
{
try {
std::string c = binToHex(compile(std::string(code)));
return c.c_str();
}
catch(std::string &error) {
*err = 1;
return error.c_str();
}
catch(...) {
return "Unknown error";
}
}

14
Godeps/_workspace/src/github.com/ethereum/serpent-go/cpp/api.h generated vendored
View File

@ -1,14 +0,0 @@
#ifndef CPP_API_H
#define CPP_API_H
#ifdef __cplusplus
extern "C" {
#endif
const char *compileGo(char *code, int *err);
#ifdef __cplusplus
}
#endif
#endif

27
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent.go generated vendored
View File

@ -1,27 +0,0 @@
package serpent
// #cgo CXXFLAGS: -I. -Ilangs/ -std=c++0x -Wall -fno-strict-aliasing
// #cgo LDFLAGS: -lstdc++
//
// #include "cpp/api.h"
//
import "C"
import (
"encoding/hex"
"errors"
"unsafe"
)
func Compile(str string) ([]byte, error) {
var err C.int
out := C.GoString(C.compileGo(C.CString(str), (*C.int)(unsafe.Pointer(&err))))
if err == C.int(1) {
return nil, errors.New(out)
}
bytes, _ := hex.DecodeString(out)
return bytes, nil
}

12
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/.gitignore generated vendored
View File

@ -1,12 +0,0 @@
[._]*.s[a-w][a-z]
[._]s[a-w][a-z]
*.un~
Session.vim
.netrwhist
*~
*.o
serpent
libserpent.a
pyserpent.so
dist
*.egg-info

5
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/MANIFEST.in generated vendored
View File

@ -1,5 +0,0 @@
include *.cpp
include *.h
include *py
include README.md
include Makefile

55
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/Makefile generated vendored
View File

@ -1,55 +0,0 @@
PLATFORM_OPTS =
PYTHON = /usr/include/python2.7
CXXFLAGS = -fPIC
# -g3 -O0
BOOST_INC = /usr/include
BOOST_LIB = /usr/lib
TARGET = pyserpent
COMMON_OBJS = bignum.o util.o tokenize.o lllparser.o parser.o opcodes.o optimize.o functions.o rewriteutils.o preprocess.o rewriter.o compiler.o funcs.o
HEADERS = bignum.h util.h tokenize.h lllparser.h parser.h opcodes.h functions.h optimize.h rewriteutils.h preprocess.h rewriter.h compiler.h funcs.h
PYTHON_VERSION = 2.7
serpent : serpentc lib
lib:
ar rvs libserpent.a $(COMMON_OBJS)
g++ $(CXXFLAGS) -shared $(COMMON_OBJS) -o libserpent.so
serpentc: $(COMMON_OBJS) cmdline.o
rm -rf serpent
g++ -Wall $(COMMON_OBJS) cmdline.o -o serpent
bignum.o : bignum.cpp bignum.h
opcodes.o : opcodes.cpp opcodes.h
util.o : util.cpp util.h bignum.o
tokenize.o : tokenize.cpp tokenize.h util.o
lllparser.o : lllparser.cpp lllparser.h tokenize.o util.o
parser.o : parser.cpp parser.h tokenize.o util.o
rewriter.o : rewriter.cpp rewriter.h lllparser.o util.o rewriteutils.o preprocess.o opcodes.o functions.o
preprocessor.o: rewriteutils.o functions.o
compiler.o : compiler.cpp compiler.h util.o
funcs.o : funcs.cpp funcs.h
cmdline.o: cmdline.cpp
pyext.o: pyext.cpp
clean:
rm -f serpent *\.o libserpent.a libserpent.so
install:
cp serpent /usr/local/bin
cp libserpent.a /usr/local/lib
cp libserpent.so /usr/local/lib
rm -rf /usr/local/include/libserpent
mkdir -p /usr/local/include/libserpent
cp $(HEADERS) /usr/local/include/libserpent

3
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/README.md generated vendored
View File

@ -1,3 +0,0 @@
Installation:
```make && sudo make install```

112
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/bignum.cpp generated vendored
View File

@ -1,112 +0,0 @@
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "bignum.h"
//Integer to string conversion
std::string unsignedToDecimal(unsigned branch) {
if (branch < 10) return nums.substr(branch, 1);
else return unsignedToDecimal(branch / 10) + nums.substr(branch % 10,1);
}
//Add two strings representing decimal values
std::string decimalAdd(std::string a, std::string b) {
std::string o = a;
while (b.length() < a.length()) b = "0" + b;
while (o.length() < b.length()) o = "0" + o;
bool carry = false;
for (int i = o.length() - 1; i >= 0; i--) {
o[i] = o[i] + b[i] - '0';
if (carry) o[i]++;
if (o[i] > '9') {
o[i] -= 10;
carry = true;
}
else carry = false;
}
if (carry) o = "1" + o;
return o;
}
//Helper function for decimalMul
std::string decimalDigitMul(std::string a, int dig) {
if (dig == 0) return "0";
else return decimalAdd(a, decimalDigitMul(a, dig - 1));
}
//Multiply two strings representing decimal values
std::string decimalMul(std::string a, std::string b) {
std::string o = "0";
for (unsigned i = 0; i < b.length(); i++) {
std::string n = decimalDigitMul(a, b[i] - '0');
if (n != "0") {
for (unsigned j = i + 1; j < b.length(); j++) n += "0";
}
o = decimalAdd(o, n);
}
return o;
}
//Modexp
std::string decimalModExp(std::string b, std::string e, std::string m) {
if (e == "0") return "1";
else if (e == "1") return b;
else if (decimalMod(e, "2") == "0") {
std::string o = decimalModExp(b, decimalDiv(e, "2"), m);
return decimalMod(decimalMul(o, o), m);
}
else {
std::string o = decimalModExp(b, decimalDiv(e, "2"), m);
return decimalMod(decimalMul(decimalMul(o, o), b), m);
}
}
//Is a greater than b? Flag allows equality
bool decimalGt(std::string a, std::string b, bool eqAllowed) {
if (a == b) return eqAllowed;
return (a.length() > b.length()) || (a.length() >= b.length() && a > b);
}
//Subtract the two strings representing decimal values
std::string decimalSub(std::string a, std::string b) {
if (b == "0") return a;
if (b == a) return "0";
while (b.length() < a.length()) b = "0" + b;
std::string c = b;
for (unsigned i = 0; i < c.length(); i++) c[i] = '0' + ('9' - c[i]);
std::string o = decimalAdd(decimalAdd(a, c).substr(1), "1");
while (o.size() > 1 && o[0] == '0') o = o.substr(1);
return o;
}
//Divide the two strings representing decimal values
std::string decimalDiv(std::string a, std::string b) {
std::string c = b;
if (decimalGt(c, a)) return "0";
int zeroes = -1;
while (decimalGt(a, c, true)) {
zeroes += 1;
c = c + "0";
}
c = c.substr(0, c.size() - 1);
std::string quot = "0";
while (decimalGt(a, c, true)) {
a = decimalSub(a, c);
quot = decimalAdd(quot, "1");
}
for (int i = 0; i < zeroes; i++) quot += "0";
return decimalAdd(quot, decimalDiv(a, b));
}
//Modulo the two strings representing decimal values
std::string decimalMod(std::string a, std::string b) {
return decimalSub(a, decimalMul(decimalDiv(a, b), b));
}
//String to int conversion
unsigned decimalToUnsigned(std::string a) {
if (a.size() == 0) return 0;
else return (a[a.size() - 1] - '0')
+ decimalToUnsigned(a.substr(0,a.size()-1)) * 10;
}

41
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/bignum.h generated vendored
View File

@ -1,41 +0,0 @@
#ifndef ETHSERP_BIGNUM
#define ETHSERP_BIGNUM
const std::string nums = "0123456789";
const std::string tt256 =
"115792089237316195423570985008687907853269984665640564039457584007913129639936"
;
const std::string tt256m1 =
"115792089237316195423570985008687907853269984665640564039457584007913129639935"
;
const std::string tt255 =
"57896044618658097711785492504343953926634992332820282019728792003956564819968";
const std::string tt176 =
"95780971304118053647396689196894323976171195136475136";
std::string unsignedToDecimal(unsigned branch);
std::string decimalAdd(std::string a, std::string b);
std::string decimalMul(std::string a, std::string b);
std::string decimalSub(std::string a, std::string b);
std::string decimalDiv(std::string a, std::string b);
std::string decimalMod(std::string a, std::string b);
std::string decimalModExp(std::string b, std::string e, std::string m);
bool decimalGt(std::string a, std::string b, bool eqAllowed=false);
unsigned decimalToUnsigned(std::string a);
#define utd unsignedToDecimal
#define dtu decimalToUnsigned
#endif

132
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/cmdline.cpp generated vendored
View File

@ -1,132 +0,0 @@
#include <stdio.h>
#include <string>
#include <iostream>
#include <vector>
#include <map>
#include "funcs.h"
int main(int argv, char** argc) {
if (argv == 1) {
std::cerr << "Must provide a command and arguments! Try parse, rewrite, compile, assemble\n";
return 0;
}
if (argv == 2 && std::string(argc[1]) == "--help" || std::string(argc[1]) == "-h" ) {
std::cout << argc[1] << "\n";
std::cout << "serpent command input\n";
std::cout << "where input -s for from stdin, a file, or interpreted as serpent code if does not exist as file.";
std::cout << "where command: \n";
std::cout << " parse: Just parses and returns s-expression code.\n";
std::cout << " rewrite: Parse, use rewrite rules print s-expressions of result.\n";
std::cout << " compile: Return resulting compiled EVM code in hex.\n";
std::cout << " assemble: Return result from step before compilation.\n";
return 0;
}
std::string flag = "";
std::string command = argc[1];
std::string input;
std::string secondInput;
if (std::string(argc[1]) == "-s") {
flag = command.substr(1);
command = argc[2];
input = "";
std::string line;
while (std::getline(std::cin, line)) {
input += line + "\n";
}
secondInput = argv == 3 ? "" : argc[3];
}
else {
if (argv == 2) {
std::cerr << "Not enough arguments for serpent cmdline\n";
throw(0);
}
input = argc[2];
secondInput = argv == 3 ? "" : argc[3];
}
bool haveSec = secondInput.length() > 0;
if (command == "parse" || command == "parse_serpent") {
std::cout << printAST(parseSerpent(input), haveSec) << "\n";
}
else if (command == "rewrite") {
std::cout << printAST(rewrite(parseLLL(input, true)), haveSec) << "\n";
}
else if (command == "compile_to_lll") {
std::cout << printAST(compileToLLL(input), haveSec) << "\n";
}
else if (command == "rewrite_chunk") {
std::cout << printAST(rewriteChunk(parseLLL(input, true)), haveSec) << "\n";
}
else if (command == "compile_chunk_to_lll") {
std::cout << printAST(compileChunkToLLL(input), haveSec) << "\n";
}
else if (command == "build_fragtree") {
std::cout << printAST(buildFragmentTree(parseLLL(input, true))) << "\n";
}
else if (command == "compile_lll") {
std::cout << binToHex(compileLLL(parseLLL(input, true))) << "\n";
}
else if (command == "dereference") {
std::cout << printAST(dereference(parseLLL(input, true)), haveSec) <<"\n";
}
else if (command == "pretty_assemble") {
std::cout << printTokens(prettyAssemble(parseLLL(input, true))) <<"\n";
}
else if (command == "pretty_compile_lll") {
std::cout << printTokens(prettyCompileLLL(parseLLL(input, true))) << "\n";
}
else if (command == "pretty_compile") {
std::cout << printTokens(prettyCompile(input)) << "\n";
}
else if (command == "pretty_compile_chunk") {
std::cout << printTokens(prettyCompileChunk(input)) << "\n";
}
else if (command == "assemble") {
std::cout << assemble(parseLLL(input, true)) << "\n";
}
else if (command == "serialize") {
std::cout << binToHex(serialize(tokenize(input, Metadata(), false))) << "\n";
}
else if (command == "flatten") {
std::cout << printTokens(flatten(parseLLL(input, true))) << "\n";
}
else if (command == "deserialize") {
std::cout << printTokens(deserialize(hexToBin(input))) << "\n";
}
else if (command == "compile") {
std::cout << binToHex(compile(input)) << "\n";
}
else if (command == "compile_chunk") {
std::cout << binToHex(compileChunk(input)) << "\n";
}
else if (command == "encode_datalist") {
std::vector<Node> tokens = tokenize(input);
std::vector<std::string> o;
for (int i = 0; i < (int)tokens.size(); i++) {
o.push_back(tokens[i].val);
}
std::cout << binToHex(encodeDatalist(o)) << "\n";
}
else if (command == "decode_datalist") {
std::vector<std::string> o = decodeDatalist(hexToBin(input));
std::vector<Node> tokens;
for (int i = 0; i < (int)o.size(); i++)
tokens.push_back(token(o[i]));
std::cout << printTokens(tokens) << "\n";
}
else if (command == "tokenize") {
std::cout << printTokens(tokenize(input));
}
else if (command == "biject") {
if (argv == 3)
std::cerr << "Not enough arguments for biject\n";
int pos = decimalToUnsigned(secondInput);
std::vector<Node> n = prettyCompile(input);
if (pos >= (int)n.size())
std::cerr << "Code position too high\n";
Metadata m = n[pos].metadata;
std::cout << "Opcode: " << n[pos].val << ", file: " << m.file <<
", line: " << m.ln << ", char: " << m.ch << "\n";
}
}

554
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/compiler.cpp generated vendored
View File

@ -1,554 +0,0 @@
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
#include "bignum.h"
#include "opcodes.h"
struct programAux {
std::map<std::string, std::string> vars;
int nextVarMem;
bool allocUsed;
bool calldataUsed;
int step;
int labelLength;
};
struct programVerticalAux {
int height;
std::string innerScopeName;
std::map<std::string, int> dupvars;
std::map<std::string, int> funvars;
std::vector<mss> scopes;
};
struct programData {
programAux aux;
Node code;
int outs;
};
programAux Aux() {
programAux o;
o.allocUsed = false;
o.calldataUsed = false;
o.step = 0;
o.nextVarMem = 32;
return o;
}
programVerticalAux verticalAux() {
programVerticalAux o;
o.height = 0;
o.dupvars = std::map<std::string, int>();
o.funvars = std::map<std::string, int>();
o.scopes = std::vector<mss>();
return o;
}
programData pd(programAux aux = Aux(), Node code=token("_"), int outs=0) {
programData o;
o.aux = aux;
o.code = code;
o.outs = outs;
return o;
}
Node multiToken(Node nodes[], int len, Metadata met) {
std::vector<Node> out;
for (int i = 0; i < len; i++) {
out.push_back(nodes[i]);
}
return astnode("_", out, met);
}
Node finalize(programData c);
Node popwrap(Node node) {
Node nodelist[] = {
node,
token("POP", node.metadata)
};
return multiToken(nodelist, 2, node.metadata);
}
// Grabs variables
mss getVariables(Node node, mss cur=mss()) {
Metadata m = node.metadata;
// Tokens don't contain any variables
if (node.type == TOKEN)
return cur;
// Don't descend into call fragments
else if (node.val == "lll")
return getVariables(node.args[1], cur);
// At global scope get/set/ref also declare
else if (node.val == "get" || node.val == "set" || node.val == "ref") {
if (node.args[0].type != TOKEN)
err("Variable name must be simple token,"
" not complex expression!", m);
if (!cur.count(node.args[0].val)) {
cur[node.args[0].val] = utd(cur.size() * 32 + 32);
//std::cerr << node.args[0].val << " " << cur[node.args[0].val] << "\n";
}
}
// Recursively process children
for (unsigned i = 0; i < node.args.size(); i++) {
cur = getVariables(node.args[i], cur);
}
return cur;
}
// Turns LLL tree into tree of code fragments
programData opcodeify(Node node,
programAux aux=Aux(),
programVerticalAux vaux=verticalAux()) {
std::string symb = "_"+mkUniqueToken();
Metadata m = node.metadata;
// Get variables
if (!aux.vars.size()) {
aux.vars = getVariables(node);
aux.nextVarMem = aux.vars.size() * 32 + 32;
}
// Numbers
if (node.type == TOKEN) {
return pd(aux, nodeToNumeric(node), 1);
}
else if (node.val == "ref" || node.val == "get" || node.val == "set") {
std::string varname = node.args[0].val;
// Determine reference to variable
Node varNode = tkn(aux.vars[varname], m);
//std::cerr << varname << " " << printSimple(varNode) << "\n";
// Set variable
if (node.val == "set") {
programData sub = opcodeify(node.args[1], aux, vaux);
if (!sub.outs)
err("Value to set variable must have nonzero arity!", m);
// What if we are setting a stack variable?
if (vaux.dupvars.count(node.args[0].val)) {
int h = vaux.height - vaux.dupvars[node.args[0].val];
if (h > 16) err("Too deep for stack variable (max 16)", m);
Node nodelist[] = {
sub.code,
token("SWAP"+unsignedToDecimal(h), m),
token("POP", m)
};
return pd(sub.aux, multiToken(nodelist, 3, m), 0);
}
// Setting a memory variable
else {
Node nodelist[] = {
sub.code,
varNode,
token("MSTORE", m),
};
return pd(sub.aux, multiToken(nodelist, 3, m), 0);
}
}
// Get variable
else if (node.val == "get") {
// Getting a stack variable
if (vaux.dupvars.count(node.args[0].val)) {
int h = vaux.height - vaux.dupvars[node.args[0].val];
if (h > 16) err("Too deep for stack variable (max 16)", m);
return pd(aux, token("DUP"+unsignedToDecimal(h)), 1);
}
// Getting a memory variable
else {
Node nodelist[] =
{ varNode, token("MLOAD", m) };
return pd(aux, multiToken(nodelist, 2, m), 1);
}
}
// Refer variable
else if (node.val == "ref") {
if (vaux.dupvars.count(node.args[0].val))
err("Cannot ref stack variable!", m);
return pd(aux, varNode, 1);
}
}
// Comments do nothing
else if (node.val == "comment") {
Node nodelist[] = { };
return pd(aux, multiToken(nodelist, 0, m), 0);
}
// Custom operation sequence
// eg. (ops bytez id msize swap1 msize add 0 swap1 mstore) == alloc
if (node.val == "ops") {
std::vector<Node> subs2;
int depth = 0;
for (unsigned i = 0; i < node.args.size(); i++) {
std::string op = upperCase(node.args[i].val);
if (node.args[i].type == ASTNODE || opinputs(op) == -1) {
programVerticalAux vaux2 = vaux;
vaux2.height = vaux.height - i - 1 + node.args.size();
programData sub = opcodeify(node.args[i], aux, vaux2);
aux = sub.aux;
depth += sub.outs;
subs2.push_back(sub.code);
}
else {
subs2.push_back(token(op, m));
depth += opoutputs(op) - opinputs(op);
}
}
if (depth < 0 || depth > 1) err("Stack depth mismatch", m);
return pd(aux, astnode("_", subs2, m), 0);
}
// Code blocks
if (node.val == "lll" && node.args.size() == 2) {
if (node.args[1].val != "0") aux.allocUsed = true;
std::vector<Node> o;
o.push_back(finalize(opcodeify(node.args[0])));
programData sub = opcodeify(node.args[1], aux, vaux);
Node code = astnode("____CODE", o, m);
Node nodelist[] = {
token("$begincode"+symb+".endcode"+symb, m), token("DUP1", m),
token("$begincode"+symb, m), sub.code, token("CODECOPY", m),
token("$endcode"+symb, m), token("JUMP", m),
token("~begincode"+symb, m), code,
token("~endcode"+symb, m), token("JUMPDEST", m)
};
return pd(sub.aux, multiToken(nodelist, 11, m), 1);
}
// Stack variables
if (node.val == "with") {
programData initial = opcodeify(node.args[1], aux, vaux);
programVerticalAux vaux2 = vaux;
vaux2.dupvars[node.args[0].val] = vaux.height;
vaux2.height += 1;
if (!initial.outs)
err("Initial variable value must have nonzero arity!", m);
programData sub = opcodeify(node.args[2], initial.aux, vaux2);
Node nodelist[] = {
initial.code,
sub.code
};
programData o = pd(sub.aux, multiToken(nodelist, 2, m), sub.outs);
if (sub.outs)
o.code.args.push_back(token("SWAP1", m));
o.code.args.push_back(token("POP", m));
return o;
}
// Seq of multiple statements
if (node.val == "seq") {
std::vector<Node> children;
int lastOut = 0;
for (unsigned i = 0; i < node.args.size(); i++) {
programData sub = opcodeify(node.args[i], aux, vaux);
aux = sub.aux;
if (sub.outs == 1) {
if (i < node.args.size() - 1) sub.code = popwrap(sub.code);
else lastOut = 1;
}
children.push_back(sub.code);
}
return pd(aux, astnode("_", children, m), lastOut);
}
// 2-part conditional (if gets rewritten to unless in rewrites)
else if (node.val == "unless" && node.args.size() == 2) {
programData cond = opcodeify(node.args[0], aux, vaux);
programData action = opcodeify(node.args[1], cond.aux, vaux);
aux = action.aux;
if (!cond.outs) err("Condition of if/unless statement has arity 0", m);
if (action.outs) action.code = popwrap(action.code);
Node nodelist[] = {
cond.code,
token("$endif"+symb, m), token("JUMPI", m),
action.code,
token("~endif"+symb, m), token("JUMPDEST", m)
};
return pd(aux, multiToken(nodelist, 6, m), 0);
}
// 3-part conditional
else if (node.val == "if" && node.args.size() == 3) {
programData ifd = opcodeify(node.args[0], aux, vaux);
programData thend = opcodeify(node.args[1], ifd.aux, vaux);
programData elsed = opcodeify(node.args[2], thend.aux, vaux);
aux = elsed.aux;
if (!ifd.outs)
err("Condition of if/unless statement has arity 0", m);
// Handle cases where one conditional outputs something
// and the other does not
int outs = (thend.outs && elsed.outs) ? 1 : 0;
if (thend.outs > outs) thend.code = popwrap(thend.code);
if (elsed.outs > outs) elsed.code = popwrap(elsed.code);
Node nodelist[] = {
ifd.code,
token("ISZERO", m),
token("$else"+symb, m), token("JUMPI", m),
thend.code,
token("$endif"+symb, m), token("JUMP", m),
token("~else"+symb, m), token("JUMPDEST", m),
elsed.code,
token("~endif"+symb, m), token("JUMPDEST", m)
};
return pd(aux, multiToken(nodelist, 12, m), outs);
}
// While (rewritten to this in rewrites)
else if (node.val == "until") {
programData cond = opcodeify(node.args[0], aux, vaux);
programData action = opcodeify(node.args[1], cond.aux, vaux);
aux = action.aux;
if (!cond.outs)
err("Condition of while/until loop has arity 0", m);
if (action.outs) action.code = popwrap(action.code);
Node nodelist[] = {
token("~beg"+symb, m), token("JUMPDEST", m),
cond.code,
token("$end"+symb, m), token("JUMPI", m),
action.code,
token("$beg"+symb, m), token("JUMP", m),
token("~end"+symb, m), token("JUMPDEST", m),
};
return pd(aux, multiToken(nodelist, 10, m));
}
// Memory allocations
else if (node.val == "alloc") {
programData bytez = opcodeify(node.args[0], aux, vaux);
aux = bytez.aux;
if (!bytez.outs)
err("Alloc input has arity 0", m);
aux.allocUsed = true;
Node nodelist[] = {
bytez.code,
token("MSIZE", m), token("SWAP1", m), token("MSIZE", m),
token("ADD", m),
token("0", m), token("SWAP1", m), token("MSTORE", m)
};
return pd(aux, multiToken(nodelist, 8, m), 1);
}
// All other functions/operators
else {
std::vector<Node> subs2;
int depth = opinputs(upperCase(node.val));
if (depth == -1)
err("Not a function or opcode: "+node.val, m);
if ((int)node.args.size() != depth)
err("Invalid arity for "+node.val, m);
for (int i = node.args.size() - 1; i >= 0; i--) {
programVerticalAux vaux2 = vaux;
vaux2.height = vaux.height - i - 1 + node.args.size();
programData sub = opcodeify(node.args[i], aux, vaux2);
aux = sub.aux;
if (!sub.outs)
err("Input "+unsignedToDecimal(i)+" has arity 0", sub.code.metadata);
subs2.push_back(sub.code);
}
subs2.push_back(token(upperCase(node.val), m));
int outdepth = opoutputs(upperCase(node.val));
return pd(aux, astnode("_", subs2, m), outdepth);
}
}
// Adds necessary wrappers to a program
Node finalize(programData c) {
std::vector<Node> bottom;
Metadata m = c.code.metadata;
// If we are using both alloc and variables, we need to pre-zfill
// some memory
if ((c.aux.allocUsed || c.aux.calldataUsed) && c.aux.vars.size() > 0) {
Node nodelist[] = {
token("0", m),
token(unsignedToDecimal(c.aux.nextVarMem - 1)),
token("MSTORE8", m)
};
bottom.push_back(multiToken(nodelist, 3, m));
}
// The actual code
bottom.push_back(c.code);
return astnode("_", bottom, m);
}
//LLL -> code fragment tree
Node buildFragmentTree(Node node) {
return finalize(opcodeify(node));
}
// Builds a dictionary mapping labels to variable names
programAux buildDict(Node program, programAux aux, int labelLength) {
Metadata m = program.metadata;
// Token
if (program.type == TOKEN) {
if (isNumberLike(program)) {
aux.step += 1 + toByteArr(program.val, m).size();
}
else if (program.val[0] == '~') {
aux.vars[program.val.substr(1)] = unsignedToDecimal(aux.step);
}
else if (program.val[0] == '$') {
aux.step += labelLength + 1;
}
else aux.step += 1;
}
// A sub-program (ie. LLL)
else if (program.val == "____CODE") {
programAux auks = Aux();
for (unsigned i = 0; i < program.args.size(); i++) {
auks = buildDict(program.args[i], auks, labelLength);
}
for (std::map<std::string,std::string>::iterator it=auks.vars.begin();
it != auks.vars.end();
it++) {
aux.vars[(*it).first] = (*it).second;
}
aux.step += auks.step;
}
// Normal sub-block
else {
for (unsigned i = 0; i < program.args.size(); i++) {
aux = buildDict(program.args[i], aux, labelLength);
}
}
return aux;
}
// Applies that dictionary
Node substDict(Node program, programAux aux, int labelLength) {
Metadata m = program.metadata;
std::vector<Node> out;
std::vector<Node> inner;
if (program.type == TOKEN) {
if (program.val[0] == '$') {
std::string tokStr = "PUSH"+unsignedToDecimal(labelLength);
out.push_back(token(tokStr, m));
int dotLoc = program.val.find('.');
if (dotLoc == -1) {
std::string val = aux.vars[program.val.substr(1)];
inner = toByteArr(val, m, labelLength);
}
else {
std::string start = aux.vars[program.val.substr(1, dotLoc-1)],
end = aux.vars[program.val.substr(dotLoc + 1)],
dist = decimalSub(end, start);
inner = toByteArr(dist, m, labelLength);
}
out.push_back(astnode("_", inner, m));
}
else if (program.val[0] == '~') { }
else if (isNumberLike(program)) {
inner = toByteArr(program.val, m);
out.push_back(token("PUSH"+unsignedToDecimal(inner.size())));
out.push_back(astnode("_", inner, m));
}
else return program;
}
else {
for (unsigned i = 0; i < program.args.size(); i++) {
Node n = substDict(program.args[i], aux, labelLength);
if (n.type == TOKEN || n.args.size()) out.push_back(n);
}
}
return astnode("_", out, m);
}
// Compiled fragtree -> compiled fragtree without labels
Node dereference(Node program) {
int sz = treeSize(program) * 4;
int labelLength = 1;
while (sz >= 256) { labelLength += 1; sz /= 256; }
programAux aux = buildDict(program, Aux(), labelLength);
return substDict(program, aux, labelLength);
}
// Dereferenced fragtree -> opcodes
std::vector<Node> flatten(Node derefed) {
std::vector<Node> o;
if (derefed.type == TOKEN) {
o.push_back(derefed);
}
else {
for (unsigned i = 0; i < derefed.args.size(); i++) {
std::vector<Node> oprime = flatten(derefed.args[i]);
for (unsigned j = 0; j < oprime.size(); j++) o.push_back(oprime[j]);
}
}
return o;
}
// Opcodes -> bin
std::string serialize(std::vector<Node> codons) {
std::string o;
for (unsigned i = 0; i < codons.size(); i++) {
int v;
if (isNumberLike(codons[i])) {
v = decimalToUnsigned(codons[i].val);
}
else if (codons[i].val.substr(0,4) == "PUSH") {
v = 95 + decimalToUnsigned(codons[i].val.substr(4));
}
else {
v = opcode(codons[i].val);
}
o += (char)v;
}
return o;
}
// Bin -> opcodes
std::vector<Node> deserialize(std::string ser) {
std::vector<Node> o;
int backCount = 0;
for (unsigned i = 0; i < ser.length(); i++) {
unsigned char v = (unsigned char)ser[i];
std::string oper = op((int)v);
if (oper != "" && backCount <= 0) o.push_back(token(oper));
else if (v >= 96 && v < 128 && backCount <= 0) {
o.push_back(token("PUSH"+unsignedToDecimal(v - 95)));
}
else o.push_back(token(unsignedToDecimal(v)));
if (v >= 96 && v < 128 && backCount <= 0) {
backCount = v - 95;
}
else backCount--;
}
return o;
}
// Fragtree -> bin
std::string assemble(Node fragTree) {
return serialize(flatten(dereference(fragTree)));
}
// Fragtree -> tokens
std::vector<Node> prettyAssemble(Node fragTree) {
return flatten(dereference(fragTree));
}
// LLL -> bin
std::string compileLLL(Node program) {
return assemble(buildFragmentTree(program));
}
// LLL -> tokens
std::vector<Node> prettyCompileLLL(Node program) {
return prettyAssemble(buildFragmentTree(program));
}
// Converts a list of integer values to binary transaction data
std::string encodeDatalist(std::vector<std::string> vals) {
std::string o;
for (unsigned i = 0; i < vals.size(); i++) {
std::vector<Node> n = toByteArr(strToNumeric(vals[i]), Metadata(), 32);
for (unsigned j = 0; j < n.size(); j++) {
int v = decimalToUnsigned(n[j].val);
o += (char)v;
}
}
return o;
}
// Converts binary transaction data into a list of integer values
std::vector<std::string> decodeDatalist(std::string ser) {
std::vector<std::string> out;
for (unsigned i = 0; i < ser.length(); i+= 32) {
std::string o = "0";
for (unsigned j = i; j < i + 32; j++) {
int vj = (int)(unsigned char)ser[j];
o = decimalAdd(decimalMul(o, "256"), unsignedToDecimal(vj));
}
out.push_back(o);
}
return out;
}

43
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/compiler.h generated vendored
View File

@ -1,43 +0,0 @@
#ifndef ETHSERP_COMPILER
#define ETHSERP_COMPILER
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
// Compiled fragtree -> compiled fragtree without labels
Node dereference(Node program);
// LLL -> fragtree
Node buildFragmentTree(Node program);
// Dereferenced fragtree -> opcodes
std::vector<Node> flatten(Node derefed);
// opcodes -> bin
std::string serialize(std::vector<Node> codons);
// Fragtree -> bin
std::string assemble(Node fragTree);
// Fragtree -> opcodes
std::vector<Node> prettyAssemble(Node fragTree);
// LLL -> bin
std::string compileLLL(Node program);
// LLL -> opcodes
std::vector<Node> prettyCompileLLL(Node program);
// bin -> opcodes
std::vector<Node> deserialize(std::string ser);
// Converts a list of integer values to binary transaction data
std::string encodeDatalist(std::vector<std::string> vals);
// Converts binary transaction data into a list of integer values
std::vector<std::string> decodeDatalist(std::string ser);
#endif

11
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/example.cpp generated vendored
View File

@ -1,11 +0,0 @@
#include <libserpent/funcs.h>
#include <libserpent/bignum.h>
#include <iostream>
using namespace std;
int main() {
cout << printAST(compileToLLL(get_file_contents("examples/namecoin.se"))) << "\n";
cout << decimalSub("10234", "10234") << "\n";
cout << decimalSub("10234", "10233") << "\n";
}

11
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/collatz.se generated vendored
View File

@ -1,11 +0,0 @@
x = msg.data[0]
steps = 0
while x > 1:
steps += 1
if (x % 2) == 0:
x /= 2
else:
x = 3 * x + 1
return(steps)

274
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/counterparty/counterparty.se generated vendored
View File

@ -1,274 +0,0 @@
# Ethereum forks Counterparty in 340 lines of serpent
# Not yet tested
# assets[i] = a registered asset, assets[i].holders[j] = former or current i-holder
data assets[2^50](creator, name, calldate, callprice, dividend_paid, holders[2^50], holdersCount)
data nextAssetId
# holdersMap: holdersMap[addr][asset] = 1 if addr holds asset
data holdersMap[2^160][2^50]
# balances[x][y] = how much of y x holds
data balances[2^160][2^50]
# orders[a][b] = heap of indices to (c, d, e)
# = c offers to sell d units of a at a price of e units of b per 10^18 units
# of a
data orderbooks[2^50][2^50]
# store of general order data
data orders[2^50](seller, asset_sold, quantity, price)
data ordersCount
# data feeds
data feeds[2^50](owner, value)
data feedCount
# heap
data heap
extern heap: [register, push, pop, top, size]
data cfds[2^50](maker, acceptor, feed, asset, strike, leverage, min, max, maturity)
data cfdCount
data bets[2^50](maker, acceptor, feed, asset, makerstake, acceptorstake, eqtest, maturity)
data betCount
def init():
heap = create('heap.se')
# Add units (internal method)
def add(to, asset, value):
assert msg.sender == self
self.balances[to][asset] += value
# Add the holder to the holders list
if not self.holdersMap[to][asset]:
self.holdersMap[to][asset] = 1
c = self.assets[asset].holdersCount
self.assets[asset].holders[c] = to
self.assets[asset].holdersCount = c + 1
# Register a new asset
def register_asset(q, name, calldate, callprice):
newid = self.nextAssetId
self.assets[newid].creator = msg.sender
self.assets[newid].name = name
self.assets[newid].calldate = calldate
self.assets[newid].callprice = callprice
self.assets[newid].holders[0] = msg.sender
self.assets[newid].holdersCount = 1
self.balances[msg.sender][newid] = q
self.holdersMap[msg.sender][newid] = 1
# Send
def send(to, asset, value):
fromval = self.balances[msg.sender][asset]
if fromval >= value:
self.balances[msg.sender][asset] -= value
self.add(to, asset, value)
# Order
def mkorder(selling, buying, quantity, price):
# Make sure you have enough to pay for the order
assert self.balances[msg.sender][selling] >= quantity:
# Try to match existing orders
o = orderbooks[buying][selling]
if not o:
o = self.heap.register()
orderbooks[selling][buying] = o
sz = self.heap.size(o)
invprice = 10^36 / price
while quantity > 0 and sz > 0:
orderid = self.heap.pop()
p = self.orders[orderid].price
if p > invprice:
sz = 0
else:
q = self.orders[orderid].quantity
oq = min(q, quantity)
b = self.orders[orderid].seller
self.balances[msg.sender][selling] -= oq * p / 10^18
self.add(msg.sender, buying, oq)
self.add(b, selling, oq * p / 10^18)
self.orders[orderid].quantity = q - oq
if oq == q:
self.orders[orderid].seller = 0
self.orders[orderid].price = 0
self.orders[orderid].asset_sold = 0
quantity -= oq
sz -= 1
assert quantity > 0
# Make the order
c = self.ordersCount
self.orders[c].seller = msg.sender
self.orders[c].asset_sold = selling
self.orders[c].quantity = quantity
self.orders[c].price = price
self.ordersCount += 1
# Add it to the heap
o = orderbooks[selling][buying]
if not o:
o = self.heap.register()
orderbooks[selling][buying] = o
self.balances[msg.sender][selling] -= quantity
self.heap.push(o, price, c)
return(c)
def cancel_order(id):
if self.orders[id].seller == msg.sender:
self.orders[id].seller = 0
self.orders[id].price = 0
self.balances[msg.sender][self.orders[id].asset_sold] += self.orders[id].quantity
self.orders[id].quantity = 0
self.orders[id].asset_sold = 0
def register_feed():
c = self.feedCount
self.feeds[c].owner = msg.sender
self.feedCount = c + 1
return(c)
def set_feed(id, v):
if self.feeds[id].owner == msg.sender:
self.feeds[id].value = v
def mk_cfd_offer(feed, asset, strike, leverage, min, max, maturity):
b = self.balances[msg.sender][asset]
req = max((strike - min) * leverage, (strike - max) * leverage)
assert b >= req
self.balances[msg.sender][asset] = b - req
c = self.cfdCount
self.cfds[c].maker = msg.sender
self.cfds[c].feed = feed
self.cfds[c].asset = asset
self.cfds[c].strike = strike
self.cfds[c].leverage = leverage
self.cfds[c].min = min
self.cfds[c].max = max
self.cfds[c].maturity = maturity
self.cfdCount = c + 1
return(c)
def accept_cfd_offer(c):
assert not self.cfds[c].acceptor and self.cfds[c].maker
asset = self.cfds[c].asset
strike = self.cfds[c].strike
min = self.cfds[c].min
max = self.cfds[c].max
leverage = self.cfds[c].leverage
b = self.balances[msg.sender][asset]
req = max((min - strike) * leverage, (max - strike) * leverage)
assert b >= req
self.balances[msg.sender][asset] = b - req
self.cfds[c].acceptor = msg.sender
self.cfds[c].maturity += block.timestamp
def claim_cfd_offer(c):
asset = self.cfds[c].asset
strike = self.cfds[c].strike
min = self.cfds[c].min
max = self.cfds[c].max
leverage = self.cfds[c].leverage
v = self.feeds[self.cfds[c].feed].value
assert v <= min or v >= max or block.timestamp >= self.cfds[c].maturity
maker_req = max((strike - min) * leverage, (strike - max) * leverage)
acceptor_req = max((min - strike) * leverage, (max - strike) * leverage)
paydelta = (strike - v) * leverage
self.add(self.cfds[c].maker, asset, maker_req + paydelta)
self.add(self.cfds[c].acceptor, asset, acceptor_req - paydelta)
self.cfds[c].maker = 0
self.cfds[c].acceptor = 0
self.cfds[c].feed = 0
self.cfds[c].asset = 0
self.cfds[c].strike = 0
self.cfds[c].leverage = 0
self.cfds[c].min = 0
self.cfds[c].max = 0
self.cfds[c].maturity = 0
def withdraw_cfd_offer(c):
if self.cfds[c].maker == msg.sender and not self.cfds[c].acceptor:
asset = self.cfds[c].asset
strike = self.cfds[c].strike
min = self.cfds[c].min
max = self.cfds[c].max
leverage = self.cfds[c].leverage
maker_req = max((strike - min) * leverage, (strike - max) * leverage)
self.balances[self.cfds[c].maker][asset] += maker_req
self.cfds[c].maker = 0
self.cfds[c].acceptor = 0
self.cfds[c].feed = 0
self.cfds[c].asset = 0
self.cfds[c].strike = 0
self.cfds[c].leverage = 0
self.cfds[c].min = 0
self.cfds[c].max = 0
self.cfds[c].maturity = 0
def mk_bet_offer(feed, asset, makerstake, acceptorstake, eqtest, maturity):
assert self.balances[msg.sender][asset] >= makerstake
c = self.betCount
self.bets[c].maker = msg.sender
self.bets[c].feed = feed
self.bets[c].asset = asset
self.bets[c].makerstake = makerstake
self.bets[c].acceptorstake = acceptorstake
self.bets[c].eqtest = eqtest
self.bets[c].maturity = maturity
self.balances[msg.sender][asset] -= makerstake
self.betCount = c + 1
return(c)
def accept_bet_offer(c):
assert self.bets[c].maker and not self.bets[c].acceptor
asset = self.bets[c].asset
acceptorstake = self.bets[c].acceptorstake
assert self.balances[msg.sender][asset] >= acceptorstake
self.balances[msg.sender][asset] -= acceptorstake
self.bets[c].acceptor = msg.sender
def claim_bet_offer(c):
assert block.timestamp >= self.bets[c].maturity
v = self.feeds[self.bets[c].feed].value
totalstake = self.bets[c].makerstake + self.bets[c].acceptorstake
if v == self.bets[c].eqtest:
self.add(self.bets[c].maker, self.bets[c].asset, totalstake)
else:
self.add(self.bets[c].acceptor, self.bets[c].asset, totalstake)
self.bets[c].maker = 0
self.bets[c].feed = 0
self.bets[c].asset = 0
self.bets[c].makerstake = 0
self.bets[c].acceptorstake = 0
self.bets[c].eqtest = 0
self.bets[c].maturity = 0
def cancel_bet(c):
assert not self.bets[c].acceptor and msg.sender == self.bets[c].maker
self.balances[msg.sender][self.bets[c].asset] += self.bets[c].makerstake
self.bets[c].maker = 0
self.bets[c].feed = 0
self.bets[c].asset = 0
self.bets[c].makerstake = 0
self.bets[c].acceptorstake = 0
self.bets[c].eqtest = 0
self.bets[c].maturity = 0
def dividend(holder_asset, divvying_asset, ratio):
i = 0
sz = self.assets[holder_asset].holdersCount
t = 0
holders = array(sz)
payments = array(sz)
while i < sz:
holders[i] = self.assets[holder_asset].holders[i]
payments[i] = self.balances[holders[i]][holder_asset] * ratio / 10^18
t += payments[i]
i += 1
if self.balances[msg.sender][divvying_asset] >= t:
i = 0
while i < sz:
self.add(holders[i], divvying_asset, payments[i])
i += 1
self.balances[msg.sender][divvying_asset] -= t

69
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/counterparty/heap.se generated vendored
View File

@ -1,69 +0,0 @@
data heaps[2^50](owner, size, nodes[2^50](key, value))
data heapIndex
def register():
i = self.heapIndex
self.heaps[i].owner = msg.sender
self.heapIndex = i + 1
return(i)
def push(heap, key, value):
assert msg.sender == self.heaps[heap].owner
sz = self.heaps[heap].size
self.heaps[heap].nodes[sz].key = key
self.heaps[heap].nodes[sz].value = value
k = sz + 1
while k > 1:
bottom = self.heaps[heap].nodes[k].key
top = self.heaps[heap].nodes[k/2].key
if bottom < top:
tvalue = self.heaps[heap].nodes[k/2].value
bvalue = self.heaps[heap].nodes[k].value
self.heaps[heap].nodes[k].key = top
self.heaps[heap].nodes[k].value = tvalue
self.heaps[heap].nodes[k/2].key = bottom
self.heaps[heap].nodes[k/2].value = bvalue
k /= 2
else:
k = 0
self.heaps[heap].size = sz + 1
def pop(heap):
sz = self.heaps[heap].size
assert sz
prevtop = self.heaps[heap].nodes[1].value
self.heaps[heap].nodes[1].key = self.heaps[heap].nodes[sz].key
self.heaps[heap].nodes[1].value = self.heaps[heap].nodes[sz].value
self.heaps[heap].nodes[sz].key = 0
self.heaps[heap].nodes[sz].value = 0
top = self.heaps[heap].nodes[1].key
k = 1
while k * 2 < sz:
bottom1 = self.heaps[heap].nodes[k * 2].key
bottom2 = self.heaps[heap].nodes[k * 2 + 1].key
if bottom1 < top and (bottom1 < bottom2 or k * 2 + 1 >= sz):
tvalue = self.heaps[heap].nodes[1].value
bvalue = self.heaps[heap].nodes[k * 2].value
self.heaps[heap].nodes[k].key = bottom1
self.heaps[heap].nodes[k].value = bvalue
self.heaps[heap].nodes[k * 2].key = top
self.heaps[heap].nodes[k * 2].value = tvalue
k = k * 2
elif bottom2 < top and bottom2 < bottom1 and k * 2 + 1 < sz:
tvalue = self.heaps[heap].nodes[1].value
bvalue = self.heaps[heap].nodes[k * 2 + 1].value
self.heaps[heap].nodes[k].key = bottom2
self.heaps[heap].nodes[k].value = bvalue
self.heaps[heap].nodes[k * 2 + 1].key = top
self.heaps[heap].nodes[k * 2 + 1].value = tvalue
k = k * 2 + 1
else:
k = sz
self.heaps[heap].size = sz - 1
return(prevtop)
def top(heap):
return(self.heaps[heap].nodes[1].value)
def size(heap):
return(self.heaps[heap].size)

53
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/crowdfund.se generated vendored
View File

@ -1,53 +0,0 @@
data campaigns[2^80](recipient, goal, deadline, contrib_total, contrib_count, contribs[2^50](sender, value))
def create_campaign(id, recipient, goal, timelimit):
if self.campaigns[id].recipient:
return(0)
self.campaigns[id].recipient = recipient
self.campaigns[id].goal = goal
self.campaigns[id].deadline = block.timestamp + timelimit
def contribute(id):
# Update contribution total
total_contributed = self.campaigns[id].contrib_total + msg.value
self.campaigns[id].contrib_total = total_contributed
# Record new contribution
sub_index = self.campaigns[id].contrib_count
self.campaigns[id].contribs[sub_index].sender = msg.sender
self.campaigns[id].contribs[sub_index].value = msg.value
self.campaigns[id].contrib_count = sub_index + 1
# Enough funding?
if total_contributed >= self.campaigns[id].goal:
send(self.campaigns[id].recipient, total_contributed)
self.clear(id)
return(1)
# Expired?
if block.timestamp > self.campaigns[id].deadline:
i = 0
c = self.campaigns[id].contrib_count
while i < c:
send(self.campaigns[id].contribs[i].sender, self.campaigns[id].contribs[i].value)
i += 1
self.clear(id)
return(2)
def progress_report(id):
return(self.campaigns[id].contrib_total)
# Clearing function for internal use
def clear(id):
if self == msg.sender:
self.campaigns[id].recipient = 0
self.campaigns[id].goal = 0
self.campaigns[id].deadline = 0
c = self.campaigns[id].contrib_count
self.campaigns[id].contrib_count = 0
self.campaigns[id].contrib_total = 0
i = 0
while i < c:
self.campaigns[id].contribs[i].sender = 0
self.campaigns[id].contribs[i].value = 0
i += 1

136
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/futarchy.se generated vendored
View File

@ -1,136 +0,0 @@
# 0: current epoch
# 1: number of proposals
# 2: master currency
# 3: last winning market
# 4: last txid
# 5: long-term ema currency units purchased
# 6: last block when currency units purchased
# 7: ether allocated to last round
# 8: last block when currency units claimed
# 9: ether allocated to current round
# 1000+: [proposal address, market ID, totprice, totvolume]
init:
# We technically have two levels of epoch here. We have
# one epoch of 1000, to synchronize with the 1000 epoch
# of the market, and then 100 of those epochs make a
# meta-epoch (I'll nominate the term "seculum") over
# which the futarchy protocol will take place
contract.storage[0] = block.number / 1000
# The master currency of the futarchy. The futarchy will
# assign currency units to whoever the prediction market
# thinks will best increase the currency's value
master_currency = create('subcurrency.se')
contract.storage[2] = master_currency
code:
curepoch = block.number / 1000
prevepoch = contract.storage[0]
if curepoch > prevepoch:
if (curepoch % 100) > 50:
# Collect price data
# We take an average over 50 subepochs to determine
# the price of each asset, weighting by volume to
# prevent abuse
contract.storage[0] = curepoch
i = 0
numprop = contract.storage[1]
while i < numprop:
market = contract.storage[1001 + i * 4]
price = call(market, 2)
volume = call(market, 3)
contract.storage[1002 + i * 4] += price
contract.storage[1003 + i * 4] += volume * price
i += 1
if (curepoch / 100) > (prevepoch / 100):
# If we are entering a new seculum, we determine the
# market with the highest total average price
best = 0
bestmarket = 0
besti = 0
i = 0
while i < numprop:
curtotprice = contract.storage[1002 + i * 4]
curvolume = contract.storage[1002 + i * 4]
curavgprice = curtotprice / curvolume
if curavgprice > best:
best = curavgprice
besti = i
bestmarket = contract.storage[1003 + i * 4]
i += 1
# Reset the number of proposals to 0
contract.storage[1] = 0
# Reward the highest proposal
call(contract.storage[2], [best, 10^9, 0], 3)
# Record the winning market so we can later appropriately
# compensate the participants
contract.storage[2] = bestmarket
# The amount of ether allocated to the last round
contract.storage[7] = contract.storage[9]
# The amount of ether allocated to the next round
contract.storage[9] = contract.balance / 2
# Make a proposal [0, address]
if msg.data[0] == 0 and curepoch % 100 < 50:
pid = contract.storage[1]
market = create('market.se')
c1 = create('subcurrency.se')
c2 = create('subcurrency.se')
call(market, [c1, c2], 2)
contract.storage[1000 + pid * 4] = msg.data[1]
contract.storage[1001 + pid * 4] = market
contract.storage[1] += 1
# Claim ether [1, address]
# One unit of the first currency in the last round's winning
# market entitles you to a quantity of ether that was decided
# at the start of that epoch
elif msg.data[0] == 1:
first_subcurrency = call(contract.storage[2], 3)
# We ask the first subcurrency contract what the last transaction was. The
# way to make a claim is to send the amount of first currency units that
# you wish to claim with, and then immediately call this contract. For security
# it makes sense to set up a tx which sends both messages in sequence atomically
data = call(first_subcurrency, [], 0, 4)
from = data[0]
to = data[1]
value = data[2]
txid = data[3]
if txid > contract.storage[4] and to == contract.address:
send(to, contract.storage[7] * value / 10^9)
contract.storage[4] = txid
# Claim second currency [2, address]
# One unit of the second currency in the last round's winning
# market entitles you to one unit of the futarchy's master
# currency
elif msg.data[0] == 2:
second_subcurrency = call(contract.storage[2], 3)
data = call(first_subcurrency, [], 0, 4)
from = data[0]
to = data[1]
value = data[2]
txid = data[3]
if txid > contract.storage[4] and to == contract.address:
call(contract.storage[2], [to, value], 2)
contract.storage[4] = txid
# Purchase currency for ether (target releasing 10^9 units per seculum)
# Price starts off 1 eth for 10^9 units but increases hyperbolically to
# limit issuance
elif msg.data[0] == 3:
pre_ema = contract.storage[5]
post_ema = pre_ema + msg.value
pre_reserve = 10^18 / (10^9 + pre_ema / 10^9)
post_reserve = 10^18 / (10^9 + post_ema / 10^9)
call(contract.storage[2], [msg.sender, pre_reserve - post_reserve], 2)
last_sold = contract.storage[6]
contract.storage[5] = pre_ema * (100000 + last_sold - block.number) + msg.value
contract.storage[6] = block.number
# Claim all currencies as the ether miner of the current block
elif msg.data[0] == 2 and msg.sender == block.coinbase and block.number > contract.storage[8]:
i = 0
numproposals = contract.storage[1]
while i < numproposals:
market = contract.storage[1001 + i * 3]
fc = call(market, 4)
sc = call(market, 5)
call(fc, [msg.sender, 1000], 2)
call(sc, [msg.sender, 1000], 2)
i += 1
contract.storage[8] = block.number

55
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/heap.se generated vendored
View File

@ -1,55 +0,0 @@
# 0: size
# 1-n: elements
init:
contract.storage[1000] = msg.sender
code:
# Only owner of the heap is allowed to modify it
if contract.storage[1000] != msg.sender:
stop
# push
if msg.data[0] == 0:
sz = contract.storage[0]
contract.storage[sz + 1] = msg.data[1]
k = sz + 1
while k > 1:
bottom = contract.storage[k]
top = contract.storage[k/2]
if bottom < top:
contract.storage[k] = top
contract.storage[k/2] = bottom
k /= 2
else:
k = 0
contract.storage[0] = sz + 1
# pop
elif msg.data[0] == 1:
sz = contract.storage[0]
if !sz:
return(0)
prevtop = contract.storage[1]
contract.storage[1] = contract.storage[sz]
contract.storage[sz] = 0
top = contract.storage[1]
k = 1
while k * 2 < sz:
bottom1 = contract.storage[k * 2]
bottom2 = contract.storage[k * 2 + 1]
if bottom1 < top and (bottom1 < bottom2 or k * 2 + 1 >= sz):
contract.storage[k] = bottom1
contract.storage[k * 2] = top
k = k * 2
elif bottom2 < top and bottom2 < bottom1 and k * 2 + 1 < sz:
contract.storage[k] = bottom2
contract.storage[k * 2 + 1] = top
k = k * 2 + 1
else:
k = sz
contract.storage[0] = sz - 1
return(prevtop)
# top
elif msg.data[0] == 2:
return(contract.storage[1])
# size
elif msg.data[0] == 3:
return(contract.storage[0])

117
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/market.se generated vendored
View File

@ -1,117 +0,0 @@
# Creates a decentralized market between any two subcurrencies
# Here, the first subcurrency is the base asset and the second
# subcurrency is the asset priced against the base asset. Hence,
# "buying" refers to trading the first for the second, and
# "selling" refers to trading the second for the first
# storage 0: buy orders
# storage 1: sell orders
# storage 1000: first subcurrency
# storage 1001: last first subcurrency txid
# storage 2000: second subcurrency
# storage 2001: last second subcurrency txid
# storage 3000: current epoch
# storage 4000: price
# storage 4001: volume
init:
# Heap for buy orders
contract.storage[0] = create('heap.se')
# Heap for sell orders
contract.storage[1] = create('heap.se')
code:
# Initialize with [ first_subcurrency, second_subcurrency ]
if !contract.storage[1000]:
contract.storage[1000] = msg.data[0] # First subcurrency
contract.storage[1001] = -1
contract.storage[2000] = msg.data[1] # Second subcurrency
contract.storage[2001] = -1
contract.storage[3000] = block.number / 1000
stop
first_subcurrency = contract.storage[1000]
second_subcurrency = contract.storage[2000]
buy_heap = contract.storage[0]
sell_heap = contract.storage[1]
# This contract operates in "epochs" of 100 blocks
# At the end of each epoch, we process all orders
# simultaneously, independent of order. This algorithm
# prevents front-running, and generates a profit from
# the spread. The profit is permanently kept in the
# market (ie. destroyed), making both subcurrencies
# more valuable
# Epoch transition code
if contract.storage[3000] < block.number / 100:
done = 0
volume = 0
while !done:
# Grab the top buy and sell order from each heap
topbuy = call(buy_heap, 1)
topsell = call(sell_heap, 1)
# An order is recorded in the heap as:
# Buys: (2^48 - 1 - price) * 2^208 + units of first currency * 2^160 + from
# Sells: price * 2^208 + units of second currency * 2^160 + from
buyprice = -(topbuy / 2^208)
buyfcvalue = (topbuy / 2^160) % 2^48
buyer = topbuy % 2^160
sellprice = topsell / 2^208
sellscvalue = (topsell / 2^160) % 2^48
seller = topsell % 2^160
# Heap empty, or no more matching orders
if not topbuy or not topsell or buyprice < sellprice:
done = 1
else:
# Add to volume counter
volume += buyfcvalue
# Calculate how much of the second currency the buyer gets, and
# how much of the first currency the seller gets
sellfcvalue = sellscvalue / buyprice
buyscvalue = buyfcvalue * sellprice
# Send the currency units along
call(second_subcurrency, [buyer, buyscvalue], 2)
call(first_subcurrency, [seller, sellfcvalue], 2)
if volume:
contract.storage[4000] = (buyprice + sellprice) / 2
contract.storage[4001] = volume
contract.storage[3000] = block.number / 100
# Make buy order [0, price]
if msg.data[0] == 0:
# We ask the first subcurrency contract what the last transaction was. The
# way to make a buy order is to send the amount of first currency units that
# you wish to buy with, and then immediately call this contract. For security
# it makes sense to set up a tx which sends both messages in sequence atomically
data = call(first_subcurrency, [], 0, 4)
from = data[0]
to = data[1]
value = data[2]
txid = data[3]
price = msg.data[1]
if txid > contract.storage[1001] and to == contract.address:
contract.storage[1001] = txid
# Adds the order to the heap
call(buy_heap, [0, -price * 2^208 + (value % 2^48) * 2^160 + from], 2)
# Make sell order [1, price]
elif msg.data[0] == 1:
# Same mechanics as buying
data = call(second_subcurrency, [], 0, 4)
from = data[0]
to = data[1]
value = data[2]
txid = data[3]
price = msg.data[1]
if txid > contract.storage[2001] and to == contract.address:
contract.storage[2001] = txid
call(sell_heap, [0, price * 2^208 + (value % 2^48) * 2^160 + from], 2)
# Ask for price
elif msg.data[0] == 2:
return(contract.storage[4000])
# Ask for volume
elif msg.data[0] == 3:
return(contract.storage[1000])
# Ask for first currency
elif msg.data[0] == 4:
return(contract.storage[2000])
# Ask for second currency
elif msg.data[0] == 5:
return(contract.storage[4001])

35
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/subcurrency.se generated vendored
View File

@ -1,35 +0,0 @@
# Initialization
# Admin can issue and delete at will
init:
contract.storage[0] = msg.sender
code:
# If a message with one item is sent, that's a balance query
if msg.datasize == 1:
addr = msg.data[0]
return(contract.storage[addr])
# If a message with two items [to, value] are sent, that's a transfer request
elif msg.datasize == 2:
from = msg.sender
fromvalue = contract.storage[from]
to = msg.data[0]
value = msg.data[1]
if fromvalue >= value and value > 0 and to > 4:
contract.storage[from] = fromvalue - value
contract.storage[to] += value
contract.storage[2] = from
contract.storage[3] = to
contract.storage[4] = value
contract.storage[5] += 1
return(1)
return(0)
elif msg.datasize == 3 and msg.sender == contract.storage[0]:
# Admin can issue at will by sending a [to, value, 0] message
if msg.data[2] == 0:
contract.storage[msg.data[0]] += msg.data[1]
# Change admin [ newadmin, 0, 1 ]
# Set admin to 0 to disable administration
elif msg.data[2] == 1:
contract.storage[0] = msg.data[0]
# Fetch last transaction
else:
return([contract.storage[2], contract.storage[3], contract.storage[4], contract.storage[5]], 4)

39
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/test.py generated vendored
View File

@ -1,39 +0,0 @@
from __future__ import print_function
import pyethereum
t = pyethereum.tester
s = t.state()
# Create currencies
c1 = s.contract('subcurrency.se')
print("First currency: %s" % c1)
c2 = s.contract('subcurrency.se')
print("First currency: %s" % c2)
# Allocate units
s.send(t.k0, c1, 0, [t.a0, 1000, 0])
s.send(t.k0, c1, 0, [t.a1, 1000, 0])
s.send(t.k0, c2, 0, [t.a2, 1000000, 0])
s.send(t.k0, c2, 0, [t.a3, 1000000, 0])
print("Allocated units")
# Market
m = s.contract('market.se')
s.send(t.k0, m, 0, [c1, c2])
# Place orders
s.send(t.k0, c1, 0, [m, 1000])
s.send(t.k0, m, 0, [0, 1200])
s.send(t.k1, c1, 0, [m, 1000])
s.send(t.k1, m, 0, [0, 1400])
s.send(t.k2, c2, 0, [m, 1000000])
s.send(t.k2, m, 0, [1, 800])
s.send(t.k3, c2, 0, [m, 1000000])
s.send(t.k3, m, 0, [1, 600])
print("Orders placed")
# Next epoch and ping
s.mine(100)
print("Mined 100")
s.send(t.k0, m, 0, [])
print("Updating")
# Check
assert s.send(t.k0, c2, 0, [t.a0]) == [800000]
assert s.send(t.k0, c2, 0, [t.a1]) == [600000]
assert s.send(t.k0, c1, 0, [t.a2]) == [833]
assert s.send(t.k0, c1, 0, [t.a3]) == [714]
print("Balance checks passed")

12
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/datafeed.se generated vendored
View File

@ -1,12 +0,0 @@
# Database updateable only by the original creator
data creator
def init():
self.creator = msg.sender
def update(k, v):
if msg.sender == self.creator:
self.storage[k] = v
def query(k):
return(self.storage[k])

40
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/ecrecover.se generated vendored
View File

@ -1,40 +0,0 @@
# So I looked up on Wikipedia what Jacobian form actually is, and noticed that it's
# actually a rather different and more clever construction than the naive version
# that I created. It may possible to achieve a further 20-50% savings by applying
# that version.
extern all: [call]
data JORDANMUL
data JORDANADD
data EXP
def init():
self.JORDANMUL = create('jacobian_mul.se')
self.JORDANADD = create('jacobian_add.se')
self.EXP = create('modexp.se')
def call(h, v, r, s):
N = -432420386565659656852420866394968145599
P = -4294968273
h = mod(h, N)
r = mod(r, P)
s = mod(s, N)
Gx = 55066263022277343669578718895168534326250603453777594175500187360389116729240
Gy = 32670510020758816978083085130507043184471273380659243275938904335757337482424
x = r
xcubed = mulmod(mulmod(x, x, P), x, P)
beta = self.EXP.call(addmod(xcubed, 7, P), div(P + 1, 4), P)
# Static-gascost ghetto conditional
y_is_positive = mod(v, 2) xor mod(beta, 2)
y = beta * y_is_positive + (P - beta) * (1 - y_is_positive)
GZ = self.JORDANMUL.call(Gx, 1, Gy, 1, N - h, outsz=4)
XY = self.JORDANMUL.call(x, 1, y, 1, s, outsz=4)
COMB = self.JORDANADD.call(GZ[0], GZ[1], GZ[2], GZ[3], XY[0], XY[1], XY[2], XY[3], 1, outsz=5)
COMB[4] = self.EXP.call(r, N - 2, N)
Q = self.JORDANMUL.call(data=COMB, datasz=5, outsz=4)
ox = mulmod(Q[0], self.EXP.call(Q[1], P - 2, P), P)
oy = mulmod(Q[2], self.EXP.call(Q[3], P - 2, P), P)
return([ox, oy], 2)

1
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/ecrecover_compiled.evm generated vendored
View File

File diff suppressed because one or more lines are too long

32
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/jacobian_add.se generated vendored
View File

@ -1,32 +0,0 @@
extern all: [call]
data DOUBLE
def init():
self.DOUBLE = create('jacobian_double.se')
def call(axn, axd, ayn, ayd, bxn, bxd, byn, byd):
if !axn and !ayn:
o = [bxn, bxd, byn, byd]
if !bxn and !byn:
o = [axn, axd, ayn, ayd]
if o:
return(o, 4)
with P = -4294968273:
if addmod(mulmod(axn, bxd, P), P - mulmod(axd, bxn, P), P) == 0:
if addmod(mulmod(ayn, byd, P), P - mulmod(ayd, byn, P), P) == 0:
return(self.DOUBLE.call(axn, axd, ayn, ayd, outsz=4), 4)
else:
return([0, 1, 0, 1], 4)
with mn = mulmod(addmod(mulmod(byn, ayd, P), P - mulmod(ayn, byd, P), P), mulmod(bxd, axd, P), P):
with md = mulmod(mulmod(byd, ayd, P), addmod(mulmod(bxn, axd, P), P - mulmod(axn, bxd, P), P), P):
with msqn = mulmod(mn, mn, P):
with msqd = mulmod(md, md, P):
with msqman = addmod(mulmod(msqn, axd, P), P - mulmod(msqd, axn, P), P):
with msqmad = mulmod(msqd, axd, P):
with xn = addmod(mulmod(msqman, bxd, P), P - mulmod(msqmad, bxn, P), P):
with xd = mulmod(msqmad, bxd, P):
with mamxn = mulmod(mn, addmod(mulmod(axn, xd, P), P - mulmod(xn, axd, P), P), P):
with mamxd = mulmod(md, mulmod(axd, xd, P), P):
with yn = addmod(mulmod(mamxn, ayd, P), P - mulmod(mamxd, ayn, P), P):
with yd = mulmod(mamxd, ayd, P):
return([xn, xd, yn, yd], 4)

16
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/jacobian_double.se generated vendored
View File

@ -1,16 +0,0 @@
def call(axn, axd, ayn, ayd):
if !axn and !ayn:
return([0, 1, 0, 1], 4)
with P = -4294968273:
# No need to add (A, 1) because A = 0 for bitcoin
with mn = mulmod(mulmod(mulmod(axn, axn, P), 3, P), ayd, P):
with md = mulmod(mulmod(axd, axd, P), mulmod(ayn, 2, P), P):
with msqn = mulmod(mn, mn, P):
with msqd = mulmod(md, md, P):
with xn = addmod(mulmod(msqn, axd, P), P - mulmod(msqd, mulmod(axn, 2, P), P), P):
with xd = mulmod(msqd, axd, P):
with mamxn = mulmod(addmod(mulmod(axn, xd, P), P - mulmod(axd, xn, P), P), mn, P):
with mamxd = mulmod(mulmod(axd, xd, P), md, P):
with yn = addmod(mulmod(mamxn, ayd, P), P - mulmod(mamxd, ayn, P), P):
with yd = mulmod(mamxd, ayd, P):
return([xn, xd, yn, yd], 4)

37
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/jacobian_mul.se generated vendored
View File

@ -1,37 +0,0 @@
# Expected gas cost
#
# def expect(n, point_at_infinity=False):
# n = n % (2**256 - 432420386565659656852420866394968145599)
# if point_at_infinity:
# return 79
# if n == 0:
# return 34479
# L = int(1 + math.log(n) / math.log(2))
# H = len([x for x in b.encode(n, 2) if x == '1'])
# return 34221 + 94 * L + 343 * H
data DOUBLE
data ADD
def init():
self.DOUBLE = create('jacobian_double.se')
self.ADD = create('jacobian_add.se')
def call(axn, axd, ayn, ayd, n):
n = mod(n, -432420386565659656852420866394968145599)
if !axn * !ayn + !n: # Constant-gas version of !axn and !ayn or !n
return([0, 1, 0, 1], 4)
with o = [0, 0, 1, 0, 1, 0, 0, 0, 0]:
with b = 2 ^ 255:
while gt(b, 0):
if n & b:
~call(20000, self.DOUBLE, 0, o + 31, 129, o + 32, 128)
o[5] = axn
o[6] = axd
o[7] = ayn
o[8] = ayd
~call(20000, self.ADD, 0, o + 31, 257, o + 32, 128)
else:
~call(20000, self.DOUBLE, 0, o + 31, 129, o + 32, 128)
b = div(b, 2)
return(o + 32, 4)

11
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/modexp.se generated vendored
View File

@ -1,11 +0,0 @@
def call(b, e, m):
with o = 1:
with bit = 2 ^ 255:
while gt(bit, 0):
# A touch of loop unrolling for 20% efficiency gain
o = mulmod(mulmod(o, o, m), b ^ !(!(e & bit)), m)
o = mulmod(mulmod(o, o, m), b ^ !(!(e & div(bit, 2))), m)
o = mulmod(mulmod(o, o, m), b ^ !(!(e & div(bit, 4))), m)
o = mulmod(mulmod(o, o, m), b ^ !(!(e & div(bit, 8))), m)
bit = div(bit, 16)
return(o)

78
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/substitutes.py generated vendored
View File

@ -1,78 +0,0 @@
import bitcoin as b
import math
import sys
def signed(o):
return map(lambda x: x - 2**256 if x >= 2**255 else x, o)
def hamming_weight(n):
return len([x for x in b.encode(n, 2) if x == '1'])
def binary_length(n):
return len(b.encode(n, 2))
def jacobian_mul_substitute(A, B, C, D, N):
if A == 0 and C == 0 or (N % b.N) == 0:
return {"gas": 86, "output": [0, 1, 0, 1]}
else:
output = b.jordan_multiply(((A, B), (C, D)), N)
return {
"gas": 35262 + 95 * binary_length(N % b.N) + 355 * hamming_weight(N % b.N),
"output": signed(list(output[0]) + list(output[1]))
}
def jacobian_add_substitute(A, B, C, D, E, F, G, H):
if A == 0 or E == 0:
gas = 149
elif (A * F - B * E) % b.P == 0:
if (C * H - D * G) % b.P == 0:
gas = 442
else:
gas = 177
else:
gas = 301
output = b.jordan_add(((A, B), (C, D)), ((E, F), (G, H)))
return {
"gas": gas,
"output": signed(list(output[0]) + list(output[1]))
}
def modexp_substitute(base, exp, mod):
return {
"gas": 5150,
"output": signed([pow(base, exp, mod) if mod > 0 else 0])
}
def ecrecover_substitute(z, v, r, s):
P, A, B, N, Gx, Gy = b.P, b.A, b.B, b.N, b.Gx, b.Gy
x = r
beta = pow(x*x*x+A*x+B, (P + 1) / 4, P)
BETA_PREMIUM = modexp_substitute(x, (P + 1) / 4, P)["gas"]
y = beta if v % 2 ^ beta % 2 else (P - beta)
Gz = b.jordan_multiply(((Gx, 1), (Gy, 1)), (N - z) % N)
GZ_PREMIUM = jacobian_mul_substitute(Gx, 1, Gy, 1, (N - z) % N)["gas"]
XY = b.jordan_multiply(((x, 1), (y, 1)), s)
XY_PREMIUM = jacobian_mul_substitute(x, 1, y, 1, s % N)["gas"]
Qr = b.jordan_add(Gz, XY)
QR_PREMIUM = jacobian_add_substitute(Gz[0][0], Gz[0][1], Gz[1][0], Gz[1][1],
XY[0][0], XY[0][1], XY[1][0], XY[1][1]
)["gas"]
Q = b.jordan_multiply(Qr, pow(r, N - 2, N))
Q_PREMIUM = jacobian_mul_substitute(Qr[0][0], Qr[0][1], Qr[1][0], Qr[1][1],
pow(r, N - 2, N))["gas"]
R_PREMIUM = modexp_substitute(r, N - 2, N)["gas"]
OX_PREMIUM = modexp_substitute(Q[0][1], P - 2, P)["gas"]
OY_PREMIUM = modexp_substitute(Q[1][1], P - 2, P)["gas"]
Q = b.from_jordan(Q)
return {
"gas": 991 + BETA_PREMIUM + GZ_PREMIUM + XY_PREMIUM + QR_PREMIUM +
Q_PREMIUM + R_PREMIUM + OX_PREMIUM + OY_PREMIUM,
"output": signed(Q)
}

129
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/test.py generated vendored
View File

@ -1,129 +0,0 @@
import bitcoin as b
import random
import sys
import math
from pyethereum import tester as t
import substitutes
import time
vals = [random.randrange(2**256) for i in range(12)]
test_points = [list(p[0]) + list(p[1]) for p in
[b.jordan_multiply(((b.Gx, 1), (b.Gy, 1)), r) for r in vals]]
G = [b.Gx, 1, b.Gy, 1]
Z = [0, 1, 0, 1]
def neg_point(p):
return [p[0], b.P - p[1], p[2], b.P - p[3]]
s = t.state()
s.block.gas_limit = 10000000
t.gas_limit = 1000000
c = s.contract('modexp.se')
print "Starting modexp tests"
for i in range(0, len(vals) - 2, 3):
o1 = substitutes.modexp_substitute(vals[i], vals[i+1], vals[i+2])
o2 = s.profile(t.k0, c, 0, funid=0, abi=vals[i:i+3])
#assert o1["gas"] == o2["gas"], (o1, o2)
assert o1["output"] == o2["output"], (o1, o2)
c = s.contract('jacobian_add.se')
print "Starting addition tests"
for i in range(2):
P = test_points[i * 2]
Q = test_points[i * 2 + 1]
NP = neg_point(P)
o1 = substitutes.jacobian_add_substitute(*(P + Q))
o2 = s.profile(t.k0, c, 0, funid=0, abi=P + Q)
#assert o1["gas"] == o2["gas"], (o1, o2)
assert o1["output"] == o2["output"], (o1, o2)
o1 = substitutes.jacobian_add_substitute(*(P + NP))
o2 = s.profile(t.k0, c, 0, funid=0, abi=P + NP)
#assert o1["gas"] == o2["gas"], (o1, o2)
assert o1["output"] == o2["output"], (o1, o2)
o1 = substitutes.jacobian_add_substitute(*(P + P))
o2 = s.profile(t.k0, c, 0, funid=0, abi=P + P)
#assert o1["gas"] == o2["gas"], (o1, o2)
assert o1["output"] == o2["output"], (o1, o2)
o1 = substitutes.jacobian_add_substitute(*(P + Z))
o2 = s.profile(t.k0, c, 0, funid=0, abi=P + Z)
#assert o1["gas"] == o2["gas"], (o1, o2)
assert o1["output"] == o2["output"], (o1, o2)
o1 = substitutes.jacobian_add_substitute(*(Z + P))
o2 = s.profile(t.k0, c, 0, funid=0, abi=Z + P)
#assert o1["gas"] == o2["gas"], (o1, o2)
assert o1["output"] == o2["output"], (o1, o2)
c = s.contract('jacobian_mul.se')
print "Starting multiplication tests"
mul_tests = [
Z + [0],
Z + [vals[0]],
test_points[0] + [0],
test_points[1] + [b.N],
test_points[2] + [1],
test_points[2] + [2],
test_points[2] + [3],
test_points[2] + [4],
test_points[3] + [5],
test_points[3] + [6],
test_points[4] + [7],
test_points[4] + [2**254],
test_points[4] + [vals[1]],
test_points[4] + [vals[2]],
test_points[4] + [vals[3]],
test_points[5] + [2**256 - 1],
]
for i, test in enumerate(mul_tests):
print 'trying mul_test %i' % i, test
o1 = substitutes.jacobian_mul_substitute(*test)
o2 = s.profile(t.k0, c, 0, funid=0, abi=test)
# assert o1["gas"] == o2["gas"], (o1, o2, test)
assert o1["output"] == o2["output"], (o1, o2, test)
c = s.contract('ecrecover.se')
print "Starting ecrecover tests"
for i in range(5):
print 'trying ecrecover_test', vals[i*2], vals[i*2+1]
k = vals[i*2]
h = vals[i*2+1]
V, R, S = b.ecdsa_raw_sign(b.encode(h, 256, 32), k)
aa = time.time()
o1 = substitutes.ecrecover_substitute(h, V, R, S)
print 'sub', time.time() - aa
a = time.time()
o2 = s.profile(t.k0, c, 0, funid=0, abi=[h, V, R, S])
print time.time() - a
# assert o1["gas"] == o2["gas"], (o1, o2, h, V, R, S)
assert o1["output"] == o2["output"], (o1, o2, h, V, R, S)
# Explicit tests
data = [[
0xf007a9c78a4b2213220adaaf50c89a49d533fbefe09d52bbf9b0da55b0b90b60,
0x1b,
0x5228fc9e2fabfe470c32f459f4dc17ef6a0a81026e57e4d61abc3bc268fc92b5,
0x697d4221cd7bc5943b482173de95d3114b9f54c5f37cc7f02c6910c6dd8bd107
]]
for datum in data:
o1 = substitutes.ecrecover_substitute(*datum)
o2 = s.profile(t.k0, c, 0, funid=0, abi=datum)
#assert o1["gas"] == o2["gas"], (o1, o2, datum)
assert o1["output"] == o2["output"], (o1, o2, datum)

45
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/channel.se generated vendored
View File

@ -1,45 +0,0 @@
if msg.data[0] == 0:
new_id = contract.storage[-1]
# store [from, to, value, maxvalue, timeout] in contract storage
contract.storage[new_id] = msg.sender
contract.storage[new_id + 1] = msg.data[1]
contract.storage[new_id + 2] = 0
contract.storage[new_id + 3] = msg.value
contract.storage[new_id + 4] = 2^254
# increment next id
contract.storage[-1] = new_id + 10
# return id of this channel
return(new_id)
# Increase payment on channel: [1, id, value, v, r, s]
elif msg.data[0] == 1:
# Ecrecover native extension; will be a different address in testnet and live
ecrecover = 0x46a8d0b21b1336d83b06829f568d7450df36883f
# Message data parameters
id = msg.data[1] % 2^160
value = msg.data[2]
# Determine sender from signature
h = sha3([id, value], 2)
sender = call(ecrecover, [h, msg.data[3], msg.data[4], msg.data[5]], 4)
# Check sender matches and new value is greater than old
if sender == contract.storage[id]:
if value > contract.storage[id + 2] and value <= contract.storage[id + 3]:
# Update channel, increasing value and setting timeout
contract.storage[id + 2] = value
contract.storage[id + 4] = block.number + 1000
# Cash out channel: [2, id]
elif msg.data[0] == 2:
id = msg.data[1] % 2^160
# Check if timeout has run out
if block.number >= contract.storage[id + 3]:
# Send funds
send(contract.storage[id + 1], contract.storage[id + 2])
# Send refund
send(contract.storage[id], contract.storage[id + 3] - contract.storage[id + 2])
# Clear storage
contract.storage[id] = 0
contract.storage[id + 1] = 0
contract.storage[id + 2] = 0
contract.storage[id + 3] = 0
contract.storage[id + 4] = 0

19
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/map.se generated vendored
View File

@ -1,19 +0,0 @@
# An implementation of a contract for storing a key/value binding
init:
# Set owner
contract.storage[0] = msg.sender
code:
# Check ownership
if msg.sender == contract.storage[0]:
# Get: returns (found, val)
if msg.data[0] == 0:
s = sha3(msg.data[1])
return([contract.storage[s], contract.storage[s+1]], 2)
# Set: sets map[k] = v
elif msg.data[0] == 1:
s = sha3(msg.data[1])
contract.storage[s] = 1
contract.storage[s + 1] = msg.data[2]
# Suicide
elif msg.data[2] == 1:
suicide(0)

14
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/multiforward.se generated vendored
View File

@ -1,14 +0,0 @@
init:
contract.storage[0] = msg.sender
code:
if msg.sender != contract.storage[0]:
stop
i = 0
while i < ~calldatasize():
to = ~calldataload(i)
value = ~calldataload(i+20) / 256^12
datasize = ~calldataload(i+32) / 256^30
data = alloc(datasize)
~calldatacopy(data, i+34, datasize)
~call(tx.gas - 25, to, value, data, datasize, 0, 0)
i += 34 + datasize

166
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/shadowchain.se generated vendored
View File

@ -1,166 +0,0 @@
# Exists in state:
# (i) last committed block
# (ii) chain of uncommitted blocks (linear only)
# (iii) transactions, each tx with an associated block number
#
# Uncommitted block =
# [ numtxs, numkvs, tx1 (N words), tx2 (N words) ..., [k1, v1], [k2, v2], [k3, v3] ... ]
#
# Block checking process
#
# Suppose last committed state is m
# Last uncommitted state is n
# Contested block is b
#
# 1. Temporarily apply all state transitions from
# m to b
# 2. Run code, get list of changes
# 3. Check is list of changes matches deltas
# * if yes, do nothing
# * if no, set last uncommitted state to pre-b
#
# Storage variables:
#
# Last committed block: 0
# Last uncommitted block: 1
# Contract holding code: 2
# Uncommitted map: 3
# Transaction length (parameter): 4
# Block b: 2^160 + b * 2^40:
# + 1: submission blknum
# + 2: submitter
# + 3: data in uncommitted block format above
# Last committed storage:
# sha3(k): index k
# Initialize: [0, c, txlength], set address of the code-holding contract and the transaction
# length
if not contract.storage[2]:
contract.storage[2] = msg.data[1]
contract.storage[4] = msg.data[2]
stop
# Sequentially commit all uncommitted blocks that are more than 1000 mainchain-blocks old
last_committed_block = contract.storage[0]
last_uncommitted_block = contract.storage[1]
lcb_storage_index = 2^160 + last_committed_block * 2^40
while contract.storage[lcb_storage_index + 1] < block.number - 1000 and last_committed_block < last_uncommitted_block:
kvpairs = contract.storage[lcb_storage_index]
i = 0
while i < kvpairs:
k = contract.storage[lcb_storage_index + 3 + i * 2]
v = contract.storage[lcb_storage_index + 4 + i * 2]
contract.storage[sha3(k)] = v
i += 1
last_committed_block += 1
lcb_storage_index += 2^40
contract.storage[0] = last_committed_block
# Propose block: [ 0, block number, data in block format above ... ]
if msg.data[0] == 0:
blknumber = msg.data[1]
# Block number must be correct
if blknumber != contract.storage[1]:
stop
# Deposit requirement
if msg.value < 10^19:
stop
# Store the proposal in storage as
# [ 0, main-chain block number, sender, block data...]
start_index = 2^160 + blknumber * 2^40
numkvs = (msg.datasize - 2) / 2
contract.storage[start_index + 1] = block.number
1ontract.storage[start_index + 2] = msg.sender
i = 0
while i < msg.datasize - 2:
contract.storage[start_index + 3 + i] = msg.data[2 + i]
i += 1
contract.storage[1] = blknumber + 1
# Challenge block: [ 1, b ]
elif msg.data[0] == 1:
blknumber = msg.data[1]
txwidth = contract.storage[4]
last_uncommitted_block = contract.storage[1]
last_committed_block = contract.storage[0]
# Cannot challenge nonexistent or committed blocks
if blknumber <= last_uncommitted_block or blknumber > last_committed_block:
stop
# Create a contract to serve as a map that maintains keys and values
# temporarily
tempstore = create('map.se')
contract.storage[3] = tempstore
# Unquestioningly apply the state transitions from the last committed block
# up to b
b = last_committed_block
cur_storage_index = 2^160 + last_committed_block * 2^40
while b < blknumber:
numtxs = contract.storage[cur_storage_index + 3]
numkvs = contract.storage[cur_storage_index + 4]
kv0index = cur_storage_index + 5 + numtxs * txwidth
i = 0
while i < numkvs:
k = contract.storage[kv0index + i * 2]
v = contract.storage[kx0index + i * 2 + 1]
call(tempstore, [1, k, v], 3)
i += 1
b += 1
cur_storage_index += 2^40
# Run the actual code, and see what state transitions it outputs
# The way that the code is expected to work is to:
#
# (1) take as input the list of transactions (the contract should
# use msg.datasize to determine how many txs there are, and it should
# be aware of the value of txwidth)
# (2) call this contract with [2, k] to read current state data
# (3) call this contract with [3, k, v] to write current state data
# (4) return as output a list of all state transitions that it made
# in the form [kvcount, k1, v1, k2, v2 ... ]
#
# The reason for separating (2) from (3) is that sometimes the state
# transition may end up changing a given key many times, and we don't
# need to inefficiently store that in storage
numkvs = contract.storage[cur_storage_index + 3]
numtxs = contract.storage[cur_storage_index + 4]
# Populate input array
inpwidth = numtxs * txwidth
inp = array(inpwidth)
i = 0
while i < inpwidth:
inp[i] = contract.storage[cur_storage_index + 5 + i]
i += 1
out = call(contract.storage[2], inp, inpwidth, numkvs * 2 + 1)
# Check that the number of state transitions is the same
if out[0] != kvcount:
send(msg.sender, 10^19)
contract.storage[0] = last_committed_block
stop
kv0index = cur_storage_index + 5 + numtxs * txwidth
i = 0
while i < kvcount:
# Check that each individual state transition matches
k = contract.storage[kv0index + i * 2 + 1]
v = contract.storage[kv0index + i * 2 + 2]
if k != out[i * 2 + 1] or v != out[i * 2 + 2]:
send(msg.sender, 10^19)
contract.storage[0] = last_committed_block
stop
i += 1
# Suicide tempstore
call(tempstore, 2)
# Read data [2, k]
elif msg.data[0] == 2:
tempstore = contract.storage[3]
o = call(tempstore, [0, msg.data[1]], 2, 2)
if o[0]:
return(o[1])
else:
return contract.storage[sha3(msg.data[1])]
# Write data [3, k, v]
elif msg.data[0] == 3:
tempstore = contract.storage[3]
call(tempstore, [1, msg.data[1], msg.data[2]], 3, 2)

31
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/fixedpoint.se generated vendored
View File

@ -1,31 +0,0 @@
type f: [a, b, c, d, e]
macro f($a) + f($b):
f(add($a, $b))
macro f($a) - f($b):
f(sub($a, $b))
macro f($a) * f($b):
f(mul($a, $b) / 10000)
macro f($a) / f($b):
f(sdiv($a * 10000, $b))
macro f($a) % f($b):
f(smod($a, $b))
macro f($v) = f($w):
$v = $w
macro unfify(f($a)):
$a / 10000
macro fify($a):
f($a * 10000)
a = fify(5)
b = fify(2)
c = a / b
e = c + (a / b)
return(unfify(e))

116
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/long_integer_macros.se generated vendored
View File

@ -1,116 +0,0 @@
macro smin($a, $b):
with $1 = $a:
with $2 = $b:
if(slt($1, $2), $1, $2)
macro smax($a, $b):
with $1 = $a:
with $2 = $b:
if(slt($1, $2), $2, $1)
def omul(x, y):
o = expose(mklong(x) * mklong(y))
return(slice(o, 1), o[0]+1)
def oadd(x, y):
o = expose(mklong(x) + mklong(y))
return(slice(o, 1), o[0]+1)
def osub(x, y):
o = expose(mklong(x) - mklong(y))
return(slice(o, 1), o[0]+1)
def odiv(x, y):
o = expose(mklong(x) / mklong(y))
return(slice(o, 1), o[0]+1)
def comb(a:a, b:a, sign):
sz = smax(a[0], b[0])
msz = smin(a[0], b[0])
c = array(sz + 2)
c[0] = sz
i = 0
carry = 0
while i < msz:
m = a[i + 1] + sign * b[i + 1] + carry
c[i + 1] = mod(m + 2^127, 2^128) - 2^127
carry = (div(m + 2^127, 2^128) + 2^127) % 2^128 - 2^127
i += 1
u = if(a[0] > msz, a, b)
s = if(a[0] > msz, 1, sign)
while i < sz:
m = s * u[i + 1] + carry
c[i + 1] = mod(m + 2^127, 2^128) - 2^127
carry = (div(m + 2^127, 2^128) + 2^127) % 2^128 - 2^127
i += 1
if carry:
c[0] += 1
c[sz + 1] = carry
return(c, c[0]+1)
def mul(a:a, b:a):
c = array(a[0] + b[0] + 2)
c[0] = a[0] + b[0]
i = 0
while i < a[0]:
j = 0
carry = 0
while j < b[0]:
m = c[i + j + 1] + a[i + 1] * b[j + 1] + carry
c[i + j + 1] = mod(m + 2^127, 2^128) - 2^127
carry = (div(m + 2^127, 2^128) + 2^127) % 2^128 - 2^127
j += 1
if carry:
c[0] = a[0] + b[0] + 1
c[i + j + 1] += carry
i += 1
return(c, c[0]+1)
macro long($a) + long($b):
long(self.comb($a:$a[0]+1, $b:$b[0]+1, 1, outsz=$a[0]+$b[0]+2))
macro long($a) - long($b):
long(self.comb($a:$a[0]+1, $b:$b[0]+1, -1, outsz=$a[0]+$b[0]+2))
macro long($a) * long($b):
long(self.mul($a:$a[0]+1, $b:$b[0]+1, outsz=$a[0]+$b[0]+2))
macro long($a) / long($b):
long(self.div($a:$a[0]+1, $b:$b[0]+1, outsz=$a[0]+$b[0]+2))
macro mulexpand(long($a), $k, $m):
long:
with $c = array($a[0]+k+2):
$c[0] = $a[0]+$k
with i = 0:
while i < $a[0]:
v = $a[i+1] * $m + $c[i+$k+1]
$c[i+$k+1] = mod(v + 2^127, 2^128) - 2^127
$c[i+$k+2] = div(v + 2^127, 2^128)
i += 1
$c
def div(a:a, b:a):
asz = a[0]
bsz = b[0]
while b[bsz] == 0 and bsz > 0:
bsz -= 1
c = array(asz+2)
c[0] = asz+1
while 1:
while a[asz] == 0 and asz > 0:
asz -= 1
if asz < bsz:
return(c, c[0]+1)
sub = expose(mulexpand(long(b), asz - bsz, a[asz] / b[bsz]))
c[asz - bsz+1] = a[asz] / b[bsz]
a = expose(long(a) - long(sub))
a[asz-1] += 2^128 * a[asz]
a[asz] = 0
macro mklong($i):
long([2, mod($i + 2^127, 2^128) - 2^127, div($i + 2^127, 2^128)])
macro expose(long($i)):
$i

2
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/mul2.se generated vendored
View File

@ -1,2 +0,0 @@
def double(v):
return(v*2)

187
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/mutuala.se generated vendored
View File

@ -1,187 +0,0 @@
# mutuala - subcurrency
# We want to issue a currency that reduces in value as you store it through negative interest.
# That negative interest would be stored in a commons account. It's like the p2p version of a
# capital tax
# the same things goes for transactions - you pay as you use the currency. However, the more
# you pay, the more you get to say about what the tax is used for
# each participant can propose a recipient for a payout to be made out of the commons account,
# others can vote on it by awarding it tax_credits.
# TODO should proposal have expiration timestamp?, after which the tax_credits are refunded
# TODO multiple proposals can take more credits that available in the Commons, how to handle this
# TODO how to handle lost accounts, after which no longer possible to get 2/3 majority
shared:
COMMONS = 42
ADMIN = 666
CAPITAL_TAX_PER_DAY = 7305 # 5% per year
PAYMENT_TAX = 20 # 5%
ACCOUNT_LIST_OFFSET = 2^160
ACCOUNT_MAP_OFFSET = 2^161
PROPOSAL_LIST_OFFSET = 2^162
PROPOSAL_MAP_OFFSET = 2^163
init:
contract.storage[ADMIN] = msg.sender
contract.storage[ACCOUNT_LIST_OFFSET - 1] = 1
contract.storage[ACCOUNT_LIST_OFFSET] = msg.sender
contract.storage[ACCOUNT_MAP_OFFSET + msg.sender] = 10^12
contract.storage[ACCOUNT_MAP_OFFSET + msg.sender + 1] = block.timestamp
# contract.storage[COMMONS] = balance commons
# contract.storage[ACCOUNT_LIST_OFFSET - 1] = number of accounts
# contract.storage[ACCOUNT_LIST_OFFSET + n] = account n
# contract.storage[PROPOSAL_LIST_OFFSET - 1] contains the number of proposals
# contract.storage[PROPOSAL_LIST_OFFSET + n] = proposal n
# per account:
# contract.storage[ACCOUNT_MAP_OFFSET + account] = balance
# contract.storage[ACCOUNT_MAP_OFFSET + account+1] = timestamp_last_transaction
# contract.storage[ACCOUNT_MAP_OFFSET + account+2] = tax_credits
# per proposal:
# contract.storage[PROPOSAL_MAP_OFFSET + proposal_id] = recipient
# contract.storage[PROPOSAL_MAP_OFFSET + proposal_id+1] = amount
# contract.storage[PROPOSAL_MAP_OFFSET + proposal_id+2] = total vote credits
code:
if msg.data[0] == "suicide" and msg.sender == contract.storage[ADMIN]:
suicide(msg.sender)
elif msg.data[0] == "balance":
addr = msg.data[1]
return(contract.storage[ACCOUNT_MAP_OFFSET + addr])
elif msg.data[0] == "pay":
from = msg.sender
fromvalue = contract.storage[ACCOUNT_MAP_OFFSET + from]
to = msg.data[1]
if to == 0 or to >= 2^160:
return([0, "invalid address"], 2)
value = msg.data[2]
tax = value / PAYMENT_TAX
if fromvalue >= value + tax:
contract.storage[ACCOUNT_MAP_OFFSET + from] = fromvalue - (value + tax)
contract.storage[ACCOUNT_MAP_OFFSET + to] += value
# tax
contract.storage[COMMONS] += tax
contract.storage[ACCOUNT_MAP_OFFSET + from + 2] += tax
# check timestamp field to see if target account exists
if contract.storage[ACCOUNT_MAP_OFFSET + to + 1] == 0:
# register new account
nr_accounts = contract.storage[ACCOUNT_LIST_OFFSET - 1]
contract.storage[ACCOUNT_LIST_OFFSET + nr_accounts] = to
contract.storage[ACCOUNT_LIST_OFFSET - 1] += 1
contract.storage[ACCOUNT_MAP_OFFSET + to + 1] = block.timestamp
return(1)
else:
return([0, "insufficient balance"], 2)
elif msg.data[0] == "hash":
proposal_id = sha3(msg.data[1])
return(proposal_id)
elif msg.data[0] == "propose":
from = msg.sender
# check if sender has an account and has tax credits
if contract.storage[ACCOUNT_MAP_OFFSET + from + 2] == 0:
return([0, "sender has no tax credits"], 2)
proposal_id = sha3(msg.data[1])
# check if proposal doesn't already exist
if contract.storage[PROPOSAL_MAP_OFFSET + proposal_id]:
return([0, "proposal already exists"])
to = msg.data[2]
# check if recipient is a valid address and has an account (with timestamp)
if to == 0 or to >= 2^160:
return([0, "invalid address"], 2)
if contract.storage[ACCOUNT_MAP_OFFSET + to + 1] == 0:
return([0, "invalid to account"], 2)
value = msg.data[3]
# check if there is enough money in the commons account
if value > contract.storage[COMMONS]:
return([0, "not enough credits in commons"], 2)
# record proposal in list
nr_proposals = contract.storage[PROPOSAL_LIST_OFFSET - 1]
contract.storage[PROPOSAL_LIST_OFFSET + nr_proposals] = proposal_id
contract.storage[PROPOSAL_LIST_OFFSET - 1] += 1
# record proposal in map
contract.storage[PROPOSAL_MAP_OFFSET + proposal_id] = to
contract.storage[PROPOSAL_MAP_OFFSET + proposal_id + 1] = value
return(proposal_id)
elif msg.data[0] == "vote":
from = msg.sender
proposal_id = sha3(msg.data[1])
value = msg.data[2]
# check if sender has an account and has tax credits
if value < contract.storage[ACCOUNT_MAP_OFFSET + from + 2]:
return([0, "sender doesn't have enough tax credits"], 2)
# check if proposal exist
if contract.storage[PROPOSAL_MAP_OFFSET + proposal_id] == 0:
return([0, "proposal doesn't exist"], 2)
# increase votes
contract.storage[PROPOSAL_MAP_OFFSET + proposal_id + 2] += value
# withdraw tax credits
contract.storage[ACCOUNT_MAP_OFFSET + from + 2] -= value
# did we reach 2/3 threshold?
if contract.storage[PROPOSAL_MAP_OFFSET + proposal_id + 2] >= contract.storage[COMMONS] * 2 / 3:
# got majority
to = contract.storage[PROPOSAL_MAP_OFFSET + proposal_id]
amount = contract.storage[PROPOSAL_MAP_OFFSET + proposal_id + 1]
# adjust balances
contract.storage[ACCOUNT_MAP_OFFSET + to] += amount
contract.storage[COMMONS] -= amount
# reset proposal
contract.storage[PROPOSAL_MAP_OFFSET + proposal_id] = 0
contract.storage[PROPOSAL_MAP_OFFSET + proposal_id + 1] = 0
contract.storage[PROPOSAL_MAP_OFFSET + proposal_id + 2] = 0
return(1)
return(proposal_id)
elif msg.data[0] == "tick":
nr_accounts = contract.storage[ACCOUNT_LIST_OFFSET - 1]
account_idx = 0
tax_paid = 0
# process all accounts and see if they have to pay their daily capital tax
while account_idx < nr_accounts:
cur_account = contract.storage[ACCOUNT_LIST_OFFSET + account_idx]
last_timestamp = contract.storage[ACCOUNT_MAP_OFFSET + cur_account + 1]
time_diff = block.timestamp - last_timestamp
if time_diff >= 86400:
tax_days = time_diff / 86400
balance = contract.storage[ACCOUNT_MAP_OFFSET + cur_account]
tax = tax_days * (balance / CAPITAL_TAX_PER_DAY)
if tax > 0:
# charge capital tax, but give tax credits in return
contract.storage[ACCOUNT_MAP_OFFSET + cur_account] -= tax
contract.storage[ACCOUNT_MAP_OFFSET + cur_account + 1] += tax_days * 86400
contract.storage[ACCOUNT_MAP_OFFSET + cur_account + 2] += tax
contract.storage[COMMONS] += tax
tax_paid += 1
account_idx += 1
return(tax_paid) # how many accounts did we charge tax on
else:
return([0, "unknown command"], 2)

7
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/namecoin.se generated vendored
View File

@ -1,7 +0,0 @@
def register(k, v):
if !self.storage[k]: # Is the key not yet taken?
# Then take it!
self.storage[k] = v
return(1)
else:
return(0) // Otherwise do nothing

43
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/peano.se generated vendored
View File

@ -1,43 +0,0 @@
macro padd($x, psuc($y)):
psuc(padd($x, $y))
macro padd($x, z()):
$x
macro dec(psuc($x)):
dec($x) + 1
macro dec(z()):
0
macro pmul($x, z()):
z()
macro pmul($x, psuc($y)):
padd(pmul($x, $y), $x)
macro pexp($x, z()):
one()
macro pexp($x, psuc($y)):
pmul($x, pexp($x, $y))
macro fac(z()):
one()
macro fac(psuc($x)):
pmul(psuc($x), fac($x))
macro one():
psuc(z())
macro two():
psuc(psuc(z()))
macro three():
psuc(psuc(psuc(z())))
macro five():
padd(three(), two())
return([dec(pmul(three(), pmul(three(), three()))), dec(fac(five()))], 2)

4
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/returnten.se generated vendored
View File

@ -1,4 +0,0 @@
extern mul2: [double]
x = create("mul2.se")
return(x.double(5))

33
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/quicksort.se generated vendored
View File

@ -1,33 +0,0 @@
def kall():
argcount = ~calldatasize() / 32
if argcount == 1:
return(~calldataload(1))
args = array(argcount)
~calldatacopy(args, 1, argcount * 32)
low = array(argcount)
lsz = 0
high = array(argcount)
hsz = 0
i = 1
while i < argcount:
if args[i] < args[0]:
low[lsz] = args[i]
lsz += 1
else:
high[hsz] = args[i]
hsz += 1
i += 1
low = self.kall(data=low, datasz=lsz, outsz=lsz)
high = self.kall(data=high, datasz=hsz, outsz=hsz)
o = array(argcount)
i = 0
while i < lsz:
o[i] = low[i]
i += 1
o[lsz] = args[0]
j = 0
while j < hsz:
o[lsz + 1 + j] = high[j]
j += 1
return(o, argcount)

46
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/quicksort_pairs.se generated vendored
View File

@ -1,46 +0,0 @@
# Quicksort pairs
# eg. input of the form [ 30, 1, 90, 2, 70, 3, 50, 4]
# outputs [ 30, 1, 50, 4, 70, 3, 90, 2 ]
#
# Note: this can be used as a generalized sorting algorithm:
# map every object to [ key, ref ] where `ref` is the index
# in memory to all of the properties and `key` is the key to
# sort by
def kall():
argcount = ~calldatasize() / 64
if argcount == 1:
return([~calldataload(1), ~calldataload(33)], 2)
args = array(argcount * 2)
~calldatacopy(args, 1, argcount * 64)
low = array(argcount * 2)
lsz = 0
high = array(argcount * 2)
hsz = 0
i = 2
while i < argcount * 2:
if args[i] < args[0]:
low[lsz] = args[i]
low[lsz + 1] = args[i + 1]
lsz += 2
else:
high[hsz] = args[i]
high[hsz + 1] = args[i + 1]
hsz += 2
i = i + 2
low = self.kall(data=low, datasz=lsz, outsz=lsz)
high = self.kall(data=high, datasz=hsz, outsz=hsz)
o = array(argcount * 2)
i = 0
while i < lsz:
o[i] = low[i]
i += 1
o[lsz] = args[0]
o[lsz + 1] = args[1]
j = 0
while j < hsz:
o[lsz + 2 + j] = high[j]
j += 1
return(o, argcount * 2)

94
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/schellingcoin.se generated vendored
View File

@ -1,94 +0,0 @@
# SchellingCoin implementation
#
# Epoch length: 100 blocks
# Target savings depletion rate: 0.1% per epoch
data epoch
data hashes_submitted
data output
data quicksort_pairs
data accounts[2^160]
data submissions[2^80](hash, deposit, address, value)
extern any: [call]
def init():
self.epoch = block.number / 100
self.quicksort_pairs = create('quicksort_pairs.se')
def any():
if block.number / 100 > epoch:
# Sort all values submitted
N = self.hashes_submitted
o = array(N * 2)
i = 0
j = 0
while i < N:
v = self.submissions[i].value
if v:
o[j] = v
o[j + 1] = i
j += 2
i += 1
values = self.quicksort_pairs.call(data=o, datasz=j, outsz=j)
# Calculate total deposit, refund non-submitters and
# cleanup
deposits = array(j / 2)
addresses = array(j / 2)
i = 0
total_deposit = 0
while i < j / 2:
base_index = HASHES + values[i * 2 + 1] * 3
deposits[i] = self.submissions[i].deposit
addresses[i] = self.submissions[i].address
if self.submissions[values[i * 2 + 1]].value:
total_deposit += deposits[i]
else:
send(addresses[i], deposits[i] * 999 / 1000)
i += 1
inverse_profit_ratio = total_deposit / (contract.balance / 1000) + 1
# Reward everyone
i = 0
running_deposit_sum = 0
halfway_passed = 0
while i < j / 2:
new_deposit_sum = running_deposit_sum + deposits[i]
if new_deposit_sum > total_deposit / 4 and running_deposit_sum < total_deposit * 3 / 4:
send(addresses[i], deposits[i] + deposits[i] / inverse_profit_ratio * 2)
else:
send(addresses[i], deposits[i] - deposits[i] / inverse_profit_ratio)
if not halfway_passed and new_deposit_sum > total_deposit / 2:
self.output = self.submissions[i].value
halfway_passed = 1
self.submissions[i].value = 0
running_deposit_sum = new_deposit_sum
i += 1
self.epoch = block.number / 100
self.hashes_submitted = 0
def submit_hash(h):
if block.number % 100 < 50:
cur = self.hashes_submitted
pos = HASHES + cur * 3
self.submissions[cur].hash = h
self.submissions[cur].deposit = msg.value
self.submissions[cur].address = msg.sender
self.hashes_submitted = cur + 1
return(cur)
def submit_value(index, v):
if sha3([msg.sender, v], 2) == self.submissions[index].hash:
self.submissions[index].value = v
return(1)
def request_balance():
return(contract.balance)
def request_output():
return(self.output)

171
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/schellingdollar.se generated vendored
View File

@ -1,171 +0,0 @@
# Hedged zero-supply dollar implementation
# Uses SchellingCoin as price-determining backend
#
# Stored variables:
#
# 0: Schelling coin contract
# 1: Last epoch
# 2: Genesis block of contract
# 3: USD exposure
# 4: ETH exposure
# 5: Cached price
# 6: Last interest rate
# 2^160 + k: interest rate accumulator at k epochs
# 2^161 + ADDR * 3: eth-balance of a particular address
# 2^161 + ADDR * 3 + 1: usd-balance of a particular address
# 2^161 + ADDR * 3 + 1: last accessed epoch of a particular address
#
# Transaction types:
#
# [1, to, val]: send ETH
# [2, to, val]: send USD
# [3, wei_amount]: convert ETH to USD
# [4, usd_amount]: converts USD to ETH
# [5]: deposit
# [6, amount]: withdraw
# [7]: my balance query
# [7, acct]: balance query for any acct
# [8]: global state query
# [9]: liquidation test any account
#
# The purpose of the contract is to serve as a sort of cryptographic
# bank account where users can store both ETH and USD. ETH must be
# stored in zero or positive quantities, but USD balances can be
# positive or negative. If the USD balance is negative, the invariant
# usdbal * 10 >= ethbal * 9 must be satisfied; if any account falls
# below this value, then that account's balances are zeroed. Note
# that there is a 2% bounty to ping the app if an account does go
# below zero; one weakness is that if no one does ping then it is
# quite possible for accounts to go negative-net-worth, then zero
# themselves out, draining the reserves of the "bank" and potentially
# bankrupting it. A 0.1% fee on ETH <-> USD trade is charged to
# minimize this risk. Additionally, the bank itself will inevitably
# end up with positive or negative USD exposure; to mitigate this,
# it automatically updates interest rates on USD to keep exposure
# near zero.
data schelling_coin
data last_epoch
data starting_block
data usd_exposure
data eth_exposure
data price
data last_interest_rate
data interest_rate_accum[2^50]
data accounts[2^160](eth, usd, last_epoch)
extern sc: [submit_hash, submit_value, request_balance, request_output]
def init():
self.schelling_coin = create('schellingcoin.se')
self.price = self.schelling_coin.request_output()
self.interest_rate_accum[0] = 10^18
self.starting_block = block.number
def any():
sender = msg.sender
epoch = (block.number - self.starting_block) / 100
last_epoch = self.last_epoch
usdprice = self.price
# Update contract epochs
if epoch > last_epoch:
delta = epoch - last_epoch
last_interest_rate = self.last_interest_rate
usd_exposure - self.usd_exposure
last_accum = self.interest_rate_accum[last_epoch]
if usd_exposure < 0:
self.last_interest_rate = last_interest_rate - 10000 * delta
elif usd_exposure > 0:
self.last_interest_rate = last_interest_rate + 10000 * delta
self.interest_rate_accum[epoch] = last_accum + last_accum * last_interest_rate * delta / 10^9
# Proceeds go to support the SchellingCoin feeding it price data, ultimately providing the depositors
# of the SchellingCoin an interest rate
bal = max(self.balance - self.eth_exposure, 0) / 10000
usdprice = self.schelling_coin.request_output()
self.price = usdprice
self.last_epoch = epoch
ethbal = self.accounts[msg.sender].eth
usdbal = self.accounts[msg.sender].usd
# Apply interest rates to sender and liquidation-test self
if msg.sender != self:
self.ping(self)
def send_eth(to, value):
if value > 0 and value <= ethbal and usdbal * usdprice * 2 + (ethbal - value) >= 0:
self.accounts[msg.sender].eth = ethbal - value
self.ping(to)
self.accounts[to].eth += value
return(1)
def send_usd(to, value):
if value > 0 and value <= usdbal and (usdbal - value) * usdprice * 2 + ethbal >= 0:
self.accounts[msg.sender].usd = usdbal - value
self.ping(to)
self.accounts[to].usd += value
return(1)
def convert_to_eth(usdvalue):
ethplus = usdvalue * usdprice * 999 / 1000
if usdvalue > 0 and (usdbal - usdvalue) * usdprice * 2 + (ethbal + ethplus) >= 0:
self.accounts[msg.sender].eth = ethbal + ethplus
self.accounts[msg.sender].usd = usdbal - usdvalue
self.eth_exposure += ethplus
self.usd_exposure -= usdvalue
return([ethbal + ethplus, usdbal - usdvalue], 2)
def convert_to_usd(ethvalue):
usdplus = ethvalue / usdprice * 999 / 1000
if ethvalue > 0 and (usdbal + usdplus) * usdprice * 2 + (ethbal - ethvalue) >= 0:
self.accounts[msg.sender].eth = ethbal - ethvalue
self.accounts[msg.sender].usd = usdbal + usdplus
self.eth_exposure -= ethvalue
self.usd_exposure += usdplus
return([ethbal - ethvalue, usdbal + usdplus], 2)
def deposit():
self.accounts[msg.sender].eth = ethbal + msg.value
self.eth_exposure += msg.value
return(ethbal + msg.value)
def withdraw(value):
if value > 0 and value <= ethbal and usdbal * usdprice * 2 + (ethbal - value) >= 0:
self.accounts[msg.sender].eth -= value
self.eth_exposure -= value
return(ethbal - value)
def balance(acct):
self.ping(acct)
return([self.accounts[acct].eth, self.accounts[acct].usd], 2)
def global_state_query(acct):
interest = self.last_interest_rate
usd_exposure = self.usd_exposure
eth_exposure = self.eth_exposure
eth_balance = self.balance
return([epoch, usdprice, interest, usd_exposure, eth_exposure, eth_balance], 6)
def ping(acct):
account_last_epoch = self.accounts[acct].last_epoch
if account_last_epoch != epoch:
cur_usd_balance = self.accounts[acct].usd
new_usd_balance = cur_usd_balance * self.interest_rate_accum[epoch] / self.interest_rate_accum[account_last_epoch]
self.accounts[acct].usd = new_usd_balance
self.accounts[acct].last_epoch = epoch
self.usd_exposure += new_usd_balance - cur_usd_balance
ethbal = self.accounts[acct].eth
if new_usd_balance * usdval * 10 + ethbal * 9 < 0:
self.accounts[acct].eth = 0
self.accounts[acct].usd = 0
self.accounts[msg.sender].eth += ethbal / 50
self.eth_exposure += -ethbal + ethbal / 50
self.usd_exposure += new_usd_balance
return(1)
return(0)

1
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellinghelper.se generated vendored
View File

@ -1 +0,0 @@
return(sha3([msg.sender, msg.data[0]], 2))

3
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/short_namecoin.se generated vendored
View File

@ -1,3 +0,0 @@
def register(k, v):
if !self.storage[k]:
self.storage[k] = v

11
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/subcurrency.se generated vendored
View File

@ -1,11 +0,0 @@
def init():
self.storage[msg.sender] = 1000000
def balance_query(k):
return(self.storage[addr])
def send(to, value):
fromvalue = self.storage[msg.sender]
if fromvalue >= value:
self.storage[from] = fromvalue - value
self.storage[to] += value

35
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/funcs.cpp generated vendored
View File

@ -1,35 +0,0 @@
#include <stdio.h>
#include <iostream>
#include <vector>
#include "funcs.h"
#include "bignum.h"
#include "util.h"
#include "parser.h"
#include "lllparser.h"
#include "compiler.h"
#include "rewriter.h"
#include "tokenize.h"
Node compileToLLL(std::string input) {
return rewrite(parseSerpent(input));
}
Node compileChunkToLLL(std::string input) {
return rewriteChunk(parseSerpent(input));
}
std::string compile(std::string input) {
return compileLLL(compileToLLL(input));
}
std::vector<Node> prettyCompile(std::string input) {
return prettyCompileLLL(compileToLLL(input));
}
std::string compileChunk(std::string input) {
return compileLLL(compileChunkToLLL(input));
}
std::vector<Node> prettyCompileChunk(std::string input) {
return prettyCompileLLL(compileChunkToLLL(input));
}

35
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/funcs.h generated vendored
View File

@ -1,35 +0,0 @@
#include <stdio.h>
#include <iostream>
#include <vector>
#include "bignum.h"
#include "util.h"
#include "parser.h"
#include "lllparser.h"
#include "compiler.h"
#include "rewriter.h"
#include "tokenize.h"
// Function listing:
//
// parseSerpent (serpent -> AST) std::string -> Node
// parseLLL (LLL -> AST) std::string -> Node
// rewrite (apply rewrite rules) Node -> Node
// compileToLLL (serpent -> LLL) std::string -> Node
// compileLLL (LLL -> EVMhex) Node -> std::string
// prettyCompileLLL (LLL -> EVMasm) Node -> std::vector<Node>
// prettyCompile (serpent -> EVMasm) std::string -> std::vector>Node>
// compile (serpent -> EVMhex) std::string -> std::string
// get_file_contents (filename -> file) std::string -> std::string
// exists (does file exist?) std::string -> bool
Node compileToLLL(std::string input);
Node compileChunkToLLL(std::string input);
std::string compile(std::string input);
std::vector<Node> prettyCompile(std::string input);
std::string compileChunk(std::string input);
std::vector<Node> prettyCompileChunk(std::string input);

203
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/functions.cpp generated vendored
View File

@ -1,203 +0,0 @@
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
#include "lllparser.h"
#include "bignum.h"
#include "optimize.h"
#include "rewriteutils.h"
#include "preprocess.h"
#include "functions.h"
std::string getSignature(std::vector<Node> args) {
std::string o;
for (unsigned i = 0; i < args.size(); i++) {
if (args[i].val == ":" && args[i].args[1].val == "s")
o += "s";
else if (args[i].val == ":" && args[i].args[1].val == "a")
o += "a";
else
o += "i";
}
return o;
}
// Convert a list of arguments into a node containing a
// < datastart, datasz > pair
Node packArguments(std::vector<Node> args, std::string sig,
int funId, Metadata m) {
// Plain old 32 byte arguments
std::vector<Node> nargs;
// Variable-sized arguments
std::vector<Node> vargs;
// Variable sizes
std::vector<Node> sizes;
// Is a variable an array?
std::vector<bool> isArray;
// Fill up above three argument lists
int argCount = 0;
for (unsigned i = 0; i < args.size(); i++) {
Metadata m = args[i].metadata;
if (args[i].val == "=") {
// do nothing
}
else {
// Determine the correct argument type
char argType;
if (sig.size() > 0) {
if (argCount >= (signed)sig.size())
err("Too many args", m);
argType = sig[argCount];
}
else argType = 'i';
// Integer (also usable for short strings)
if (argType == 'i') {
if (args[i].val == ":")
err("Function asks for int, provided string or array", m);
nargs.push_back(args[i]);
}
// Long string
else if (argType == 's') {
if (args[i].val != ":")
err("Must specify string length", m);
vargs.push_back(args[i].args[0]);
sizes.push_back(args[i].args[1]);
isArray.push_back(false);
}
// Array
else if (argType == 'a') {
if (args[i].val != ":")
err("Must specify array length", m);
vargs.push_back(args[i].args[0]);
sizes.push_back(args[i].args[1]);
isArray.push_back(true);
}
else err("Invalid arg type in signature", m);
argCount++;
}
}
int static_arg_size = 1 + (vargs.size() + nargs.size()) * 32;
// Start off by saving the size variables and calculating the total
msn kwargs;
kwargs["funid"] = tkn(utd(funId), m);
std::string pattern =
"(with _sztot "+utd(static_arg_size)+" "
" (with _sizes (alloc "+utd(sizes.size() * 32)+") "
" (seq ";
for (unsigned i = 0; i < sizes.size(); i++) {
std::string sizeIncrement =
isArray[i] ? "(mul 32 _x)" : "_x";
pattern +=
"(with _x $sz"+utd(i)+"(seq "
" (mstore (add _sizes "+utd(i * 32)+") _x) "
" (set _sztot (add _sztot "+sizeIncrement+" )))) ";
kwargs["sz"+utd(i)] = sizes[i];
}
// Allocate memory, and set first data byte
pattern +=
"(with _datastart (alloc (add _sztot 32)) (seq "
" (mstore8 _datastart $funid) ";
// Copy over size variables
for (unsigned i = 0; i < sizes.size(); i++) {
int v = 1 + i * 32;
pattern +=
" (mstore "
" (add _datastart "+utd(v)+") "
" (mload (add _sizes "+utd(v-1)+"))) ";
}
// Store normal arguments
for (unsigned i = 0; i < nargs.size(); i++) {
int v = 1 + (i + sizes.size()) * 32;
pattern +=
" (mstore (add _datastart "+utd(v)+") $"+utd(i)+") ";
kwargs[utd(i)] = nargs[i];
}
// Loop through variable-sized arguments, store them
pattern +=
" (with _pos (add _datastart "+utd(static_arg_size)+") (seq";
for (unsigned i = 0; i < vargs.size(); i++) {
std::string copySize =
isArray[i] ? "(mul 32 (mload (add _sizes "+utd(i * 32)+")))"
: "(mload (add _sizes "+utd(i * 32)+"))";
pattern +=
" (unsafe_mcopy _pos $vl"+utd(i)+" "+copySize+") "
" (set _pos (add _pos "+copySize+")) ";
kwargs["vl"+utd(i)] = vargs[i];
}
// Return a 2-item array containing the start and size
pattern += " (array_lit _datastart _sztot))))))))";
std::string prefix = "_temp_"+mkUniqueToken();
// Fill in pattern, return triple
return subst(parseLLL(pattern), kwargs, prefix, m);
}
// Create a node for argument unpacking
Node unpackArguments(std::vector<Node> vars, Metadata m) {
std::vector<std::string> varNames;
std::vector<std::string> longVarNames;
std::vector<bool> longVarIsArray;
// Fill in variable and long variable names, as well as which
// long variables are arrays and which are strings
for (unsigned i = 0; i < vars.size(); i++) {
if (vars[i].val == ":") {
if (vars[i].args.size() != 2)
err("Malformed def!", m);
longVarNames.push_back(vars[i].args[0].val);
std::string tag = vars[i].args[1].val;
if (tag == "s")
longVarIsArray.push_back(false);
else if (tag == "a")
longVarIsArray.push_back(true);
else
err("Function value can only be string or array", m);
}
else {
varNames.push_back(vars[i].val);
}
}
std::vector<Node> sub;
if (!varNames.size() && !longVarNames.size()) {
// do nothing if we have no arguments
}
else {
std::vector<Node> varNodes;
for (unsigned i = 0; i < longVarNames.size(); i++)
varNodes.push_back(token(longVarNames[i], m));
for (unsigned i = 0; i < varNames.size(); i++)
varNodes.push_back(token(varNames[i], m));
// Copy over variable lengths and short variables
for (unsigned i = 0; i < varNodes.size(); i++) {
int pos = 1 + i * 32;
std::string prefix = (i < longVarNames.size()) ? "_len_" : "";
sub.push_back(asn("untyped", asn("set",
token(prefix+varNodes[i].val, m),
asn("calldataload", tkn(utd(pos), m), m),
m)));
}
// Copy over long variables
if (longVarNames.size() > 0) {
std::vector<Node> sub2;
int pos = varNodes.size() * 32 + 1;
Node tot = tkn("_tot", m);
for (unsigned i = 0; i < longVarNames.size(); i++) {
Node var = tkn(longVarNames[i], m);
Node varlen = longVarIsArray[i]
? asn("mul", tkn("32", m), tkn("_len_"+longVarNames[i], m))
: tkn("_len_"+longVarNames[i], m);
sub2.push_back(asn("untyped",
asn("set", var, asn("alloc", varlen))));
sub2.push_back(asn("calldatacopy", var, tot, varlen));
sub2.push_back(asn("set", tot, asn("add", tot, varlen)));
}
std::string prefix = "_temp_"+mkUniqueToken();
sub.push_back(subst(
astnode("with", tot, tkn(utd(pos), m), asn("seq", sub2)),
msn(),
prefix,
m));
}
}
return asn("seq", sub, m);
}

39
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/functions.h generated vendored
View File

@ -1,39 +0,0 @@
#ifndef ETHSERP_FUNCTIONS
#define ETHSERP_FUNCTIONS
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
#include "lllparser.h"
#include "bignum.h"
#include "optimize.h"
#include "rewriteutils.h"
#include "preprocess.h"
class argPack {
public:
argPack(Node a, Node b, Node c) {
pre = a;
datastart = b;
datasz = c;
}
Node pre;
Node datastart;
Node datasz;
};
// Get a signature from a function
std::string getSignature(std::vector<Node> args);
// Convert a list of arguments into a <pre, mstart, msize> node
// triple, given the signature of a function
Node packArguments(std::vector<Node> args, std::string sig,
int funId, Metadata m);
// Create a node for argument unpacking
Node unpackArguments(std::vector<Node> vars, Metadata m);
#endif

70
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/lllparser.cpp generated vendored
View File

@ -1,70 +0,0 @@
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
#include "lllparser.h"
#include "tokenize.h"
struct _parseOutput {
Node node;
int newpos;
};
// Helper, returns subtree and position of start of next node
_parseOutput _parse(std::vector<Node> inp, int pos) {
Metadata met = inp[pos].metadata;
_parseOutput o;
// Bracket: keep grabbing tokens until we get to the
// corresponding closing bracket
if (inp[pos].val == "(" || inp[pos].val == "[") {
std::string fun, rbrack;
std::vector<Node> args;
pos += 1;
if (inp[pos].val == "[") {
fun = "access";
rbrack = "]";
}
else rbrack = ")";
// First argument is the function
while (inp[pos].val != ")") {
_parseOutput po = _parse(inp, pos);
if (fun.length() == 0 && po.node.type == 1) {
std::cerr << "Error: first arg must be function\n";
fun = po.node.val;
}
else if (fun.length() == 0) {
fun = po.node.val;
}
else {
args.push_back(po.node);
}
pos = po.newpos;
}
o.newpos = pos + 1;
o.node = astnode(fun, args, met);
}
// Normal token, return it and advance to next token
else {
o.newpos = pos + 1;
o.node = token(inp[pos].val, met);
}
return o;
}
// stream of tokens -> lisp parse tree
Node parseLLLTokenStream(std::vector<Node> inp) {
_parseOutput o = _parse(inp, 0);
return o.node;
}
// Parses LLL
Node parseLLL(std::string s, bool allowFileRead) {
std::string input = s;
std::string file = "main";
if (exists(s) && allowFileRead) {
file = s;
input = get_file_contents(s);
}
return parseLLLTokenStream(tokenize(s, Metadata(file, 0, 0), true));
}

13
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/lllparser.h generated vendored
View File

@ -1,13 +0,0 @@
#ifndef ETHSERP_LLLPARSER
#define ETHSERP_LLLPARSER
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
// LLL text -> parse tree
Node parseLLL(std::string s, bool allowFileRead=false);
#endif

154
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/opcodes.cpp generated vendored
View File

@ -1,154 +0,0 @@
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "opcodes.h"
#include "util.h"
#include "bignum.h"
Mapping mapping[] = {
Mapping("STOP", 0x00, 0, 0),
Mapping("ADD", 0x01, 2, 1),
Mapping("MUL", 0x02, 2, 1),
Mapping("SUB", 0x03, 2, 1),
Mapping("DIV", 0x04, 2, 1),
Mapping("SDIV", 0x05, 2, 1),
Mapping("MOD", 0x06, 2, 1),
Mapping("SMOD", 0x07, 2, 1),
Mapping("ADDMOD", 0x08, 3, 1),
Mapping("MULMOD", 0x09, 3, 1),
Mapping("EXP", 0x0a, 2, 1),
Mapping("SIGNEXTEND", 0x0b, 2, 1),
Mapping("LT", 0x10, 2, 1),
Mapping("GT", 0x11, 2, 1),
Mapping("SLT", 0x12, 2, 1),
Mapping("SGT", 0x13, 2, 1),
Mapping("EQ", 0x14, 2, 1),
Mapping("ISZERO", 0x15, 1, 1),
Mapping("AND", 0x16, 2, 1),
Mapping("OR", 0x17, 2, 1),
Mapping("XOR", 0x18, 2, 1),
Mapping("NOT", 0x19, 1, 1),
Mapping("BYTE", 0x1a, 2, 1),
Mapping("SHA3", 0x20, 2, 1),
Mapping("ADDRESS", 0x30, 0, 1),
Mapping("BALANCE", 0x31, 1, 1),
Mapping("ORIGIN", 0x32, 0, 1),
Mapping("CALLER", 0x33, 0, 1),
Mapping("CALLVALUE", 0x34, 0, 1),
Mapping("CALLDATALOAD", 0x35, 1, 1),
Mapping("CALLDATASIZE", 0x36, 0, 1),
Mapping("CALLDATACOPY", 0x37, 3, 0),
Mapping("CODESIZE", 0x38, 0, 1),
Mapping("CODECOPY", 0x39, 3, 0),
Mapping("GASPRICE", 0x3a, 0, 1),
Mapping("EXTCODESIZE", 0x3b, 1, 1),
Mapping("EXTCODECOPY", 0x3c, 4, 0),
Mapping("PREVHASH", 0x40, 0, 1),
Mapping("COINBASE", 0x41, 0, 1),
Mapping("TIMESTAMP", 0x42, 0, 1),
Mapping("NUMBER", 0x43, 0, 1),
Mapping("DIFFICULTY", 0x44, 0, 1),
Mapping("GASLIMIT", 0x45, 0, 1),
Mapping("POP", 0x50, 1, 0),
Mapping("MLOAD", 0x51, 1, 1),
Mapping("MSTORE", 0x52, 2, 0),
Mapping("MSTORE8", 0x53, 2, 0),
Mapping("SLOAD", 0x54, 1, 1),
Mapping("SSTORE", 0x55, 2, 0),
Mapping("JUMP", 0x56, 1, 0),
Mapping("JUMPI", 0x57, 2, 0),
Mapping("PC", 0x58, 0, 1),
Mapping("MSIZE", 0x59, 0, 1),
Mapping("GAS", 0x5a, 0, 1),
Mapping("JUMPDEST", 0x5b, 0, 0),
Mapping("LOG0", 0xa0, 2, 0),
Mapping("LOG1", 0xa1, 3, 0),
Mapping("LOG2", 0xa2, 4, 0),
Mapping("LOG3", 0xa3, 5, 0),
Mapping("LOG4", 0xa4, 6, 0),
Mapping("CREATE", 0xf0, 3, 1),
Mapping("CALL", 0xf1, 7, 1),
Mapping("CALLCODE", 0xf2, 7, 1),
Mapping("RETURN", 0xf3, 2, 0),
Mapping("SUICIDE", 0xff, 1, 0),
Mapping("---END---", 0x00, 0, 0),
};
std::map<std::string, std::vector<int> > opcodes;
std::map<int, std::string> reverseOpcodes;
// Fetches everything EXCEPT PUSH1..32
std::pair<std::string, std::vector<int> > _opdata(std::string ops, int opi) {
if (!opcodes.size()) {
int i = 0;
while (mapping[i].op != "---END---") {
Mapping mi = mapping[i];
opcodes[mi.op] = triple(mi.opcode, mi.in, mi.out);
i++;
}
for (i = 1; i <= 16; i++) {
opcodes["DUP"+unsignedToDecimal(i)] = triple(0x7f + i, i, i+1);
opcodes["SWAP"+unsignedToDecimal(i)] = triple(0x8f + i, i+1, i+1);
}
for (std::map<std::string, std::vector<int> >::iterator it=opcodes.begin();
it != opcodes.end();
it++) {
reverseOpcodes[(*it).second[0]] = (*it).first;
}
}
ops = upperCase(ops);
std::string op;
std::vector<int> opdata;
op = reverseOpcodes.count(opi) ? reverseOpcodes[opi] : "";
opdata = opcodes.count(ops) ? opcodes[ops] : triple(-1, -1, -1);
return std::pair<std::string, std::vector<int> >(op, opdata);
}
int opcode(std::string op) {
return _opdata(op, -1).second[0];
}
int opinputs(std::string op) {
return _opdata(op, -1).second[1];
}
int opoutputs(std::string op) {
return _opdata(op, -1).second[2];
}
std::string op(int opcode) {
return _opdata("", opcode).first;
}
std::string lllSpecials[][3] = {
{ "ref", "1", "1" },
{ "get", "1", "1" },
{ "set", "2", "2" },
{ "with", "3", "3" },
{ "comment", "0", "2147483647" },
{ "ops", "0", "2147483647" },
{ "lll", "2", "2" },
{ "seq", "0", "2147483647" },
{ "if", "3", "3" },
{ "unless", "2", "2" },
{ "until", "2", "2" },
{ "alloc", "1", "1" },
{ "---END---", "0", "0" },
};
std::map<std::string, std::pair<int, int> > lllMap;
// Is a function name one of the valid functions above?
bool isValidLLLFunc(std::string f, int argc) {
if (lllMap.size() == 0) {
for (int i = 0; ; i++) {
if (lllSpecials[i][0] == "---END---") break;
lllMap[lllSpecials[i][0]] = std::pair<int, int>(
dtu(lllSpecials[i][1]), dtu(lllSpecials[i][2]));
}
}
return lllMap.count(f)
&& argc >= lllMap[f].first
&& argc <= lllMap[f].second;
}

45
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/opcodes.h generated vendored
View File

@ -1,45 +0,0 @@
#ifndef ETHSERP_OPCODES
#define ETHSERP_OPCODES
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
class Mapping {
public:
Mapping(std::string Op, int Opcode, int In, int Out) {
op = Op;
opcode = Opcode;
in = In;
out = Out;
}
std::string op;
int opcode;
int in;
int out;
};
extern Mapping mapping[];
extern std::map<std::string, std::vector<int> > opcodes;
extern std::map<int, std::string> reverseOpcodes;
std::pair<std::string, std::vector<int> > _opdata(std::string ops, int opi);
int opcode(std::string op);
int opinputs(std::string op);
int opoutputs(std::string op);
std::string op(int opcode);
extern std::string lllSpecials[][3];
extern std::map<std::string, std::pair<int, int> > lllMap;
bool isValidLLLFunc(std::string f, int argc);
#endif

98
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/optimize.cpp generated vendored
View File

@ -1,98 +0,0 @@
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
#include "lllparser.h"
#include "bignum.h"
// Compile-time arithmetic calculations
Node optimize(Node inp) {
if (inp.type == TOKEN) {
Node o = tryNumberize(inp);
if (decimalGt(o.val, tt256, true))
err("Value too large (exceeds 32 bytes or 2^256)", inp.metadata);
return o;
}
for (unsigned i = 0; i < inp.args.size(); i++) {
inp.args[i] = optimize(inp.args[i]);
}
// Arithmetic-specific transform
if (inp.val == "+") inp.val = "add";
if (inp.val == "*") inp.val = "mul";
if (inp.val == "-") inp.val = "sub";
if (inp.val == "/") inp.val = "sdiv";
if (inp.val == "^") inp.val = "exp";
if (inp.val == "**") inp.val = "exp";
if (inp.val == "%") inp.val = "smod";
// Degenerate cases for add and mul
if (inp.args.size() == 2) {
if (inp.val == "add" && inp.args[0].type == TOKEN &&
inp.args[0].val == "0") {
Node x = inp.args[1];
inp = x;
}
if (inp.val == "add" && inp.args[1].type == TOKEN &&
inp.args[1].val == "0") {
Node x = inp.args[0];
inp = x;
}
if (inp.val == "mul" && inp.args[0].type == TOKEN &&
inp.args[0].val == "1") {
Node x = inp.args[1];
inp = x;
}
if (inp.val == "mul" && inp.args[1].type == TOKEN &&
inp.args[1].val == "1") {
Node x = inp.args[0];
inp = x;
}
}
// Arithmetic computation
if (inp.args.size() == 2
&& inp.args[0].type == TOKEN
&& inp.args[1].type == TOKEN) {
std::string o;
if (inp.val == "add") {
o = decimalMod(decimalAdd(inp.args[0].val, inp.args[1].val), tt256);
}
else if (inp.val == "sub") {
if (decimalGt(inp.args[0].val, inp.args[1].val, true))
o = decimalSub(inp.args[0].val, inp.args[1].val);
}
else if (inp.val == "mul") {
o = decimalMod(decimalMul(inp.args[0].val, inp.args[1].val), tt256);
}
else if (inp.val == "div" && inp.args[1].val != "0") {
o = decimalDiv(inp.args[0].val, inp.args[1].val);
}
else if (inp.val == "sdiv" && inp.args[1].val != "0"
&& decimalGt(tt255, inp.args[0].val)
&& decimalGt(tt255, inp.args[1].val)) {
o = decimalDiv(inp.args[0].val, inp.args[1].val);
}
else if (inp.val == "mod" && inp.args[1].val != "0") {
o = decimalMod(inp.args[0].val, inp.args[1].val);
}
else if (inp.val == "smod" && inp.args[1].val != "0"
&& decimalGt(tt255, inp.args[0].val)
&& decimalGt(tt255, inp.args[1].val)) {
o = decimalMod(inp.args[0].val, inp.args[1].val);
}
else if (inp.val == "exp") {
o = decimalModExp(inp.args[0].val, inp.args[1].val, tt256);
}
if (o.length()) return token(o, inp.metadata);
}
return inp;
}
// Is a node degenerate (ie. trivial to calculate) ?
bool isDegenerate(Node n) {
return optimize(n).type == TOKEN;
}
// Is a node purely arithmetic?
bool isPureArithmetic(Node n) {
return isNumberLike(optimize(n));
}

19
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/optimize.h generated vendored
View File

@ -1,19 +0,0 @@
#ifndef ETHSERP_OPTIMIZER
#define ETHSERP_OPTIMIZER
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
// Compile-time arithmetic calculations
Node optimize(Node inp);
// Is a node degenerate (ie. trivial to calculate) ?
bool isDegenerate(Node n);
// Is a node purely arithmetic?
bool isPureArithmetic(Node n);
#endif

430
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/parser.cpp generated vendored
View File

@ -1,430 +0,0 @@
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
#include "parser.h"
#include "tokenize.h"
// Extended BEDMAS precedence order
int precedence(Node tok) {
std::string v = tok.val;
if (v == ".") return -1;
else if (v == "!" || v == "not") return 1;
else if (v=="^" || v == "**") return 2;
else if (v=="*" || v=="/" || v=="%") return 3;
else if (v=="+" || v=="-") return 4;
else if (v=="<" || v==">" || v=="<=" || v==">=") return 5;
else if (v=="&" || v=="|" || v=="xor" || v=="==" || v == "!=") return 6;
else if (v=="&&" || v=="and") return 7;
else if (v=="||" || v=="or") return 8;
else if (v=="=") return 10;
else if (v=="+=" || v=="-=" || v=="*=" || v=="/=" || v=="%=") return 10;
else if (v==":" || v == "::") return 11;
else return 0;
}
// Token classification for shunting-yard purposes
int toktype(Node tok) {
if (tok.type == ASTNODE) return COMPOUND;
std::string v = tok.val;
if (v == "(" || v == "[" || v == "{") return LPAREN;
else if (v == ")" || v == "]" || v == "}") return RPAREN;
else if (v == ",") return COMMA;
else if (v == "!" || v == "~" || v == "not") return UNARY_OP;
else if (precedence(tok) > 0) return BINARY_OP;
else if (precedence(tok) < 0) return TOKEN_SPLITTER;
if (tok.val[0] != '"' && tok.val[0] != '\'') {
for (unsigned i = 0; i < tok.val.length(); i++) {
if (chartype(tok.val[i]) == SYMB) {
err("Invalid symbol: "+tok.val, tok.metadata);
}
}
}
return ALPHANUM;
}
// Converts to reverse polish notation
std::vector<Node> shuntingYard(std::vector<Node> tokens) {
std::vector<Node> iq;
for (int i = tokens.size() - 1; i >= 0; i--) {
iq.push_back(tokens[i]);
}
std::vector<Node> oq;
std::vector<Node> stack;
Node prev, tok;
int prevtyp = 0, toktyp = 0;
while (iq.size()) {
prev = tok;
prevtyp = toktyp;
tok = iq.back();
toktyp = toktype(tok);
iq.pop_back();
// Alphanumerics go straight to output queue
if (toktyp == ALPHANUM) {
oq.push_back(tok);
}
// Left parens go on stack and output queue
else if (toktyp == LPAREN) {
while (stack.size() && toktype(stack.back()) == TOKEN_SPLITTER) {
oq.push_back(stack.back());
stack.pop_back();
}
if (prevtyp != ALPHANUM && prevtyp != RPAREN) {
oq.push_back(token("id", tok.metadata));
}
stack.push_back(tok);
oq.push_back(tok);
}
// If rparen, keep moving from stack to output queue until lparen
else if (toktyp == RPAREN) {
while (stack.size() && toktype(stack.back()) != LPAREN) {
oq.push_back(stack.back());
stack.pop_back();
}
if (stack.size()) {
stack.pop_back();
}
oq.push_back(tok);
}
else if (toktyp == UNARY_OP) {
stack.push_back(tok);
}
// If token splitter, just push it to the stack
else if (toktyp == TOKEN_SPLITTER) {
while (stack.size() && toktype(stack.back()) == TOKEN_SPLITTER) {
oq.push_back(stack.back());
stack.pop_back();
}
stack.push_back(tok);
}
// If binary op, keep popping from stack while higher bedmas precedence
else if (toktyp == BINARY_OP) {
if (tok.val == "-" && prevtyp != ALPHANUM && prevtyp != RPAREN) {
stack.push_back(tok);
oq.push_back(token("0", tok.metadata));
}
else {
int prec = precedence(tok);
while (stack.size()
&& (toktype(stack.back()) == BINARY_OP
|| toktype(stack.back()) == UNARY_OP
|| toktype(stack.back()) == TOKEN_SPLITTER)
&& precedence(stack.back()) <= prec) {
oq.push_back(stack.back());
stack.pop_back();
}
stack.push_back(tok);
}
}
// Comma means finish evaluating the argument
else if (toktyp == COMMA) {
while (stack.size() && toktype(stack.back()) != LPAREN) {
oq.push_back(stack.back());
stack.pop_back();
}
}
}
while (stack.size()) {
oq.push_back(stack.back());
stack.pop_back();
}
return oq;
}
// Converts reverse polish notation into tree
Node treefy(std::vector<Node> stream) {
std::vector<Node> iq;
for (int i = stream.size() -1; i >= 0; i--) {
iq.push_back(stream[i]);
}
std::vector<Node> oq;
while (iq.size()) {
Node tok = iq.back();
iq.pop_back();
int typ = toktype(tok);
// If unary, take node off end of oq and wrap it with the operator
// If binary, do the same with two nodes
if (typ == UNARY_OP || typ == BINARY_OP || typ == TOKEN_SPLITTER) {
std::vector<Node> args;
int rounds = (typ == UNARY_OP) ? 1 : 2;
for (int i = 0; i < rounds; i++) {
if (oq.size() == 0) {
err("Line malformed, not enough args for "+tok.val,
tok.metadata);
}
args.push_back(oq.back());
oq.pop_back();
}
std::vector<Node> args2;
while (args.size()) {
args2.push_back(args.back());
args.pop_back();
}
oq.push_back(astnode(tok.val, args2, tok.metadata));
}
// If rparen, keep grabbing until we get to an lparen
else if (typ == RPAREN) {
std::vector<Node> args;
while (1) {
if (toktype(oq.back()) == LPAREN) break;
args.push_back(oq.back());
oq.pop_back();
if (!oq.size()) err("Bracket without matching", tok.metadata);
}
oq.pop_back();
args.push_back(oq.back());
oq.pop_back();
// We represent a[b] as (access a b)
if (tok.val == "]")
args.push_back(token("access", tok.metadata));
if (args.back().type == ASTNODE)
args.push_back(token("fun", tok.metadata));
std::string fun = args.back().val;
args.pop_back();
// We represent [1,2,3] as (array_lit 1 2 3)
if (fun == "access" && args.size() && args.back().val == "id") {
fun = "array_lit";
args.pop_back();
}
std::vector<Node> args2;
while (args.size()) {
args2.push_back(args.back());
args.pop_back();
}
// When evaluating 2 + (3 * 5), the shunting yard algo turns that
// into 2 ( id 3 5 * ) +, effectively putting "id" as a dummy
// function where the algo was expecting a function to call the
// thing inside the brackets. This reverses that step
if (fun == "id" && args2.size() == 1) {
oq.push_back(args2[0]);
}
else {
oq.push_back(astnode(fun, args2, tok.metadata));
}
}
else oq.push_back(tok);
// This is messy, but has to be done. Import/inset other files here
std::string v = oq.back().val;
if ((v == "inset" || v == "import" || v == "create")
&& oq.back().args.size() == 1
&& oq.back().args[0].type == TOKEN) {
int lastSlashPos = tok.metadata.file.rfind("/");
std::string root;
if (lastSlashPos >= 0)
root = tok.metadata.file.substr(0, lastSlashPos) + "/";
else
root = "";
std::string filename = oq.back().args[0].val;
filename = filename.substr(1, filename.length() - 2);
if (!exists(root + filename))
err("File does not exist: "+root + filename, tok.metadata);
oq.back().args.pop_back();
oq.back().args.push_back(parseSerpent(root + filename));
}
//Useful for debugging
//for (int i = 0; i < oq.size(); i++) {
// std::cerr << printSimple(oq[i]) << " ";
//}
//std::cerr << " <-\n";
}
// Output must have one argument
if (oq.size() == 0) {
err("Output blank", Metadata());
}
else if (oq.size() > 1) {
return asn("multi", oq, oq[0].metadata);
}
return oq[0];
}
// Parses one line of serpent
Node parseSerpentTokenStream(std::vector<Node> s) {
return treefy(shuntingYard(s));
}
// Count spaces at beginning of line
int spaceCount(std::string s) {
unsigned pos = 0;
while (pos < s.length() && (s[pos] == ' ' || s[pos] == '\t'))
pos++;
return pos;
}
// Is this a command that takes an argument on the same line?
bool bodied(std::string tok) {
return tok == "if" || tok == "elif" || tok == "while"
|| tok == "with" || tok == "def" || tok == "extern"
|| tok == "data" || tok == "assert" || tok == "return"
|| tok == "fun" || tok == "scope" || tok == "macro"
|| tok == "type";
}
// Are the two commands meant to continue each other?
bool bodiedContinued(std::string prev, std::string tok) {
return (prev == "if" && tok == "elif")
|| (prev == "elif" && tok == "else")
|| (prev == "elif" && tok == "elif")
|| (prev == "if" && tok == "else");
}
// Is a line of code empty?
bool isLineEmpty(std::string line) {
std::vector<Node> tokens = tokenize(line);
if (!tokens.size() || tokens[0].val == "#" || tokens[0].val == "//")
return true;
return false;
}
// Parse lines of serpent (helper function)
Node parseLines(std::vector<std::string> lines, Metadata metadata, int sp) {
std::vector<Node> o;
int origLine = metadata.ln;
unsigned i = 0;
while (i < lines.size()) {
metadata.ln = origLine + i;
std::string main = lines[i];
if (isLineEmpty(main)) {
i += 1;
continue;
}
int spaces = spaceCount(main);
if (spaces != sp) {
err("Indent mismatch", metadata);
}
// Tokenize current line
std::vector<Node> tokens = tokenize(main.substr(sp), metadata);
// Remove comments
std::vector<Node> tokens2;
for (unsigned j = 0; j < tokens.size(); j++) {
if (tokens[j].val == "#" || tokens[j].val == "//") break;
tokens2.push_back(tokens[j]);
}
bool expectingChildBlock = false;
if (tokens2.size() > 0 && tokens2.back().val == ":") {
tokens2.pop_back();
expectingChildBlock = true;
}
// Parse current line
Node out = parseSerpentTokenStream(tokens2);
// Parse child block
int childIndent = 999999;
std::vector<std::string> childBlock;
while (1) {
i++;
if (i >= lines.size())
break;
bool ile = isLineEmpty(lines[i]);
if (!ile) {
int spaces = spaceCount(lines[i]);
if (spaces <= sp) break;
childBlock.push_back(lines[i]);
if (spaces < childIndent) childIndent = spaces;
}
else childBlock.push_back("");
}
// Child block empty?
bool cbe = true;
for (unsigned i = 0; i < childBlock.size(); i++) {
if (childBlock[i].length() > 0) { cbe = false; break; }
}
// Add child block to AST
if (expectingChildBlock) {
if (cbe)
err("Expected indented child block!", out.metadata);
out.type = ASTNODE;
metadata.ln += 1;
out.args.push_back(parseLines(childBlock, metadata, childIndent));
metadata.ln -= 1;
}
else if (!cbe)
err("Did not expect indented child block!", out.metadata);
else if (out.args.size() && out.args[out.args.size() - 1].val == ":") {
Node n = out.args[out.args.size() - 1];
out.args.pop_back();
out.args.push_back(n.args[0]);
out.args.push_back(n.args[1]);
}
// Bring back if / elif into AST
if (bodied(tokens[0].val)) {
if (out.val != "multi") {
// token not being used in bodied form
}
else if (out.args[0].val == "id")
out = astnode(tokens[0].val, out.args[1].args, out.metadata);
else if (out.args[0].type == TOKEN) {
std::vector<Node> out2;
for (unsigned i = 1; i < out.args.size(); i++)
out2.push_back(out.args[i]);
out = astnode(tokens[0].val, out2, out.metadata);
}
else
out = astnode("fun", out.args, out.metadata);
}
// Multi not supported
if (out.val == "multi")
err("Multiple expressions or unclosed bracket", out.metadata);
// Convert top-level colon expressions into non-colon expressions;
// makes if statements and the like equivalent indented or not
//if (out.val == ":" && out.args[0].type == TOKEN)
// out = asn(out.args[0].val, out.args[1], out.metadata);
//if (bodied(tokens[0].val) && out.args[0].val == ":")
// out = asn(tokens[0].val, out.args[0].args);
if (o.size() == 0 || o.back().type == TOKEN) {
o.push_back(out);
continue;
}
// This is a little complicated. Basically, the idea here is to build
// constructions like [if [< x 5] [a] [elif [< x 10] [b] [else [c]]]]
std::vector<Node> u;
u.push_back(o.back());
if (bodiedContinued(o.back().val, out.val)) {
while (1) {
if (!bodiedContinued(u.back().val, out.val)) {
u.pop_back();
break;
}
if (!u.back().args.size()
|| !bodiedContinued(u.back().val, u.back().args.back().val)) {
break;
}
u.push_back(u.back().args.back());
}
u.back().args.push_back(out);
while (u.size() > 1) {
Node v = u.back();
u.pop_back();
u.back().args.pop_back();
u.back().args.push_back(v);
}
o.pop_back();
o.push_back(u[0]);
}
else o.push_back(out);
}
if (o.size() == 1)
return o[0];
else if (o.size())
return astnode("seq", o, o[0].metadata);
else
return astnode("seq", o, Metadata());
}
// Parses serpent code
Node parseSerpent(std::string s) {
std::string input = s;
std::string file = "main";
if (exists(s)) {
file = s;
input = get_file_contents(s);
}
return parseLines(splitLines(input), Metadata(file, 0, 0), 0);
}
using namespace std;

13
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/parser.h generated vendored
View File

@ -1,13 +0,0 @@
#ifndef ETHSERP_PARSER
#define ETHSERP_PARSER
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
// Serpent text -> parse tree
Node parseSerpent(std::string s);
#endif

299
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/preprocess.cpp generated vendored
View File

@ -1,299 +0,0 @@
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
#include "lllparser.h"
#include "bignum.h"
#include "rewriteutils.h"
#include "optimize.h"
#include "preprocess.h"
#include "functions.h"
#include "opcodes.h"
// Convert a function of the form (def (f x y z) (do stuff)) into
// (if (first byte of ABI is correct) (seq (setup x y z) (do stuff)))
Node convFunction(Node node, int functionCount) {
std::string prefix = "_temp"+mkUniqueToken()+"_";
Metadata m = node.metadata;
if (node.args.size() != 2)
err("Malformed def!", m);
// Collect the list of variable names and variable byte counts
Node unpack = unpackArguments(node.args[0].args, m);
// And the actual code
Node body = node.args[1];
// Main LLL-based function body
return astnode("if",
astnode("eq",
astnode("get", token("__funid", m), m),
token(unsignedToDecimal(functionCount), m),
m),
astnode("seq", unpack, body, m));
}
// Populate an svObj with the arguments needed to determine
// the storage position of a node
svObj getStorageVars(svObj pre, Node node, std::string prefix,
int index) {
Metadata m = node.metadata;
if (!pre.globalOffset.size()) pre.globalOffset = "0";
std::vector<Node> h;
std::vector<std::string> coefficients;
// Array accesses or atoms
if (node.val == "access" || node.type == TOKEN) {
std::string tot = "1";
h = listfyStorageAccess(node);
coefficients.push_back("1");
for (unsigned i = h.size() - 1; i >= 1; i--) {
// Array sizes must be constant or at least arithmetically
// evaluable at compile time
if (!isPureArithmetic(h[i]))
err("Array size must be fixed value", m);
// Create a list of the coefficient associated with each
// array index
coefficients.push_back(decimalMul(coefficients.back(), h[i].val));
}
}
// Tuples
else {
int startc;
// Handle the (fun <fun_astnode> args...) case
if (node.val == "fun") {
startc = 1;
h = listfyStorageAccess(node.args[0]);
}
// Handle the (<fun_name> args...) case, which
// the serpent parser produces when the function
// is a simple name and not a complex astnode
else {
startc = 0;
h = listfyStorageAccess(token(node.val, m));
}
svObj sub = pre;
sub.globalOffset = "0";
// Evaluate tuple elements recursively
for (unsigned i = startc; i < node.args.size(); i++) {
sub = getStorageVars(sub,
node.args[i],
prefix+h[0].val.substr(2)+".",
i-startc);
}
coefficients.push_back(sub.globalOffset);
for (unsigned i = h.size() - 1; i >= 1; i--) {
// Array sizes must be constant or at least arithmetically
// evaluable at compile time
if (!isPureArithmetic(h[i]))
err("Array size must be fixed value", m);
// Create a list of the coefficient associated with each
// array index
coefficients.push_back(decimalMul(coefficients.back(), h[i].val));
}
pre.offsets = sub.offsets;
pre.coefficients = sub.coefficients;
pre.nonfinal = sub.nonfinal;
pre.nonfinal[prefix+h[0].val.substr(2)] = true;
}
pre.coefficients[prefix+h[0].val.substr(2)] = coefficients;
pre.offsets[prefix+h[0].val.substr(2)] = pre.globalOffset;
pre.indices[prefix+h[0].val.substr(2)] = index;
if (decimalGt(tt176, coefficients.back()))
pre.globalOffset = decimalAdd(pre.globalOffset, coefficients.back());
return pre;
}
// Preprocess input containing functions
//
// localExterns is a map of the form, eg,
//
// { x: { foo: 0, bar: 1, baz: 2 }, y: { qux: 0, foo: 1 } ... }
//
// localExternSigs is a map of the form, eg,
//
// { x : { foo: iii, bar: iis, baz: ia }, y: { qux: i, foo: as } ... }
//
// Signifying that x.foo = 0, x.baz = 2, y.foo = 1, etc
// and that x.foo has three integers as arguments, x.bar has two
// integers and a variable-length string, and baz has an integer
// and an array
//
// globalExterns is a one-level map, eg from above
//
// { foo: 1, bar: 1, baz: 2, qux: 0 }
//
// globalExternSigs is a one-level map, eg from above
//
// { foo: as, bar: iis, baz: ia, qux: i}
//
// Note that globalExterns and globalExternSigs may be ambiguous
// Also, a null signature implies an infinite tail of integers
preprocessResult preprocessInit(Node inp) {
Metadata m = inp.metadata;
if (inp.val != "seq")
inp = astnode("seq", inp, m);
std::vector<Node> empty = std::vector<Node>();
Node init = astnode("seq", empty, m);
Node shared = astnode("seq", empty, m);
std::vector<Node> any;
std::vector<Node> functions;
preprocessAux out = preprocessAux();
out.localExterns["self"] = std::map<std::string, int>();
int functionCount = 0;
int storageDataCount = 0;
for (unsigned i = 0; i < inp.args.size(); i++) {
Node obj = inp.args[i];
// Functions
if (obj.val == "def") {
if (obj.args.size() == 0)
err("Empty def", m);
std::string funName = obj.args[0].val;
// Init, shared and any are special functions
if (funName == "init" || funName == "shared" || funName == "any") {
if (obj.args[0].args.size())
err(funName+" cannot have arguments", m);
}
if (funName == "init") init = obj.args[1];
else if (funName == "shared") shared = obj.args[1];
else if (funName == "any") any.push_back(obj.args[1]);
else {
// Other functions
functions.push_back(convFunction(obj, functionCount));
out.localExterns["self"][obj.args[0].val] = functionCount;
out.localExternSigs["self"][obj.args[0].val]
= getSignature(obj.args[0].args);
functionCount++;
}
}
// Extern declarations
else if (obj.val == "extern") {
std::string externName = obj.args[0].val;
Node al = obj.args[1];
if (!out.localExterns.count(externName))
out.localExterns[externName] = std::map<std::string, int>();
for (unsigned i = 0; i < al.args.size(); i++) {
if (al.args[i].val == ":") {
std::string v = al.args[i].args[0].val;
std::string sig = al.args[i].args[1].val;
out.globalExterns[v] = i;
out.globalExternSigs[v] = sig;
out.localExterns[externName][v] = i;
out.localExternSigs[externName][v] = sig;
}
else {
std::string v = al.args[i].val;
out.globalExterns[v] = i;
out.globalExternSigs[v] = "";
out.localExterns[externName][v] = i;
out.localExternSigs[externName][v] = "";
}
}
}
// Custom macros
else if (obj.val == "macro") {
// Rules for valid macros:
//
// There are only four categories of valid macros:
//
// 1. a macro where the outer function is something
// which is NOT an existing valid function/extern/datum
// 2. a macro of the form set(c(x), d) where c must NOT
// be an existing valid function/extern/datum
// 3. something of the form access(c(x)), where c must NOT
// be an existing valid function/extern/datum
// 4. something of the form set(access(c(x)), d) where c must
// NOT be an existing valid function/extern/datum
bool valid = false;
Node pattern = obj.args[0];
Node substitution = obj.args[1];
if (opcode(pattern.val) < 0 && !isValidFunctionName(pattern.val))
valid = true;
if (pattern.val == "set" &&
opcode(pattern.args[0].val) < 0 &&
!isValidFunctionName(pattern.args[0].val))
valid = true;
if (pattern.val == "access" &&
opcode(pattern.args[0].val) < 0 &&
!isValidFunctionName(pattern.args[0].val))
if (pattern.val == "set" &&
pattern.args[0].val == "access" &&
opcode(pattern.args[0].args[0].val) < 0 &&
!isValidFunctionName(pattern.args[0].args[0].val))
valid = true;
if (valid) {
out.customMacros.push_back(rewriteRule(pattern, substitution));
}
}
// Variable types
else if (obj.val == "type") {
std::string typeName = obj.args[0].val;
std::vector<Node> vars = obj.args[1].args;
for (unsigned i = 0; i < vars.size(); i++)
out.types[vars[i].val] = typeName;
}
// Storage variables/structures
else if (obj.val == "data") {
out.storageVars = getStorageVars(out.storageVars,
obj.args[0],
"",
storageDataCount);
storageDataCount += 1;
}
else any.push_back(obj);
}
std::vector<Node> main;
if (shared.args.size()) main.push_back(shared);
if (init.args.size()) main.push_back(init);
std::vector<Node> code;
if (shared.args.size()) code.push_back(shared);
for (unsigned i = 0; i < any.size(); i++)
code.push_back(any[i]);
for (unsigned i = 0; i < functions.size(); i++)
code.push_back(functions[i]);
Node codeNode;
if (functions.size() > 0) {
codeNode = astnode("with",
token("__funid", m),
astnode("byte",
token("0", m),
astnode("calldataload", token("0", m), m),
m),
astnode("seq", code, m),
m);
}
else codeNode = astnode("seq", code, m);
main.push_back(astnode("~return",
token("0", m),
astnode("lll",
codeNode,
token("0", m),
m),
m));
Node result;
if (main.size() == 1) result = main[0];
else result = astnode("seq", main, inp.metadata);
return preprocessResult(result, out);
}
preprocessResult processTypes (preprocessResult pr) {
preprocessAux aux = pr.second;
Node node = pr.first;
if (node.type == TOKEN && aux.types.count(node.val)) {
node = asn(aux.types[node.val], node, node.metadata);
}
else if (node.val == "untyped")
return preprocessResult(node.args[0], aux);
else {
for (unsigned i = 0; i < node.args.size(); i++) {
node.args[i] =
processTypes(preprocessResult(node.args[i], aux)).first;
}
}
return preprocessResult(node, aux);
}
preprocessResult preprocess(Node n) {
return processTypes(preprocessInit(n));
}

58
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/preprocess.h generated vendored
View File

@ -1,58 +0,0 @@
#ifndef ETHSERP_PREPROCESSOR
#define ETHSERP_PREPROCESSOR
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
// Storage variable index storing object
struct svObj {
std::map<std::string, std::string> offsets;
std::map<std::string, int> indices;
std::map<std::string, std::vector<std::string> > coefficients;
std::map<std::string, bool> nonfinal;
std::string globalOffset;
};
class rewriteRule {
public:
rewriteRule(Node p, Node s) {
pattern = p;
substitution = s;
}
Node pattern;
Node substitution;
};
// Preprocessing result storing object
class preprocessAux {
public:
preprocessAux() {
globalExterns = std::map<std::string, int>();
localExterns = std::map<std::string, std::map<std::string, int> >();
localExterns["self"] = std::map<std::string, int>();
}
std::map<std::string, int> globalExterns;
std::map<std::string, std::string> globalExternSigs;
std::map<std::string, std::map<std::string, int> > localExterns;
std::map<std::string, std::map<std::string, std::string> > localExternSigs;
std::vector<rewriteRule> customMacros;
std::map<std::string, std::string> types;
svObj storageVars;
};
#define preprocessResult std::pair<Node, preprocessAux>
// Populate an svObj with the arguments needed to determine
// the storage position of a node
svObj getStorageVars(svObj pre, Node node, std::string prefix="",
int index=0);
// Preprocess a function (see cpp for details)
preprocessResult preprocess(Node inp);
#endif

173
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/pyserpent.cpp generated vendored
View File

@ -1,173 +0,0 @@
#include <Python.h>
#include "structmember.h"
#include <stdlib.h>
#include <stdio.h>
#include <iostream>
#include "funcs.h"
#define PYMETHOD(name, FROM, method, TO) \
static PyObject * name(PyObject *, PyObject *args) { \
try { \
FROM(med) \
return TO(method(med)); \
} \
catch (std::string e) { \
PyErr_SetString(PyExc_Exception, e.c_str()); \
return NULL; \
} \
}
#define FROMSTR(v) \
const char *command; \
int len; \
if (!PyArg_ParseTuple(args, "s#", &command, &len)) \
return NULL; \
std::string v = std::string(command, len); \
#define FROMNODE(v) \
PyObject *node; \
if (!PyArg_ParseTuple(args, "O", &node)) \
return NULL; \
Node v = cppifyNode(node);
#define FROMLIST(v) \
PyObject *node; \
if (!PyArg_ParseTuple(args, "O", &node)) \
return NULL; \
std::vector<Node> v = cppifyNodeList(node);
// Convert metadata into python wrapper form [file, ln, ch]
PyObject* pyifyMetadata(Metadata m) {
PyObject* a = PyList_New(0);
PyList_Append(a, Py_BuildValue("s#", m.file.c_str(), m.file.length()));
PyList_Append(a, Py_BuildValue("i", m.ln));
PyList_Append(a, Py_BuildValue("i", m.ch));
return a;
}
// Convert node into python wrapper form
// [token=0/astnode=1, val, metadata, args]
PyObject* pyifyNode(Node n) {
PyObject* a = PyList_New(0);
PyList_Append(a, Py_BuildValue("i", n.type == ASTNODE));
PyList_Append(a, Py_BuildValue("s#", n.val.c_str(), n.val.length()));
PyList_Append(a, pyifyMetadata(n.metadata));
for (unsigned i = 0; i < n.args.size(); i++)
PyList_Append(a, pyifyNode(n.args[i]));
return a;
}
// Convert string into python wrapper form
PyObject* pyifyString(std::string s) {
return Py_BuildValue("s#", s.c_str(), s.length());
}
// Convert list of nodes into python wrapper form
PyObject* pyifyNodeList(std::vector<Node> n) {
PyObject* a = PyList_New(0);
for (unsigned i = 0; i < n.size(); i++)
PyList_Append(a, pyifyNode(n[i]));
return a;
}
// Convert pyobject int into normal form
int cppifyInt(PyObject* o) {
int out;
if (!PyArg_Parse(o, "i", &out))
err("Argument should be integer", Metadata());
return out;
}
// Convert pyobject string into normal form
std::string cppifyString(PyObject* o) {
const char *command;
if (!PyArg_Parse(o, "s", &command))
err("Argument should be string", Metadata());
return std::string(command);
}
// Convert metadata from python wrapper form
Metadata cppifyMetadata(PyObject* o) {
std::string file = cppifyString(PyList_GetItem(o, 0));
int ln = cppifyInt(PyList_GetItem(o, 1));
int ch = cppifyInt(PyList_GetItem(o, 2));
return Metadata(file, ln, ch);
}
// Convert node from python wrapper form
Node cppifyNode(PyObject* o) {
Node n;
int isAstNode = cppifyInt(PyList_GetItem(o, 0));
n.type = isAstNode ? ASTNODE : TOKEN;
n.val = cppifyString(PyList_GetItem(o, 1));
n.metadata = cppifyMetadata(PyList_GetItem(o, 2));
std::vector<Node> args;
for (int i = 3; i < PyList_Size(o); i++) {
args.push_back(cppifyNode(PyList_GetItem(o, i)));
}
n.args = args;
return n;
}
//Convert list of nodes into normal form
std::vector<Node> cppifyNodeList(PyObject* o) {
std::vector<Node> out;
for (int i = 0; i < PyList_Size(o); i++) {
out.push_back(cppifyNode(PyList_GetItem(o,i)));
}
return out;
}
PYMETHOD(ps_compile, FROMSTR, compile, pyifyString)
PYMETHOD(ps_compile_chunk, FROMSTR, compileChunk, pyifyString)
PYMETHOD(ps_compile_to_lll, FROMSTR, compileToLLL, pyifyNode)
PYMETHOD(ps_compile_chunk_to_lll, FROMSTR, compileChunkToLLL, pyifyNode)
PYMETHOD(ps_compile_lll, FROMNODE, compileLLL, pyifyString)
PYMETHOD(ps_parse, FROMSTR, parseSerpent, pyifyNode)
PYMETHOD(ps_rewrite, FROMNODE, rewrite, pyifyNode)
PYMETHOD(ps_rewrite_chunk, FROMNODE, rewriteChunk, pyifyNode)
PYMETHOD(ps_pretty_compile, FROMSTR, prettyCompile, pyifyNodeList)
PYMETHOD(ps_pretty_compile_chunk, FROMSTR, prettyCompileChunk, pyifyNodeList)
PYMETHOD(ps_pretty_compile_lll, FROMNODE, prettyCompileLLL, pyifyNodeList)
PYMETHOD(ps_serialize, FROMLIST, serialize, pyifyString)
PYMETHOD(ps_deserialize, FROMSTR, deserialize, pyifyNodeList)
PYMETHOD(ps_parse_lll, FROMSTR, parseLLL, pyifyNode)
static PyMethodDef PyextMethods[] = {
{"compile", ps_compile, METH_VARARGS,
"Compile code."},
{"compile_chunk", ps_compile_chunk, METH_VARARGS,
"Compile code chunk (no wrappers)."},
{"compile_to_lll", ps_compile_to_lll, METH_VARARGS,
"Compile code to LLL."},
{"compile_chunk_to_lll", ps_compile_chunk_to_lll, METH_VARARGS,
"Compile code chunk to LLL (no wrappers)."},
{"compile_lll", ps_compile_lll, METH_VARARGS,
"Compile LLL to EVM."},
{"parse", ps_parse, METH_VARARGS,
"Parse serpent"},
{"rewrite", ps_rewrite, METH_VARARGS,
"Rewrite parsed serpent to LLL"},
{"rewrite_chunk", ps_rewrite_chunk, METH_VARARGS,
"Rewrite parsed serpent to LLL (no wrappers)"},
{"pretty_compile", ps_pretty_compile, METH_VARARGS,
"Compile to EVM opcodes"},
{"pretty_compile_chunk", ps_pretty_compile_chunk, METH_VARARGS,
"Compile chunk to EVM opcodes (no wrappers)"},
{"pretty_compile_lll", ps_pretty_compile_lll, METH_VARARGS,
"Compile LLL to EVM opcodes"},
{"serialize", ps_serialize, METH_VARARGS,
"Convert EVM opcodes to bin"},
{"deserialize", ps_deserialize, METH_VARARGS,
"Convert EVM bin to opcodes"},
{"parse_lll", ps_parse_lll, METH_VARARGS,
"Parse LLL"},
{NULL, NULL, 0, NULL} /* Sentinel */
};
PyMODINIT_FUNC initserpent_pyext(void)
{
Py_InitModule( "serpent_pyext", PyextMethods );
}

1
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/pyserpent.py generated vendored
View File

@ -1 +0,0 @@
from serpent import *

804
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriter.cpp generated vendored
View File

@ -1,804 +0,0 @@
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
#include "lllparser.h"
#include "bignum.h"
#include "optimize.h"
#include "rewriteutils.h"
#include "preprocess.h"
#include "functions.h"
#include "opcodes.h"
// Rewrite rules
std::string macros[][2] = {
{
"(seq $x)",
"$x"
},
{
"(seq (seq) $x)",
"$x"
},
{
"(+= $a $b)",
"(set $a (+ $a $b))"
},
{
"(*= $a $b)",
"(set $a (* $a $b))"
},
{
"(-= $a $b)",
"(set $a (- $a $b))"
},
{
"(/= $a $b)",
"(set $a (/ $a $b))"
},
{
"(%= $a $b)",
"(set $a (% $a $b))"
},
{
"(^= $a $b)",
"(set $a (^ $a $b))"
},
{
"(!= $a $b)",
"(iszero (eq $a $b))"
},
{
"(assert $x)",
"(unless $x (stop))"
},
{
"(min $a $b)",
"(with $1 $a (with $2 $b (if (lt $1 $2) $1 $2)))"
},
{
"(max $a $b)",
"(with $1 $a (with $2 $b (if (lt $1 $2) $2 $1)))"
},
{
"(smin $a $b)",
"(with $1 $a (with $2 $b (if (slt $1 $2) $1 $2)))"
},
{
"(smax $a $b)",
"(with $1 $a (with $2 $b (if (slt $1 $2) $2 $1)))"
},
{
"(if $cond $do (else $else))",
"(if $cond $do $else)"
},
{
"(code $code)",
"$code"
},
{
"(slice $arr $pos)",
"(add $arr (mul 32 $pos))",
},
{
"(array $len)",
"(alloc (mul 32 $len))"
},
{
"(while $cond $do)",
"(until (iszero $cond) $do)",
},
{
"(while (iszero $cond) $do)",
"(until $cond $do)",
},
{
"(if $cond $do)",
"(unless (iszero $cond) $do)",
},
{
"(if (iszero $cond) $do)",
"(unless $cond $do)",
},
{
"(access (. self storage) $ind)",
"(sload $ind)"
},
{
"(access $var $ind)",
"(mload (add $var (mul 32 $ind)))"
},
{
"(set (access (. self storage) $ind) $val)",
"(sstore $ind $val)"
},
{
"(set (access $var $ind) $val)",
"(mstore (add $var (mul 32 $ind)) $val)"
},
{
"(getch $var $ind)",
"(mod (mload (sub (add $var $ind) 31)) 256)"
},
{
"(setch $var $ind $val)",
"(mstore8 (add $var $ind) $val)",
},
{
"(send $to $value)",
"(~call (sub (gas) 25) $to $value 0 0 0 0)"
},
{
"(send $gas $to $value)",
"(~call $gas $to $value 0 0 0 0)"
},
{
"(sha3 $x)",
"(seq (set $1 $x) (~sha3 (ref $1) 32))"
},
{
"(sha3 $mstart (= chars $msize))",
"(~sha3 $mstart $msize)"
},
{
"(sha3 $mstart $msize)",
"(~sha3 $mstart (mul 32 $msize))"
},
{
"(id $0)",
"$0"
},
{
"(return $x)",
"(seq (set $1 $x) (~return (ref $1) 32))"
},
{
"(return $mstart (= chars $msize))",
"(~return $mstart $msize)"
},
{
"(return $start $len)",
"(~return $start (mul 32 $len))"
},
{
"(&& $x $y)",
"(if $x $y 0)"
},
{
"(|| $x $y)",
"(with $1 $x (if $1 $1 $y))"
},
{
"(>= $x $y)",
"(iszero (slt $x $y))"
},
{
"(<= $x $y)",
"(iszero (sgt $x $y))"
},
{
"(create $code)",
"(create 0 $code)"
},
{
"(create $endowment $code)",
"(with $1 (msize) (create $endowment (get $1) (lll (outer $code) (msize))))"
},
{
"(sha256 $x)",
"(with $1 (alloc 64) (seq (mstore (add (get $1) 32) $x) (pop (~call 101 2 0 (add (get $1) 32) 32 (get $1) 32)) (mload (get $1))))"
},
{
"(sha256 $arr (= chars $sz))",
"(with $1 (alloc 32) (seq (pop (~call 101 2 0 $arr $sz (get $1) 32)) (mload (get $1))))"
},
{
"(sha256 $arr $sz)",
"(with $1 (alloc 32) (seq (pop (~call 101 2 0 $arr (mul 32 $sz) (get $1) 32)) (mload (get $1))))"
},
{
"(ripemd160 $x)",
"(with $1 (alloc 64) (seq (mstore (add (get $1) 32) $x) (pop (~call 101 3 0 (add (get $1) 32) 32 (get $1) 32)) (mload (get $1))))"
},
{
"(ripemd160 $arr (= chars $sz))",
"(with $1 (alloc 32) (seq (pop (~call 101 3 0 $arr $sz (mload $1) 32)) (mload (get $1))))"
},
{
"(ripemd160 $arr $sz)",
"(with $1 (alloc 32) (seq (pop (~call 101 3 0 $arr (mul 32 $sz) (get $1) 32)) (mload (get $1))))"
},
{
"(ecrecover $h $v $r $s)",
"(with $1 (alloc 160) (seq (mstore (get $1) $h) (mstore (add (get $1) 32) $v) (mstore (add (get $1) 64) $r) (mstore (add (get $1) 96) $s) (pop (~call 101 1 0 (get $1) 128 (add (get $1 128)) 32)) (mload (add (get $1) 128))))"
},
{
"(inset $x)",
"$x"
},
{
"(create $x)",
"(with $1 (msize) (create $val (get $1) (lll $code (get $1))))"
},
{
"(with (= $var $val) $cond)",
"(with $var $val $cond)"
},
{
"(log $t1)",
"(~log1 0 0 $t1)"
},
{
"(log $t1 $t2)",
"(~log2 0 0 $t1 $t2)"
},
{
"(log $t1 $t2 $t3)",
"(~log3 0 0 $t1 $t2 $t3)"
},
{
"(log $t1 $t2 $t3 $t4)",
"(~log4 0 0 $t1 $t2 $t3 $t4)"
},
{
"(logarr $a $sz)",
"(~log0 $a (mul 32 $sz))"
},
{
"(logarr $a $sz $t1)",
"(~log1 $a (mul 32 $sz) $t1)"
},
{
"(logarr $a $sz $t1 $t2)",
"(~log2 $a (mul 32 $sz) $t1 $t2)"
},
{
"(logarr $a $sz $t1 $t2 $t3)",
"(~log3 $a (mul 32 $sz) $t1 $t2 $t3)"
},
{
"(logarr $a $sz $t1 $t2 $t3 $t4)",
"(~log4 $a (mul 32 $sz) $t1 $t2 $t3 $t4)"
},
{
"(save $loc $array (= chars $count))",
"(with $location (ref $loc) (with $c $count (with $end (div $c 32) (with $i 0 (seq (while (slt $i $end) (seq (sstore (add $i $location) (access $array $i)) (set $i (add $i 1)))) (sstore (add $i $location) (~and (access $array $i) (sub 0 (exp 256 (sub 32 (mod $c 32)))))))))))"
},
{
"(save $loc $array $count)",
"(with $location (ref $loc) (with $end $count (with $i 0 (while (slt $i $end) (seq (sstore (add $i $location) (access $array $i)) (set $i (add $i 1)))))))"
},
{
"(load $loc (= chars $count))",
"(with $location (ref $loc) (with $c $count (with $a (alloc $c) (with $i 0 (seq (while (slt $i (div $c 32)) (seq (set (access $a $i) (sload (add $location $i))) (set $i (add $i 1)))) (set (access $a $i) (~and (sload (add $location $i)) (sub 0 (exp 256 (sub 32 (mod $c 32)))))) $a)))))"
},
{
"(load $loc $count)",
"(with $location (ref $loc) (with $c $count (with $a (alloc $c) (with $i 0 (seq (while (slt $i $c) (seq (set (access $a $i) (sload (add $location $i))) (set $i (add $i 1)))) $a)))))"
},
{
"(unsafe_mcopy $to $from $sz)",
"(with _sz $sz (with _from $from (with _to $to (seq (comment STARTING UNSAFE MCOPY) (with _i 0 (while (lt _i _sz) (seq (mstore (add $to _i) (mload (add _from _i))) (set _i (add _i 32)))))))))"
},
{
"(mcopy $to $from $_sz)",
"(with _to $to (with _from $from (with _sz $sz (seq (comment STARTING MCOPY (with _i 0 (seq (while (lt (add _i 31) _sz) (seq (mstore (add _to _i) (mload (add _from _i))) (set _i (add _i 32)))) (with _mask (exp 256 (sub 32 (mod _sz 32))) (mstore (add $to _i) (add (mod (mload (add $to _i)) _mask) (and (mload (add $from _i)) (sub 0 _mask))))))))))))"
},
{ "(. msg sender)", "(caller)" },
{ "(. msg value)", "(callvalue)" },
{ "(. tx gasprice)", "(gasprice)" },
{ "(. tx origin)", "(origin)" },
{ "(. tx gas)", "(gas)" },
{ "(. $x balance)", "(balance $x)" },
{ "self", "(address)" },
{ "(. block prevhash)", "(prevhash)" },
{ "(. block coinbase)", "(coinbase)" },
{ "(. block timestamp)", "(timestamp)" },
{ "(. block number)", "(number)" },
{ "(. block difficulty)", "(difficulty)" },
{ "(. block gaslimit)", "(gaslimit)" },
{ "stop", "(stop)" },
{ "---END---", "" } //Keep this line at the end of the list
};
std::vector<rewriteRule> nodeMacros;
// Token synonyms
std::string synonyms[][2] = {
{ "or", "||" },
{ "and", "&&" },
{ "|", "~or" },
{ "&", "~and" },
{ "elif", "if" },
{ "!", "iszero" },
{ "~", "~not" },
{ "not", "iszero" },
{ "string", "alloc" },
{ "+", "add" },
{ "-", "sub" },
{ "*", "mul" },
{ "/", "sdiv" },
{ "^", "exp" },
{ "**", "exp" },
{ "%", "smod" },
{ "<", "slt" },
{ ">", "sgt" },
{ "=", "set" },
{ "==", "eq" },
{ ":", "kv" },
{ "---END---", "" } //Keep this line at the end of the list
};
// Custom setters (need to be registered separately
// for use with managed storage)
std::string setters[][2] = {
{ "+=", "+" },
{ "-=", "-" },
{ "*=", "*" },
{ "/=", "/" },
{ "%=", "%" },
{ "^=", "^" },
{ "---END---", "" } //Keep this line at the end of the list
};
// Processes mutable array literals
Node array_lit_transform(Node node) {
std::string prefix = "_temp"+mkUniqueToken() + "_";
Metadata m = node.metadata;
std::map<std::string, Node> d;
std::string o = "(seq (set $arr (alloc "+utd(node.args.size()*32)+"))";
for (unsigned i = 0; i < node.args.size(); i++) {
o += " (mstore (add (get $arr) "+utd(i * 32)+") $"+utd(i)+")";
d[utd(i)] = node.args[i];
}
o += " (get $arr))";
return subst(parseLLL(o), d, prefix, m);
}
Node apply_rules(preprocessResult pr);
// Transform "<variable>.<fun>(args...)" into
// a call
Node dotTransform(Node node, preprocessAux aux) {
Metadata m = node.metadata;
// We're gonna make lots of temporary variables,
// so set up a unique flag for them
std::string prefix = "_temp"+mkUniqueToken()+"_";
// Check that the function name is a token
if (node.args[0].args[1].type == ASTNODE)
err("Function name must be static", m);
Node dotOwner = node.args[0].args[0];
std::string dotMember = node.args[0].args[1].val;
// kwargs = map of special arguments
std::map<std::string, Node> kwargs;
kwargs["value"] = token("0", m);
kwargs["gas"] = subst(parseLLL("(- (gas) 25)"), msn(), prefix, m);
// Search for as=? and call=code keywords, and isolate the actual
// function arguments
std::vector<Node> fnargs;
std::string as = "";
std::string op = "call";
for (unsigned i = 1; i < node.args.size(); i++) {
fnargs.push_back(node.args[i]);
Node arg = fnargs.back();
if (arg.val == "=" || arg.val == "set") {
if (arg.args[0].val == "as")
as = arg.args[1].val;
if (arg.args[0].val == "call" && arg.args[1].val == "code")
op = "callcode";
if (arg.args[0].val == "gas")
kwargs["gas"] = arg.args[1];
if (arg.args[0].val == "value")
kwargs["value"] = arg.args[1];
if (arg.args[0].val == "outsz")
kwargs["outsz"] = arg.args[1];
}
}
if (dotOwner.val == "self") {
if (as.size()) err("Cannot use \"as\" when calling self!", m);
as = dotOwner.val;
}
// Determine the funId and sig assuming the "as" keyword was used
int funId = 0;
std::string sig;
if (as.size() > 0 && aux.localExterns.count(as)) {
if (!aux.localExterns[as].count(dotMember))
err("Invalid call: "+printSimple(dotOwner)+"."+dotMember, m);
funId = aux.localExterns[as][dotMember];
sig = aux.localExternSigs[as][dotMember];
}
// Determine the funId and sig otherwise
else if (!as.size()) {
if (!aux.globalExterns.count(dotMember))
err("Invalid call: "+printSimple(dotOwner)+"."+dotMember, m);
std::string key = unsignedToDecimal(aux.globalExterns[dotMember]);
funId = aux.globalExterns[dotMember];
sig = aux.globalExternSigs[dotMember];
}
else err("Invalid call: "+printSimple(dotOwner)+"."+dotMember, m);
// Pack arguments
kwargs["data"] = packArguments(fnargs, sig, funId, m);
kwargs["to"] = dotOwner;
Node main;
// Pack output
if (!kwargs.count("outsz")) {
main = parseLLL(
"(with _data $data (seq "
"(pop (~"+op+" $gas $to $value (access _data 0) (access _data 1) (ref $dataout) 32))"
"(get $dataout)))");
}
else {
main = parseLLL(
"(with _data $data (with _outsz (mul 32 $outsz) (with _out (alloc _outsz) (seq "
"(pop (~"+op+" $gas $to $value (access _data 0) (access _data 1) _out _outsz))"
"(get _out)))))");
}
// Set up main call
Node o = subst(main, kwargs, prefix, m);
return o;
}
// Transform an access of the form self.bob, self.users[5], etc into
// a storage access
//
// There exist two types of objects: finite objects, and infinite
// objects. Finite objects are packed optimally tightly into storage
// accesses; for example:
//
// data obj[100](a, b[2][4], c)
//
// obj[0].a -> 0
// obj[0].b[0][0] -> 1
// obj[0].b[1][3] -> 8
// obj[45].c -> 459
//
// Infinite objects are accessed by sha3([v1, v2, v3 ... ]), where
// the values are a list of array indices and keyword indices, for
// example:
// data obj[](a, b[2][4], c)
// data obj2[](a, b[][], c)
//
// obj[0].a -> sha3([0, 0, 0])
// obj[5].b[1][3] -> sha3([0, 5, 1, 1, 3])
// obj[45].c -> sha3([0, 45, 2])
// obj2[0].a -> sha3([1, 0, 0])
// obj2[5].b[1][3] -> sha3([1, 5, 1, 1, 3])
// obj2[45].c -> sha3([1, 45, 2])
Node storageTransform(Node node, preprocessAux aux,
bool mapstyle=false, bool ref=false) {
Metadata m = node.metadata;
// Get a list of all of the "access parameters" used in order
// eg. self.users[5].cow[4][m[2]][woof] ->
// [--self, --users, 5, --cow, 4, m[2], woof]
std::vector<Node> hlist = listfyStorageAccess(node);
// For infinite arrays, the terms array will just provide a list
// of indices. For finite arrays, it's a list of index*coefficient
std::vector<Node> terms;
std::string offset = "0";
std::string prefix = "";
std::string varPrefix = "_temp"+mkUniqueToken()+"_";
int c = 0;
std::vector<std::string> coefficients;
coefficients.push_back("");
for (unsigned i = 1; i < hlist.size(); i++) {
// We pre-add the -- flag to parameter-like terms. For example,
// self.users[m] -> [--self, --users, m]
// self.users.m -> [--self, --users, --m]
if (hlist[i].val.substr(0, 2) == "--") {
prefix += hlist[i].val.substr(2) + ".";
std::string tempPrefix = prefix.substr(0, prefix.size()-1);
if (!aux.storageVars.offsets.count(tempPrefix))
return node;
if (c < (signed)coefficients.size() - 1)
err("Too few array index lookups", m);
if (c > (signed)coefficients.size() - 1)
err("Too many array index lookups", m);
coefficients = aux.storageVars.coefficients[tempPrefix];
// If the size of an object exceeds 2^176, we make it an infinite
// array
if (decimalGt(coefficients.back(), tt176) && !mapstyle)
return storageTransform(node, aux, true, ref);
offset = decimalAdd(offset, aux.storageVars.offsets[tempPrefix]);
c = 0;
if (mapstyle)
terms.push_back(token(unsignedToDecimal(
aux.storageVars.indices[tempPrefix])));
}
else if (mapstyle) {
terms.push_back(hlist[i]);
c += 1;
}
else {
if (c > (signed)coefficients.size() - 2)
err("Too many array index lookups", m);
terms.push_back(
astnode("mul",
hlist[i],
token(coefficients[coefficients.size() - 2 - c], m),
m));
c += 1;
}
}
if (aux.storageVars.nonfinal.count(prefix.substr(0, prefix.size()-1)))
err("Storage variable access not deep enough", m);
if (c < (signed)coefficients.size() - 1) {
err("Too few array index lookups", m);
}
if (c > (signed)coefficients.size() - 1) {
err("Too many array index lookups", m);
}
Node o;
if (mapstyle) {
std::string t = "_temp_"+mkUniqueToken();
std::vector<Node> sub;
for (unsigned i = 0; i < terms.size(); i++)
sub.push_back(asn("mstore",
asn("add",
tkn(utd(i * 32), m),
asn("get", tkn(t+"pos", m), m),
m),
terms[i],
m));
sub.push_back(tkn(t+"pos", m));
Node main = asn("with",
tkn(t+"pos", m),
asn("alloc", tkn(utd(terms.size() * 32), m), m),
asn("seq", sub, m),
m);
Node sz = token(utd(terms.size() * 32), m);
o = astnode("~sha3",
main,
sz,
m);
}
else {
// We add up all the index*coefficients
Node out = token(offset, node.metadata);
for (unsigned i = 0; i < terms.size(); i++) {
std::vector<Node> temp;
temp.push_back(out);
temp.push_back(terms[i]);
out = astnode("add", temp, node.metadata);
}
o = out;
}
if (ref) return o;
else return astnode("sload", o, node.metadata);
}
// Recursively applies rewrite rules
std::pair<Node, bool> apply_rules_iter(preprocessResult pr) {
bool changed = false;
Node node = pr.first;
// If the rewrite rules have not yet been parsed, parse them
if (!nodeMacros.size()) {
for (int i = 0; i < 9999; i++) {
std::vector<Node> o;
if (macros[i][0] == "---END---") break;
nodeMacros.push_back(rewriteRule(
parseLLL(macros[i][0]),
parseLLL(macros[i][1])
));
}
}
// Assignment transformations
for (int i = 0; i < 9999; i++) {
if (setters[i][0] == "---END---") break;
if (node.val == setters[i][0]) {
node = astnode("=",
node.args[0],
astnode(setters[i][1],
node.args[0],
node.args[1],
node.metadata),
node.metadata);
}
}
// Do nothing to macros
if (node.val == "macro") {
return std::pair<Node, bool>(node, changed);
}
// Ignore comments
if (node.val == "comment") {
return std::pair<Node, bool>(node, changed);
}
// Special storage transformation
if (isNodeStorageVariable(node)) {
node = storageTransform(node, pr.second);
changed = true;
}
if (node.val == "ref" && isNodeStorageVariable(node.args[0])) {
node = storageTransform(node.args[0], pr.second, false, true);
changed = true;
}
if (node.val == "=" && isNodeStorageVariable(node.args[0])) {
Node t = storageTransform(node.args[0], pr.second);
if (t.val == "sload") {
std::vector<Node> o;
o.push_back(t.args[0]);
o.push_back(node.args[1]);
node = astnode("sstore", o, node.metadata);
}
changed = true;
}
// Main code
unsigned pos = 0;
std::string prefix = "_temp"+mkUniqueToken()+"_";
while(1) {
if (synonyms[pos][0] == "---END---") {
break;
}
else if (node.type == ASTNODE && node.val == synonyms[pos][0]) {
node.val = synonyms[pos][1];
changed = true;
}
pos++;
}
for (pos = 0; pos < nodeMacros.size() + pr.second.customMacros.size(); pos++) {
rewriteRule macro = pos < nodeMacros.size()
? nodeMacros[pos]
: pr.second.customMacros[pos - nodeMacros.size()];
matchResult mr = match(macro.pattern, node);
if (mr.success) {
node = subst(macro.substitution, mr.map, prefix, node.metadata);
std::pair<Node, bool> o =
apply_rules_iter(preprocessResult(node, pr.second));
o.second = true;
return o;
}
}
// Special transformations
if (node.val == "outer") {
node = apply_rules(preprocess(node.args[0]));
changed = true;
}
if (node.val == "array_lit") {
node = array_lit_transform(node);
changed = true;
}
if (node.val == "fun" && node.args[0].val == ".") {
node = dotTransform(node, pr.second);
changed = true;
}
if (node.type == ASTNODE) {
unsigned i = 0;
if (node.val == "set" || node.val == "ref"
|| node.val == "get" || node.val == "with") {
if (node.args[0].val.size() > 0 && node.args[0].val[0] != '\''
&& node.args[0].type == TOKEN && node.args[0].val[0] != '$') {
node.args[0].val = "'" + node.args[0].val;
changed = true;
}
i = 1;
}
else if (node.val == "arglen") {
node.val = "get";
node.args[0].val = "'_len_" + node.args[0].val;
i = 1;
changed = true;
}
for (; i < node.args.size(); i++) {
std::pair<Node, bool> r =
apply_rules_iter(preprocessResult(node.args[i], pr.second));
node.args[i] = r.first;
changed = changed || r.second;
}
}
else if (node.type == TOKEN && !isNumberLike(node)) {
if (node.val.size() >= 2
&& node.val[0] == '"'
&& node.val[node.val.size() - 1] == '"') {
std::string bin = node.val.substr(1, node.val.size() - 2);
unsigned sz = bin.size();
std::vector<Node> o;
for (unsigned i = 0; i < sz; i += 32) {
std::string t = binToNumeric(bin.substr(i, 32));
if ((sz - i) < 32 && (sz - i) > 0) {
while ((sz - i) < 32) {
t = decimalMul(t, "256");
i--;
}
i = sz;
}
o.push_back(token(t, node.metadata));
}
node = astnode("array_lit", o, node.metadata);
std::pair<Node, bool> r =
apply_rules_iter(preprocessResult(node, pr.second));
node = r.first;
changed = true;
}
else if (node.val.size() && node.val[0] != '\'' && node.val[0] != '$') {
node.val = "'" + node.val;
std::vector<Node> args;
args.push_back(node);
std::string v = node.val.substr(1);
node = astnode("get", args, node.metadata);
changed = true;
}
}
return std::pair<Node, bool>(node, changed);
}
Node apply_rules(preprocessResult pr) {
for (unsigned i = 0; i < pr.second.customMacros.size(); i++) {
pr.second.customMacros[i].pattern =
apply_rules(preprocessResult(pr.second.customMacros[i].pattern, preprocessAux()));
}
while (1) {
//std::cerr << printAST(pr.first) <<
// " " << pr.second.customMacros.size() << "\n";
std::pair<Node, bool> r = apply_rules_iter(pr);
if (!r.second) {
return r.first;
}
pr.first = r.first;
}
}
Node validate(Node inp) {
Metadata m = inp.metadata;
if (inp.type == ASTNODE) {
int i = 0;
while(validFunctions[i][0] != "---END---") {
if (inp.val == validFunctions[i][0]) {
std::string sz = unsignedToDecimal(inp.args.size());
if (decimalGt(validFunctions[i][1], sz)) {
err("Too few arguments for "+inp.val, inp.metadata);
}
if (decimalGt(sz, validFunctions[i][2])) {
err("Too many arguments for "+inp.val, inp.metadata);
}
}
i++;
}
}
for (unsigned i = 0; i < inp.args.size(); i++) validate(inp.args[i]);
return inp;
}
Node postValidate(Node inp) {
// This allows people to use ~x as a way of having functions with the same
// name and arity as macros; the idea is that ~x is a "final" form, and
// should not be remacroed, but it is converted back at the end
if (inp.val.size() > 0 && inp.val[0] == '~') {
inp.val = inp.val.substr(1);
}
if (inp.type == ASTNODE) {
if (inp.val == ".")
err("Invalid object member (ie. a foo.bar not mapped to anything)",
inp.metadata);
else if (opcode(inp.val) >= 0) {
if ((signed)inp.args.size() < opinputs(inp.val))
err("Too few arguments for "+inp.val, inp.metadata);
if ((signed)inp.args.size() > opinputs(inp.val))
err("Too many arguments for "+inp.val, inp.metadata);
}
else if (isValidLLLFunc(inp.val, inp.args.size())) {
// do nothing
}
else err ("Invalid argument count or LLL function: "+inp.val, inp.metadata);
for (unsigned i = 0; i < inp.args.size(); i++) {
inp.args[i] = postValidate(inp.args[i]);
}
}
return inp;
}
Node rewrite(Node inp) {
return postValidate(optimize(apply_rules(preprocess(inp))));
}
Node rewriteChunk(Node inp) {
return postValidate(optimize(apply_rules(
preprocessResult(
validate(inp), preprocessAux()))));
}
using namespace std;

16
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriter.h generated vendored
View File

@ -1,16 +0,0 @@
#ifndef ETHSERP_REWRITER
#define ETHSERP_REWRITER
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
// Applies rewrite rules
Node rewrite(Node inp);
// Applies rewrite rules adding without wrapper
Node rewriteChunk(Node inp);
#endif

211
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriteutils.cpp generated vendored
View File

@ -1,211 +0,0 @@
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
#include "lllparser.h"
#include "bignum.h"
#include "rewriteutils.h"
#include "optimize.h"
// Valid functions and their min and max argument counts
std::string validFunctions[][3] = {
{ "if", "2", "3" },
{ "unless", "2", "2" },
{ "while", "2", "2" },
{ "until", "2", "2" },
{ "alloc", "1", "1" },
{ "array", "1", "1" },
{ "call", "2", tt256 },
{ "callcode", "2", tt256 },
{ "create", "1", "4" },
{ "getch", "2", "2" },
{ "setch", "3", "3" },
{ "sha3", "1", "2" },
{ "return", "1", "2" },
{ "inset", "1", "1" },
{ "min", "2", "2" },
{ "max", "2", "2" },
{ "array_lit", "0", tt256 },
{ "seq", "0", tt256 },
{ "log", "1", "6" },
{ "outer", "1", "1" },
{ "set", "2", "2" },
{ "get", "1", "1" },
{ "ref", "1", "1" },
{ "declare", "1", tt256 },
{ "with", "3", "3" },
{ "outer", "1", "1" },
{ "mcopy", "3", "3" },
{ "unsafe_mcopy", "3", "3" },
{ "save", "3", "3" },
{ "load", "2", "2" },
{ "---END---", "", "" } //Keep this line at the end of the list
};
std::map<std::string, bool> vfMap;
// Is a function name one of the valid functions above?
bool isValidFunctionName(std::string f) {
if (vfMap.size() == 0) {
for (int i = 0; ; i++) {
if (validFunctions[i][0] == "---END---") break;
vfMap[validFunctions[i][0]] = true;
}
}
return vfMap.count(f);
}
// Cool function for debug purposes (named cerrStringList to make
// all prints searchable via 'cerr')
void cerrStringList(std::vector<std::string> s, std::string suffix) {
for (unsigned i = 0; i < s.size(); i++) std::cerr << s[i] << " ";
std::cerr << suffix << "\n";
}
// Convert:
// self.cow -> ["cow"]
// self.horse[0] -> ["horse", "0"]
// self.a[6][7][self.storage[3]].chicken[9] ->
// ["6", "7", (sload 3), "chicken", "9"]
std::vector<Node> listfyStorageAccess(Node node) {
std::vector<Node> out;
std::vector<Node> nodez;
nodez.push_back(node);
while (1) {
if (nodez.back().type == TOKEN) {
out.push_back(token("--" + nodez.back().val, node.metadata));
std::vector<Node> outrev;
for (int i = (signed)out.size() - 1; i >= 0; i--) {
outrev.push_back(out[i]);
}
return outrev;
}
if (nodez.back().val == ".")
nodez.back().args[1].val = "--" + nodez.back().args[1].val;
if (nodez.back().args.size() == 0)
err("Error parsing storage variable statement", node.metadata);
if (nodez.back().args.size() == 1)
out.push_back(token(tt256m1, node.metadata));
else
out.push_back(nodez.back().args[1]);
nodez.push_back(nodez.back().args[0]);
}
}
// Is the given node something of the form
// self.cow
// self.horse[0]
// self.a[6][7][self.storage[3]].chicken[9]
bool isNodeStorageVariable(Node node) {
std::vector<Node> nodez;
nodez.push_back(node);
while (1) {
if (nodez.back().type == TOKEN) return false;
if (nodez.back().args.size() == 0) return false;
if (nodez.back().val != "." && nodez.back().val != "access")
return false;
if (nodez.back().args[0].val == "self") return true;
nodez.push_back(nodez.back().args[0]);
}
}
// Main pattern matching routine, for those patterns that can be expressed
// using our standard mini-language above
//
// Returns two values. First, a boolean to determine whether the node matches
// the pattern, second, if the node does match then a map mapping variables
// in the pattern to nodes
matchResult match(Node p, Node n) {
matchResult o;
o.success = false;
if (p.type == TOKEN) {
if (p.val == n.val && n.type == TOKEN) o.success = true;
else if (p.val[0] == '$' || p.val[0] == '@') {
o.success = true;
o.map[p.val.substr(1)] = n;
}
}
else if (n.type==TOKEN || p.val!=n.val || p.args.size()!=n.args.size()) {
// do nothing
}
else {
for (unsigned i = 0; i < p.args.size(); i++) {
matchResult oPrime = match(p.args[i], n.args[i]);
if (!oPrime.success) {
o.success = false;
return o;
}
for (std::map<std::string, Node>::iterator it = oPrime.map.begin();
it != oPrime.map.end();
it++) {
o.map[(*it).first] = (*it).second;
}
}
o.success = true;
}
return o;
}
// Fills in the pattern with a dictionary mapping variable names to
// nodes (these dicts are generated by match). Match and subst together
// create a full pattern-matching engine.
Node subst(Node pattern,
std::map<std::string, Node> dict,
std::string varflag,
Metadata m) {
// Swap out patterns at the token level
if (pattern.metadata.ln == -1)
pattern.metadata = m;
if (pattern.type == TOKEN &&
pattern.val[0] == '$') {
if (dict.count(pattern.val.substr(1))) {
return dict[pattern.val.substr(1)];
}
else {
return token(varflag + pattern.val.substr(1), m);
}
}
// Other tokens are untouched
else if (pattern.type == TOKEN) {
return pattern;
}
// Substitute recursively for ASTs
else {
std::vector<Node> args;
for (unsigned i = 0; i < pattern.args.size(); i++) {
args.push_back(subst(pattern.args[i], dict, varflag, m));
}
return asn(pattern.val, args, m);
}
}
// Transforms a sequence containing two-argument with statements
// into a statement containing those statements in nested form
Node withTransform (Node source) {
Node o = token("--");
Metadata m = source.metadata;
std::vector<Node> args;
for (int i = source.args.size() - 1; i >= 0; i--) {
Node a = source.args[i];
if (a.val == "with" && a.args.size() == 2) {
std::vector<Node> flipargs;
for (int j = args.size() - 1; j >= 0; j--)
flipargs.push_back(args[i]);
if (o.val != "--")
flipargs.push_back(o);
o = asn("with", a.args[0], a.args[1], asn("seq", flipargs, m), m);
args = std::vector<Node>();
}
else {
args.push_back(a);
}
}
std::vector<Node> flipargs;
for (int j = args.size() - 1; j >= 0; j--)
flipargs.push_back(args[j]);
if (o.val != "--")
flipargs.push_back(o);
return asn("seq", flipargs, m);
}

51
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriteutils.h generated vendored
View File

@ -1,51 +0,0 @@
#ifndef ETHSERP_REWRITEUTILS
#define ETHSERP_REWRITEUTILS
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
// Valid functions and their min and max argument counts
extern std::string validFunctions[][3];
extern std::map<std::string, bool> vfMap;
bool isValidFunctionName(std::string f);
// Converts deep array access into ordered list of the arguments
// along the descent
std::vector<Node> listfyStorageAccess(Node node);
// Cool function for debug purposes (named cerrStringList to make
// all prints searchable via 'cerr')
void cerrStringList(std::vector<std::string> s, std::string suffix="");
// Is the given node something of the form
// self.cow
// self.horse[0]
// self.a[6][7][self.storage[3]].chicken[9]
bool isNodeStorageVariable(Node node);
// Applies rewrite rules adding without wrapper
Node rewriteChunk(Node inp);
// Match result storing object
struct matchResult {
bool success;
std::map<std::string, Node> map;
};
// Match node to pattern
matchResult match(Node p, Node n);
// Substitute node using pattern
Node subst(Node pattern,
std::map<std::string, Node> dict,
std::string varflag,
Metadata m);
Node withTransform(Node source);
#endif

201
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/serpent.py generated vendored
View File

@ -1,201 +0,0 @@
import serpent_pyext as pyext
import sys
import re
VERSION = '1.7.7'
class Metadata(object):
def __init__(self, li):
self.file = li[0]
self.ln = li[1]
self.ch = li[2]
def out(self):
return [self.file, self.ln, self.ch]
class Token(object):
def __init__(self, val, metadata):
self.val = val
self.metadata = Metadata(metadata)
def out(self):
return [0, self.val, self.metadata.out()]
def __repr__(self):
return str(self.val)
class Astnode(object):
def __init__(self, val, args, metadata):
self.val = val
self.args = map(node, args)
self.metadata = Metadata(metadata)
def out(self):
o = [1, self.val, self.metadata.out()]+[x.out() for x in self.args]
return o
def __repr__(self):
o = '(' + self.val
subs = map(repr, self.args)
k = 0
out = " "
while k < len(subs) and o != "(seq":
if '\n' in subs[k] or len(out + subs[k]) >= 80:
break
out += subs[k] + " "
k += 1
if k < len(subs):
o += out + "\n "
o += '\n '.join('\n'.join(subs[k:]).split('\n'))
o += '\n)'
else:
o += out[:-1] + ')'
return o
def node(li):
if li[0]:
return Astnode(li[1], li[3:], li[2])
else:
return Token(li[1], li[2])
def take(x):
return pyext.parse_lll(x) if isinstance(x, (str, unicode)) else x.out()
def takelist(x):
return map(take, parse(x).args if isinstance(x, (str, unicode)) else x)
compile = lambda x: pyext.compile(x)
compile_chunk = lambda x: pyext.compile_chunk(x)
compile_to_lll = lambda x: node(pyext.compile_to_lll(x))
compile_chunk_to_lll = lambda x: node(pyext.compile_chunk_to_lll(x))
compile_lll = lambda x: pyext.compile_lll(take(x))
parse = lambda x: node(pyext.parse(x))
rewrite = lambda x: node(pyext.rewrite(take(x)))
rewrite_chunk = lambda x: node(pyext.rewrite_chunk(take(x)))
pretty_compile = lambda x: map(node, pyext.pretty_compile(x))
pretty_compile_chunk = lambda x: map(node, pyext.pretty_compile_chunk(x))
pretty_compile_lll = lambda x: map(node, pyext.pretty_compile_lll(take(x)))
serialize = lambda x: pyext.serialize(takelist(x))
deserialize = lambda x: map(node, pyext.deserialize(x))
is_numeric = lambda x: isinstance(x, (int, long))
is_string = lambda x: isinstance(x, (str, unicode))
tobytearr = lambda n, L: [] if L == 0 else tobytearr(n / 256, L - 1)+[n % 256]
# A set of methods for detecting raw values (numbers and strings) and
# converting them to integers
def frombytes(b):
return 0 if len(b) == 0 else ord(b[-1]) + 256 * frombytes(b[:-1])
def fromhex(b):
hexord = lambda x: '0123456789abcdef'.find(x)
return 0 if len(b) == 0 else hexord(b[-1]) + 16 * fromhex(b[:-1])
def numberize(b):
if is_numeric(b):
return b
elif b[0] in ["'", '"']:
return frombytes(b[1:-1])
elif b[:2] == '0x':
return fromhex(b[2:])
elif re.match('^[0-9]*$', b):
return int(b)
elif len(b) == 40:
return fromhex(b)
else:
raise Exception("Cannot identify data type: %r" % b)
def enc(n):
if is_numeric(n):
return ''.join(map(chr, tobytearr(n, 32)))
elif is_string(n) and len(n) == 40:
return '\x00' * 12 + n.decode('hex')
elif is_string(n):
return '\x00' * (32 - len(n)) + n
elif n is True:
return 1
elif n is False or n is None:
return 0
def encode_datalist(*args):
if isinstance(args, (tuple, list)):
return ''.join(map(enc, args))
elif not len(args) or args[0] == '':
return ''
else:
# Assume you're getting in numbers or addresses or 0x...
return ''.join(map(enc, map(numberize, args)))
def decode_datalist(arr):
if isinstance(arr, list):
arr = ''.join(map(chr, arr))
o = []
for i in range(0, len(arr), 32):
o.append(frombytes(arr[i:i + 32]))
return o
def encode_abi(funid, *args):
len_args = ''
normal_args = ''
var_args = ''
for arg in args:
if isinstance(arg, str) and len(arg) and \
arg[0] == '"' and arg[-1] == '"':
len_args += enc(numberize(len(arg[1:-1])))
var_args += arg[1:-1]
elif isinstance(arg, list):
for a in arg:
var_args += enc(numberize(a))
len_args += enc(numberize(len(arg)))
else:
normal_args += enc(numberize(arg))
return chr(int(funid)) + len_args + normal_args + var_args
def decode_abi(arr, *lens):
o = []
pos = 1
i = 0
if len(lens) == 1 and isinstance(lens[0], list):
lens = lens[0]
while pos < len(arr):
bytez = int(lens[i]) if i < len(lens) else 32
o.append(frombytes(arr[pos: pos + bytez]))
i, pos = i + 1, pos + bytez
return o
def main():
if len(sys.argv) == 1:
print "serpent <command> <arg1> <arg2> ..."
else:
cmd = sys.argv[2] if sys.argv[1] == '-s' else sys.argv[1]
if sys.argv[1] == '-s':
args = [sys.stdin.read()] + sys.argv[3:]
elif sys.argv[1] == '-v':
print VERSION
sys.exit()
else:
cmd = sys.argv[1]
args = sys.argv[2:]
if cmd in ['deserialize', 'decode_datalist', 'decode_abi']:
args[0] = args[0].strip().decode('hex')
o = globals()[cmd](*args)
if isinstance(o, (Token, Astnode, list)):
print repr(o)
else:
print o.encode('hex')

46
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/setup.py generated vendored
View File

@ -1,46 +0,0 @@
from setuptools import setup, Extension
import os
from distutils.sysconfig import get_config_vars
(opt,) = get_config_vars('OPT')
os.environ['OPT'] = " ".join(
flag for flag in opt.split() if flag != '-Wstrict-prototypes'
)
setup(
# Name of this package
name="ethereum-serpent",
# Package version
version='1.7.7',
description='Serpent compiler',
maintainer='Vitalik Buterin',
maintainer_email='v@buterin.com',
license='WTFPL',
url='http://www.ethereum.org/',
# Describes how to build the actual extension module from C source files.
ext_modules=[
Extension(
'serpent_pyext', # Python name of the module
['bignum.cpp', 'util.cpp', 'tokenize.cpp',
'lllparser.cpp', 'parser.cpp', 'functions.cpp',
'optimize.cpp', 'opcodes.cpp',
'rewriteutils.cpp', 'preprocess.cpp', 'rewriter.cpp',
'compiler.cpp', 'funcs.cpp', 'pyserpent.cpp']
)],
py_modules=[
'serpent',
'pyserpent'
],
scripts=[
'serpent.py'
],
entry_points={
'console_scripts': [
'serpent = serpent:main',
],
}
),

115
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/tokenize.cpp generated vendored
View File

@ -1,115 +0,0 @@
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
// These appear as independent tokens even if inside a stream of symbols
const std::string atoms[] = { "#", "//", "(", ")", "[", "]", "{", "}" };
const int numAtoms = 8;
// Is the char alphanumeric, a space, a bracket, a quote, a symbol?
int chartype(char c) {
if (c >= '0' && c <= '9') return ALPHANUM;
else if (c >= 'a' && c <= 'z') return ALPHANUM;
else if (c >= 'A' && c <= 'Z') return ALPHANUM;
else if (std::string("~_$@").find(c) != std::string::npos) return ALPHANUM;
else if (c == '\t' || c == ' ' || c == '\n' || c == '\r') return SPACE;
else if (std::string("()[]{}").find(c) != std::string::npos) return BRACK;
else if (c == '"') return DQUOTE;
else if (c == '\'') return SQUOTE;
else return SYMB;
}
// "y = f(45,124)/3" -> [ "y", "f", "(", "45", ",", "124", ")", "/", "3"]
std::vector<Node> tokenize(std::string inp, Metadata metadata, bool lispMode) {
int curtype = SPACE;
unsigned pos = 0;
int lastNewline = 0;
metadata.ch = 0;
std::string cur;
std::vector<Node> out;
inp += " ";
while (pos < inp.length()) {
int headtype = chartype(inp[pos]);
if (lispMode) {
if (inp[pos] == '\'') headtype = ALPHANUM;
}
// Are we inside a quote?
if (curtype == SQUOTE || curtype == DQUOTE) {
// Close quote
if (headtype == curtype) {
cur += inp[pos];
out.push_back(token(cur, metadata));
cur = "";
metadata.ch = pos - lastNewline;
curtype = SPACE;
pos += 1;
}
// eg. \xc3
else if (inp.length() >= pos + 4 && inp.substr(pos, 2) == "\\x") {
cur += (std::string("0123456789abcdef").find(inp[pos+2]) * 16
+ std::string("0123456789abcdef").find(inp[pos+3]));
pos += 4;
}
// Newline
else if (inp.substr(pos, 2) == "\\n") {
cur += '\n';
pos += 2;
}
// Backslash escape
else if (inp.length() >= pos + 2 && inp[pos] == '\\') {
cur += inp[pos + 1];
pos += 2;
}
// Normal character
else {
cur += inp[pos];
pos += 1;
}
}
else {
// Handle atoms ( '//', '#', brackets )
for (int i = 0; i < numAtoms; i++) {
int split = cur.length() - atoms[i].length();
if (split >= 0 && cur.substr(split) == atoms[i]) {
if (split > 0) {
out.push_back(token(cur.substr(0, split), metadata));
}
metadata.ch += split;
out.push_back(token(cur.substr(split), metadata));
metadata.ch = pos - lastNewline;
cur = "";
curtype = SPACE;
}
}
// Special case the minus sign
if (cur.length() > 1 && (cur.substr(cur.length() - 1) == "-"
|| cur.substr(cur.length() - 1) == "!")) {
out.push_back(token(cur.substr(0, cur.length() - 1), metadata));
out.push_back(token(cur.substr(cur.length() - 1), metadata));
cur = "";
}
// Boundary between different char types
if (headtype != curtype) {
if (curtype != SPACE && cur != "") {
out.push_back(token(cur, metadata));
}
metadata.ch = pos - lastNewline;
cur = "";
}
cur += inp[pos];
curtype = headtype;
pos += 1;
}
if (inp[pos] == '\n') {
lastNewline = pos;
metadata.ch = 0;
metadata.ln += 1;
}
}
return out;
}

16
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/tokenize.h generated vendored
View File

@ -1,16 +0,0 @@
#ifndef ETHSERP_TOKENIZE
#define ETHSERP_TOKENIZE
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
int chartype(char c);
std::vector<Node> tokenize(std::string inp,
Metadata meta=Metadata(),
bool lispMode=false);
#endif

305
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/util.cpp generated vendored
View File

@ -1,305 +0,0 @@
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include "util.h"
#include "bignum.h"
#include <fstream>
#include <cerrno>
//Token or value node constructor
Node token(std::string val, Metadata met) {
Node o;
o.type = 0;
o.val = val;
o.metadata = met;
return o;
}
//AST node constructor
Node astnode(std::string val, std::vector<Node> args, Metadata met) {
Node o;
o.type = 1;
o.val = val;
o.args = args;
o.metadata = met;
return o;
}
//AST node constructors for a specific number of children
Node astnode(std::string val, Metadata met) {
std::vector<Node> args;
return astnode(val, args, met);
}
Node astnode(std::string val, Node a, Metadata met) {
std::vector<Node> args;
args.push_back(a);
return astnode(val, args, met);
}
Node astnode(std::string val, Node a, Node b, Metadata met) {
std::vector<Node> args;
args.push_back(a);
args.push_back(b);
return astnode(val, args, met);
}
Node astnode(std::string val, Node a, Node b, Node c, Metadata met) {
std::vector<Node> args;
args.push_back(a);
args.push_back(b);
args.push_back(c);
return astnode(val, args, met);
}
Node astnode(std::string val, Node a, Node b, Node c, Node d, Metadata met) {
std::vector<Node> args;
args.push_back(a);
args.push_back(b);
args.push_back(c);
args.push_back(d);
return astnode(val, args, met);
}
// Print token list
std::string printTokens(std::vector<Node> tokens) {
std::string s = "";
for (unsigned i = 0; i < tokens.size(); i++) {
s += tokens[i].val + " ";
}
return s;
}
// Prints a lisp AST on one line
std::string printSimple(Node ast) {
if (ast.type == TOKEN) return ast.val;
std::string o = "(" + ast.val;
std::vector<std::string> subs;
for (unsigned i = 0; i < ast.args.size(); i++) {
o += " " + printSimple(ast.args[i]);
}
return o + ")";
}
// Number of tokens in a tree
int treeSize(Node prog) {
if (prog.type == TOKEN) return 1;
int o = 0;
for (unsigned i = 0; i < prog.args.size(); i++) o += treeSize(prog.args[i]);
return o;
}
// Pretty-prints a lisp AST
std::string printAST(Node ast, bool printMetadata) {
if (ast.type == TOKEN) return ast.val;
std::string o = "(";
if (printMetadata) {
o += ast.metadata.file + " ";
o += unsignedToDecimal(ast.metadata.ln) + " ";
o += unsignedToDecimal(ast.metadata.ch) + ": ";
}
o += ast.val;
std::vector<std::string> subs;
for (unsigned i = 0; i < ast.args.size(); i++) {
subs.push_back(printAST(ast.args[i], printMetadata));
}
unsigned k = 0;
std::string out = " ";
// As many arguments as possible go on the same line as the function,
// except when seq is used
while (k < subs.size() && o != "(seq") {
if (subs[k].find("\n") != std::string::npos || (out + subs[k]).length() >= 80) break;
out += subs[k] + " ";
k += 1;
}
// All remaining arguments go on their own lines
if (k < subs.size()) {
o += out + "\n";
std::vector<std::string> subsSliceK;
for (unsigned i = k; i < subs.size(); i++) subsSliceK.push_back(subs[i]);
o += indentLines(joinLines(subsSliceK));
o += "\n)";
}
else {
o += out.substr(0, out.size() - 1) + ")";
}
return o;
}
// Splits text by line
std::vector<std::string> splitLines(std::string s) {
unsigned pos = 0;
int lastNewline = 0;
std::vector<std::string> o;
while (pos < s.length()) {
if (s[pos] == '\n') {
o.push_back(s.substr(lastNewline, pos - lastNewline));
lastNewline = pos + 1;
}
pos = pos + 1;
}
o.push_back(s.substr(lastNewline));
return o;
}
// Inverse of splitLines
std::string joinLines(std::vector<std::string> lines) {
std::string o = "\n";
for (unsigned i = 0; i < lines.size(); i++) {
o += lines[i] + "\n";
}
return o.substr(1, o.length() - 2);
}
// Indent all lines by 4 spaces
std::string indentLines(std::string inp) {
std::vector<std::string> lines = splitLines(inp);
for (unsigned i = 0; i < lines.size(); i++) lines[i] = " "+lines[i];
return joinLines(lines);
}
// Binary to hexadecimal
std::string binToNumeric(std::string inp) {
std::string o = "0";
for (unsigned i = 0; i < inp.length(); i++) {
o = decimalAdd(decimalMul(o,"256"), unsignedToDecimal((unsigned char)inp[i]));
}
return o;
}
// Converts string to simple numeric format
std::string strToNumeric(std::string inp) {
std::string o = "0";
if (inp == "") {
o = "";
}
else if (inp.substr(0,2) == "0x") {
for (unsigned i = 2; i < inp.length(); i++) {
int dig = std::string("0123456789abcdef0123456789ABCDEF").find(inp[i]) % 16;
if (dig < 0) return "";
o = decimalAdd(decimalMul(o,"16"), unsignedToDecimal(dig));
}
}
else {
bool isPureNum = true;
for (unsigned i = 0; i < inp.length(); i++) {
isPureNum = isPureNum && inp[i] >= '0' && inp[i] <= '9';
}
o = isPureNum ? inp : "";
}
return o;
}
// Does the node contain a number (eg. 124, 0xf012c, "george")
bool isNumberLike(Node node) {
if (node.type == ASTNODE) return false;
return strToNumeric(node.val) != "";
}
//Normalizes number representations
Node nodeToNumeric(Node node) {
std::string o = strToNumeric(node.val);
return token(o == "" ? node.val : o, node.metadata);
}
Node tryNumberize(Node node) {
if (node.type == TOKEN && isNumberLike(node)) return nodeToNumeric(node);
return node;
}
//Converts a value to an array of byte number nodes
std::vector<Node> toByteArr(std::string val, Metadata metadata, int minLen) {
std::vector<Node> o;
int L = 0;
while (val != "0" || L < minLen) {
o.push_back(token(decimalMod(val, "256"), metadata));
val = decimalDiv(val, "256");
L++;
}
std::vector<Node> o2;
for (int i = o.size() - 1; i >= 0; i--) o2.push_back(o[i]);
return o2;
}
int counter = 0;
//Makes a unique token
std::string mkUniqueToken() {
counter++;
return unsignedToDecimal(counter);
}
//Does a file exist? http://stackoverflow.com/questions/12774207
bool exists(std::string fileName) {
std::ifstream infile(fileName.c_str());
return infile.good();
}
//Reads a file: http://stackoverflow.com/questions/2602013
std::string get_file_contents(std::string filename)
{
std::ifstream in(filename.c_str(), std::ios::in | std::ios::binary);
if (in)
{
std::string contents;
in.seekg(0, std::ios::end);
contents.resize(in.tellg());
in.seekg(0, std::ios::beg);
in.read(&contents[0], contents.size());
in.close();
return(contents);
}
throw(errno);
}
//Report error
void err(std::string errtext, Metadata met) {
std::string err = "Error (file \"" + met.file + "\", line " +
unsignedToDecimal(met.ln + 1) + ", char " + unsignedToDecimal(met.ch) +
"): " + errtext;
std::cerr << err << "\n";
throw(err);
}
//Bin to hex
std::string binToHex(std::string inp) {
std::string o = "";
for (unsigned i = 0; i < inp.length(); i++) {
unsigned char v = inp[i];
o += std::string("0123456789abcdef").substr(v/16, 1)
+ std::string("0123456789abcdef").substr(v%16, 1);
}
return o;
}
//Hex to bin
std::string hexToBin(std::string inp) {
std::string o = "";
for (unsigned i = 0; i+1 < inp.length(); i+=2) {
char v = (char)(std::string("0123456789abcdef").find(inp[i]) * 16 +
std::string("0123456789abcdef").find(inp[i+1]));
o += v;
}
return o;
}
//Lower to upper
std::string upperCase(std::string inp) {
std::string o = "";
for (unsigned i = 0; i < inp.length(); i++) {
if (inp[i] >= 97 && inp[i] <= 122) o += inp[i] - 32;
else o += inp[i];
}
return o;
}
//Three-int vector
std::vector<int> triple(int a, int b, int c) {
std::vector<int> v;
v.push_back(a);
v.push_back(b);
v.push_back(c);
return v;
}

127
Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/util.h generated vendored
View File

@ -1,127 +0,0 @@
#ifndef ETHSERP_UTIL
#define ETHSERP_UTIL
#include <stdio.h>
#include <iostream>
#include <vector>
#include <map>
#include <fstream>
#include <cerrno>
const int TOKEN = 0,
ASTNODE = 1,
SPACE = 2,
BRACK = 3,
SQUOTE = 4,
DQUOTE = 5,
SYMB = 6,
ALPHANUM = 7,
LPAREN = 8,
RPAREN = 9,
COMMA = 10,
COLON = 11,
UNARY_OP = 12,
BINARY_OP = 13,
COMPOUND = 14,
TOKEN_SPLITTER = 15;
// Stores metadata about each token
class Metadata {
public:
Metadata(std::string File="main", int Ln=-1, int Ch=-1) {
file = File;
ln = Ln;
ch = Ch;
fixed = false;
}
std::string file;
int ln;
int ch;
bool fixed;
};
std::string mkUniqueToken();
// type can be TOKEN or ASTNODE
struct Node {
int type;
std::string val;
std::vector<Node> args;
Metadata metadata;
};
Node token(std::string val, Metadata met=Metadata());
Node astnode(std::string val, std::vector<Node> args, Metadata met=Metadata());
Node astnode(std::string val, Metadata met=Metadata());
Node astnode(std::string val, Node a, Metadata met=Metadata());
Node astnode(std::string val, Node a, Node b, Metadata met=Metadata());
Node astnode(std::string val, Node a, Node b, Node c, Metadata met=Metadata());
Node astnode(std::string val, Node a, Node b,
Node c, Node d, Metadata met=Metadata());
// Number of tokens in a tree
int treeSize(Node prog);
// Print token list
std::string printTokens(std::vector<Node> tokens);
// Prints a lisp AST on one line
std::string printSimple(Node ast);
// Pretty-prints a lisp AST
std::string printAST(Node ast, bool printMetadata=false);
// Splits text by line
std::vector<std::string> splitLines(std::string s);
// Inverse of splitLines
std::string joinLines(std::vector<std::string> lines);
// Indent all lines by 4 spaces
std::string indentLines(std::string inp);
// Converts binary to simple numeric format
std::string binToNumeric(std::string inp);
// Converts string to simple numeric format
std::string strToNumeric(std::string inp);
// Does the node contain a number (eg. 124, 0xf012c, "george")
bool isNumberLike(Node node);
//Normalizes number representations
Node nodeToNumeric(Node node);
//If a node is numeric, normalize its representation
Node tryNumberize(Node node);
//Converts a value to an array of byte number nodes
std::vector<Node> toByteArr(std::string val, Metadata metadata, int minLen=1);
//Reads a file
std::string get_file_contents(std::string filename);
//Does a file exist?
bool exists(std::string fileName);
//Report error
void err(std::string errtext, Metadata met);
//Bin to hex
std::string binToHex(std::string inp);
//Hex to bin
std::string hexToBin(std::string inp);
//Lower to upper
std::string upperCase(std::string inp);
//Three-int vector
std::vector<int> triple(int a, int b, int c);
#define asn astnode
#define tkn token
#define msi std::map<std::string, int>
#define msn std::map<std::string, Node>
#define mss std::map<std::string, std::string>
#endif

21
Godeps/_workspace/src/github.com/ethereum/serpent-go/tests/main.go generated vendored
View File

@ -1,21 +0,0 @@
package main
import (
"fmt"
"github.com/ethereum/serpent-go"
)
func main() {
out, _ := serpent.Compile(`
// Namecoin
if !contract.storage[msg.data[0]]: # Is the key not yet taken?
# Then take it!
contract.storage[msg.data[0]] = msg.data[1]
return(1)
else:
return(0) // Otherwise do nothing
`)
fmt.Printf("%x\n", out)
}

228
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch.go generated vendored
View File

@ -8,65 +8,84 @@ package leveldb
import (
"encoding/binary"
"errors"
"fmt"
"github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/memdb"
)
var (
errBatchTooShort = errors.New("leveldb: batch is too short")
errBatchBadRecord = errors.New("leveldb: bad record in batch")
type ErrBatchCorrupted struct {
Reason string
}
func (e *ErrBatchCorrupted) Error() string {
return fmt.Sprintf("leveldb: batch corrupted: %s", e.Reason)
}
func newErrBatchCorrupted(reason string) error {
return errors.NewErrCorrupted(nil, &ErrBatchCorrupted{reason})
}
const (
batchHdrLen = 8 + 4
batchGrowRec = 3000
)
const kBatchHdrLen = 8 + 4
type batchReplay interface {
put(key, value []byte, seq uint64)
delete(key []byte, seq uint64)
type BatchReplay interface {
Put(key, value []byte)
Delete(key []byte)
}
// Batch is a write batch.
type Batch struct {
buf []byte
data []byte
rLen, bLen int
seq uint64
sync bool
}
func (b *Batch) grow(n int) {
off := len(b.buf)
off := len(b.data)
if off == 0 {
// include headers
off = kBatchHdrLen
n += off
off = batchHdrLen
if b.data != nil {
b.data = b.data[:off]
}
}
if cap(b.data)-off < n {
if b.data == nil {
b.data = make([]byte, off, off+n)
} else {
odata := b.data
div := 1
if b.rLen > batchGrowRec {
div = b.rLen / batchGrowRec
}
b.data = make([]byte, off, off+n+(off-batchHdrLen)/div)
copy(b.data, odata)
}
if cap(b.buf)-off >= n {
return
}
buf := make([]byte, 2*cap(b.buf)+n)
copy(buf, b.buf)
b.buf = buf[:off]
}
func (b *Batch) appendRec(t vType, key, value []byte) {
func (b *Batch) appendRec(kt kType, key, value []byte) {
n := 1 + binary.MaxVarintLen32 + len(key)
if t == tVal {
if kt == ktVal {
n += binary.MaxVarintLen32 + len(value)
}
b.grow(n)
off := len(b.buf)
buf := b.buf[:off+n]
buf[off] = byte(t)
off := len(b.data)
data := b.data[:off+n]
data[off] = byte(kt)
off += 1
off += binary.PutUvarint(buf[off:], uint64(len(key)))
copy(buf[off:], key)
off += binary.PutUvarint(data[off:], uint64(len(key)))
copy(data[off:], key)
off += len(key)
if t == tVal {
off += binary.PutUvarint(buf[off:], uint64(len(value)))
copy(buf[off:], value)
if kt == ktVal {
off += binary.PutUvarint(data[off:], uint64(len(value)))
copy(data[off:], value)
off += len(value)
}
b.buf = buf[:off]
b.data = data[:off]
b.rLen++
// Include 8-byte ikey header
b.bLen += len(key) + len(value) + 8
@ -75,18 +94,51 @@ func (b *Batch) appendRec(t vType, key, value []byte) {
// Put appends 'put operation' of the given key/value pair to the batch.
// It is safe to modify the contents of the argument after Put returns.
func (b *Batch) Put(key, value []byte) {
b.appendRec(tVal, key, value)
b.appendRec(ktVal, key, value)
}
// Delete appends 'delete operation' of the given key to the batch.
// It is safe to modify the contents of the argument after Delete returns.
func (b *Batch) Delete(key []byte) {
b.appendRec(tDel, key, nil)
b.appendRec(ktDel, key, nil)
}
// Dump dumps batch contents. The returned slice can be loaded into the
// batch using Load method.
// The returned slice is not its own copy, so the contents should not be
// modified.
func (b *Batch) Dump() []byte {
return b.encode()
}
// Load loads given slice into the batch. Previous contents of the batch
// will be discarded.
// The given slice will not be copied and will be used as batch buffer, so
// it is not safe to modify the contents of the slice.
func (b *Batch) Load(data []byte) error {
return b.decode(0, data)
}
// Replay replays batch contents.
func (b *Batch) Replay(r BatchReplay) error {
return b.decodeRec(func(i int, kt kType, key, value []byte) {
switch kt {
case ktVal:
r.Put(key, value)
case ktDel:
r.Delete(key)
}
})
}
// Len returns number of records in the batch.
func (b *Batch) Len() int {
return b.rLen
}
// Reset resets the batch.
func (b *Batch) Reset() {
b.buf = nil
b.data = b.data[:0]
b.seq = 0
b.rLen = 0
b.bLen = 0
@ -97,24 +149,10 @@ func (b *Batch) init(sync bool) {
b.sync = sync
}
func (b *Batch) put(key, value []byte, seq uint64) {
if b.rLen == 0 {
b.seq = seq
}
b.Put(key, value)
}
func (b *Batch) delete(key []byte, seq uint64) {
if b.rLen == 0 {
b.seq = seq
}
b.Delete(key)
}
func (b *Batch) append(p *Batch) {
if p.rLen > 0 {
b.grow(len(p.buf) - kBatchHdrLen)
b.buf = append(b.buf, p.buf[kBatchHdrLen:]...)
b.grow(len(p.data) - batchHdrLen)
b.data = append(b.data, p.data[batchHdrLen:]...)
b.rLen += p.rLen
}
if p.sync {
@ -122,95 +160,93 @@ func (b *Batch) append(p *Batch) {
}
}
func (b *Batch) len() int {
return b.rLen
}
// size returns sums of key/value pair length plus 8-bytes ikey.
func (b *Batch) size() int {
return b.bLen
}
func (b *Batch) encode() []byte {
b.grow(0)
binary.LittleEndian.PutUint64(b.buf, b.seq)
binary.LittleEndian.PutUint32(b.buf[8:], uint32(b.rLen))
binary.LittleEndian.PutUint64(b.data, b.seq)
binary.LittleEndian.PutUint32(b.data[8:], uint32(b.rLen))
return b.buf
return b.data
}
func (b *Batch) decode(buf []byte) error {
if len(buf) < kBatchHdrLen {
return errBatchTooShort
func (b *Batch) decode(prevSeq uint64, data []byte) error {
if len(data) < batchHdrLen {
return newErrBatchCorrupted("too short")
}
b.seq = binary.LittleEndian.Uint64(buf)
b.rLen = int(binary.LittleEndian.Uint32(buf[8:]))
b.seq = binary.LittleEndian.Uint64(data)
if b.seq < prevSeq {
return newErrBatchCorrupted("invalid sequence number")
}
b.rLen = int(binary.LittleEndian.Uint32(data[8:]))
if b.rLen < 0 {
return newErrBatchCorrupted("invalid records length")
}
// No need to be precise at this point, it won't be used anyway
b.bLen = len(buf) - kBatchHdrLen
b.buf = buf
b.bLen = len(data) - batchHdrLen
b.data = data
return nil
}
func (b *Batch) decodeRec(f func(i int, t vType, key, value []byte)) error {
off := kBatchHdrLen
func (b *Batch) decodeRec(f func(i int, kt kType, key, value []byte)) (err error) {
off := batchHdrLen
for i := 0; i < b.rLen; i++ {
if off >= len(b.buf) {
return errors.New("leveldb: invalid batch record length")
if off >= len(b.data) {
return newErrBatchCorrupted("invalid records length")
}
t := vType(b.buf[off])
if t > tVal {
return errors.New("leveldb: invalid batch record type in batch")
kt := kType(b.data[off])
if kt > ktVal {
return newErrBatchCorrupted("bad record: invalid type")
}
off += 1
x, n := binary.Uvarint(b.buf[off:])
x, n := binary.Uvarint(b.data[off:])
off += n
if n <= 0 || off+int(x) > len(b.buf) {
return errBatchBadRecord
if n <= 0 || off+int(x) > len(b.data) {
return newErrBatchCorrupted("bad record: invalid key length")
}
key := b.buf[off : off+int(x)]
key := b.data[off : off+int(x)]
off += int(x)
var value []byte
if t == tVal {
x, n := binary.Uvarint(b.buf[off:])
if kt == ktVal {
x, n := binary.Uvarint(b.data[off:])
off += n
if n <= 0 || off+int(x) > len(b.buf) {
return errBatchBadRecord
if n <= 0 || off+int(x) > len(b.data) {
return newErrBatchCorrupted("bad record: invalid value length")
}
value = b.buf[off : off+int(x)]
value = b.data[off : off+int(x)]
off += int(x)
}
f(i, t, key, value)
f(i, kt, key, value)
}
return nil
}
func (b *Batch) replay(to batchReplay) error {
return b.decodeRec(func(i int, t vType, key, value []byte) {
switch t {
case tVal:
to.put(key, value, b.seq+uint64(i))
case tDel:
to.delete(key, b.seq+uint64(i))
}
})
}
func (b *Batch) memReplay(to *memdb.DB) error {
return b.decodeRec(func(i int, t vType, key, value []byte) {
ikey := newIKey(key, b.seq+uint64(i), t)
return b.decodeRec(func(i int, kt kType, key, value []byte) {
ikey := newIkey(key, b.seq+uint64(i), kt)
to.Put(ikey, value)
})
}
func (b *Batch) memDecodeAndReplay(prevSeq uint64, data []byte, to *memdb.DB) error {
if err := b.decode(prevSeq, data); err != nil {
return err
}
return b.memReplay(to)
}
func (b *Batch) revertMemReplay(to *memdb.DB) error {
return b.decodeRec(func(i int, t vType, key, value []byte) {
ikey := newIKey(key, b.seq+uint64(i), t)
return b.decodeRec(func(i int, kt kType, key, value []byte) {
ikey := newIkey(key, b.seq+uint64(i), kt)
to.Delete(ikey)
})
}

26
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch_test.go generated vendored
View File

@ -15,7 +15,7 @@ import (
)
type tbRec struct {
t vType
kt kType
key, value []byte
}
@ -23,39 +23,39 @@ type testBatch struct {
rec []*tbRec
}
func (p *testBatch) put(key, value []byte, seq uint64) {
p.rec = append(p.rec, &tbRec{tVal, key, value})
func (p *testBatch) Put(key, value []byte) {
p.rec = append(p.rec, &tbRec{ktVal, key, value})
}
func (p *testBatch) delete(key []byte, seq uint64) {
p.rec = append(p.rec, &tbRec{tDel, key, nil})
func (p *testBatch) Delete(key []byte) {
p.rec = append(p.rec, &tbRec{ktDel, key, nil})
}
func compareBatch(t *testing.T, b1, b2 *Batch) {
if b1.seq != b2.seq {
t.Errorf("invalid seq number want %d, got %d", b1.seq, b2.seq)
}
if b1.len() != b2.len() {
t.Fatalf("invalid record length want %d, got %d", b1.len(), b2.len())
if b1.Len() != b2.Len() {
t.Fatalf("invalid record length want %d, got %d", b1.Len(), b2.Len())
}
p1, p2 := new(testBatch), new(testBatch)
err := b1.replay(p1)
err := b1.Replay(p1)
if err != nil {
t.Fatal("error when replaying batch 1: ", err)
}
err = b2.replay(p2)
err = b2.Replay(p2)
if err != nil {
t.Fatal("error when replaying batch 2: ", err)
}
for i := range p1.rec {
r1, r2 := p1.rec[i], p2.rec[i]
if r1.t != r2.t {
t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.t, r2.t)
if r1.kt != r2.kt {
t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.kt, r2.kt)
}
if !bytes.Equal(r1.key, r2.key) {
t.Errorf("invalid key on record '%d' want %s, got %s", i, string(r1.key), string(r2.key))
}
if r1.t == tVal {
if r1.kt == ktVal {
if !bytes.Equal(r1.value, r2.value) {
t.Errorf("invalid value on record '%d' want %s, got %s", i, string(r1.value), string(r2.value))
}
@ -75,7 +75,7 @@ func TestBatch_EncodeDecode(t *testing.T) {
b1.Delete([]byte("k"))
buf := b1.encode()
b2 := new(Batch)
err := b2.decode(buf)
err := b2.decode(0, buf)
if err != nil {
t.Error("error when decoding batch: ", err)
}

58
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench2_test.go generated vendored Normal file
View File

@ -0,0 +1,58 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// +build !go1.2
package leveldb
import (
"sync/atomic"
"testing"
)
func BenchmarkDBReadConcurrent(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
p.gc()
defer p.close()
b.ResetTimer()
b.SetBytes(116)
b.RunParallel(func(pb *testing.PB) {
iter := p.newIter()
defer iter.Release()
for pb.Next() && iter.Next() {
}
})
}
func BenchmarkDBReadConcurrent2(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
p.gc()
defer p.close()
b.ResetTimer()
b.SetBytes(116)
var dir uint32
b.RunParallel(func(pb *testing.PB) {
iter := p.newIter()
defer iter.Release()
if atomic.AddUint32(&dir, 1)%2 == 0 {
for pb.Next() && iter.Next() {
}
} else {
if pb.Next() && iter.Last() {
for pb.Next() && iter.Prev() {
}
}
}
})
}

15
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench_test.go generated vendored
View File

@ -170,7 +170,7 @@ func (p *dbBench) writes(perBatch int) {
b.SetBytes(116)
}
func (p *dbBench) drop() {
func (p *dbBench) gc() {
p.keys, p.values = nil, nil
runtime.GC()
}
@ -249,6 +249,9 @@ func (p *dbBench) newIter() iterator.Iterator {
}
func (p *dbBench) close() {
if bp, err := p.db.GetProperty("leveldb.blockpool"); err == nil {
p.b.Log("Block pool stats: ", bp)
}
p.db.Close()
p.stor.Close()
os.RemoveAll(benchDB)
@ -331,7 +334,7 @@ func BenchmarkDBRead(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
p.drop()
p.gc()
iter := p.newIter()
b.ResetTimer()
@ -362,7 +365,7 @@ func BenchmarkDBReadUncompressed(b *testing.B) {
p := openDBBench(b, true)
p.populate(b.N)
p.fill()
p.drop()
p.gc()
iter := p.newIter()
b.ResetTimer()
@ -379,7 +382,7 @@ func BenchmarkDBReadTable(b *testing.B) {
p.populate(b.N)
p.fill()
p.reopen()
p.drop()
p.gc()
iter := p.newIter()
b.ResetTimer()
@ -395,7 +398,7 @@ func BenchmarkDBReadReverse(b *testing.B) {
p := openDBBench(b, false)
p.populate(b.N)
p.fill()
p.drop()
p.gc()
iter := p.newIter()
b.ResetTimer()
@ -413,7 +416,7 @@ func BenchmarkDBReadReverseTable(b *testing.B) {
p.populate(b.N)
p.fill()
p.reopen()
p.drop()
p.gc()
iter := p.newIter()
b.ResetTimer()

30
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/bench2_test.go generated vendored Normal file
View File

@ -0,0 +1,30 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// +build !go1.2
package cache
import (
"math/rand"
"testing"
)
func BenchmarkLRUCache(b *testing.B) {
c := NewCache(NewLRU(10000))
b.SetParallelism(10)
b.RunParallel(func(pb *testing.PB) {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for pb.Next() {
key := uint64(r.Intn(1000000))
c.Get(0, key, func() (int, Value) {
return 1, key
}).Release()
}
})
}

721
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go generated vendored
View File

@ -8,118 +8,669 @@
package cache
import (
"sync"
"sync/atomic"
"unsafe"
"github.com/syndtr/goleveldb/leveldb/util"
)
// SetFunc used by Namespace.Get method to create a cache object. SetFunc
// may return ok false, in that case the cache object will not be created.
type SetFunc func() (ok bool, value interface{}, charge int, fin SetFin)
// Cacher provides interface to implements a caching functionality.
// An implementation must be goroutine-safe.
type Cacher interface {
// Capacity returns cache capacity.
Capacity() int
// SetFin will be called when corresponding cache object are released.
type SetFin func()
// DelFin will be called when corresponding cache object are released.
// DelFin will be called after SetFin. The exist is true if the corresponding
// cache object is actually exist in the cache tree.
type DelFin func(exist bool)
// PurgeFin will be called when corresponding cache object are released.
// PurgeFin will be called after SetFin. If PurgeFin present DelFin will
// not be executed but passed to the PurgeFin, it is up to the caller
// to call it or not.
type PurgeFin func(ns, key uint64, delfin DelFin)
// Cache is a cache tree.
type Cache interface {
// SetCapacity sets cache capacity.
SetCapacity(capacity int)
// GetNamespace gets or creates a cache namespace for the given id.
GetNamespace(id uint64) Namespace
// Promote promotes the 'cache node'.
Promote(n *Node)
// Purge purges all cache namespaces, read Namespace.Purge method documentation.
Purge(fin PurgeFin)
// Ban evicts the 'cache node' and prevent subsequent 'promote'.
Ban(n *Node)
// Zap zaps all cache namespaces, read Namespace.Zap method documentation.
Zap(closed bool)
// Evict evicts the 'cache node'.
Evict(n *Node)
// EvictNS evicts 'cache node' with the given namespace.
EvictNS(ns uint64)
// EvictAll evicts all 'cache node'.
EvictAll()
// Close closes the 'cache tree'
Close() error
}
// Namespace is a cache namespace.
type Namespace interface {
// Get gets cache object for the given key. The given SetFunc (if not nil) will
// be called if the given key does not exist.
// If the given key does not exist, SetFunc is nil or SetFunc return ok false, Get
// will return ok false.
Get(key uint64, setf SetFunc) (obj Object, ok bool)
// Value is a 'cacheable object'. It may implements util.Releaser, if
// so the the Release method will be called once object is released.
type Value interface{}
// Get deletes cache object for the given key. If exist the cache object will
// be deleted later when all of its handles have been released (i.e. no one use
// it anymore) and the given DelFin (if not nil) will finally be executed. If
// such cache object does not exist the given DelFin will be executed anyway.
//
// Delete returns true if such cache object exist.
Delete(key uint64, fin DelFin) bool
// Purge deletes all cache objects, read Delete method documentation.
Purge(fin PurgeFin)
// Zap detaches the namespace from the cache tree and delete all its cache
// objects. The cache objects deletion and finalizers execution are happen
// immediately, even if its existing handles haven't yet been released.
// A zapped namespace can't never be filled again.
// If closed is false then the Get function will always call the given SetFunc
// if it is not nil, but resultant of the SetFunc will not be cached.
Zap(closed bool)
type CacheGetter struct {
Cache *Cache
NS uint64
}
// Object is a cache object.
type Object interface {
// Release releases the cache object. Other methods should not be called
// after the cache object has been released.
Release()
// Value returns value of the cache object.
Value() interface{}
func (g *CacheGetter) Get(key uint64, setFunc func() (size int, value Value)) *Handle {
return g.Cache.Get(g.NS, key, setFunc)
}
// Namespace state.
type nsState int
// The hash tables implementation is based on:
// "Dynamic-Sized Nonblocking Hash Tables", by Yujie Liu, Kunlong Zhang, and Michael Spear. ACM Symposium on Principles of Distributed Computing, Jul 2014.
const (
nsEffective nsState = iota
nsZapped
nsClosed
mInitialSize = 1 << 4
mOverflowThreshold = 1 << 5
mOverflowGrowThreshold = 1 << 7
)
// Node state.
type nodeState int
const (
nodeEffective nodeState = iota
nodeEvicted
nodeRemoved
)
// Fake object.
type fakeObject struct {
value interface{}
fin func()
once uint32
type mBucket struct {
mu sync.Mutex
node []*Node
frozen bool
}
func (o *fakeObject) Value() interface{} {
if atomic.LoadUint32(&o.once) == 0 {
return o.value
func (b *mBucket) freeze() []*Node {
b.mu.Lock()
defer b.mu.Unlock()
if !b.frozen {
b.frozen = true
}
return b.node
}
func (b *mBucket) get(r *Cache, h *mNode, hash uint32, ns, key uint64, noset bool) (done, added bool, n *Node) {
b.mu.Lock()
if b.frozen {
b.mu.Unlock()
return
}
// Scan the node.
for _, n := range b.node {
if n.hash == hash && n.ns == ns && n.key == key {
atomic.AddInt32(&n.ref, 1)
b.mu.Unlock()
return true, false, n
}
}
// Get only.
if noset {
b.mu.Unlock()
return true, false, nil
}
// Create node.
n = &Node{
r: r,
hash: hash,
ns: ns,
key: key,
ref: 1,
}
// Add node to bucket.
b.node = append(b.node, n)
bLen := len(b.node)
b.mu.Unlock()
// Update counter.
grow := atomic.AddInt32(&r.nodes, 1) >= h.growThreshold
if bLen > mOverflowThreshold {
grow = grow || atomic.AddInt32(&h.overflow, 1) >= mOverflowGrowThreshold
}
// Grow.
if grow && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
nhLen := len(h.buckets) << 1
nh := &mNode{
buckets: make([]unsafe.Pointer, nhLen),
mask: uint32(nhLen) - 1,
pred: unsafe.Pointer(h),
growThreshold: int32(nhLen * mOverflowThreshold),
shrinkThreshold: int32(nhLen >> 1),
}
ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
if !ok {
panic("BUG: failed swapping head")
}
go nh.initBuckets()
}
return true, true, n
}
func (b *mBucket) delete(r *Cache, h *mNode, hash uint32, ns, key uint64) (done, deleted bool) {
b.mu.Lock()
if b.frozen {
b.mu.Unlock()
return
}
// Scan the node.
var (
n *Node
bLen int
)
for i := range b.node {
n = b.node[i]
if n.ns == ns && n.key == key {
if atomic.LoadInt32(&n.ref) == 0 {
deleted = true
// Call releaser.
if n.value != nil {
if r, ok := n.value.(util.Releaser); ok {
r.Release()
}
n.value = nil
}
// Remove node from bucket.
b.node = append(b.node[:i], b.node[i+1:]...)
bLen = len(b.node)
}
break
}
}
b.mu.Unlock()
if deleted {
// Call OnDel.
for _, f := range n.onDel {
f()
}
// Update counter.
atomic.AddInt32(&r.size, int32(n.size)*-1)
shrink := atomic.AddInt32(&r.nodes, -1) < h.shrinkThreshold
if bLen >= mOverflowThreshold {
atomic.AddInt32(&h.overflow, -1)
}
// Shrink.
if shrink && len(h.buckets) > mInitialSize && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
nhLen := len(h.buckets) >> 1
nh := &mNode{
buckets: make([]unsafe.Pointer, nhLen),
mask: uint32(nhLen) - 1,
pred: unsafe.Pointer(h),
growThreshold: int32(nhLen * mOverflowThreshold),
shrinkThreshold: int32(nhLen >> 1),
}
ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
if !ok {
panic("BUG: failed swapping head")
}
go nh.initBuckets()
}
}
return true, deleted
}
type mNode struct {
buckets []unsafe.Pointer // []*mBucket
mask uint32
pred unsafe.Pointer // *mNode
resizeInProgess int32
overflow int32
growThreshold int32
shrinkThreshold int32
}
func (n *mNode) initBucket(i uint32) *mBucket {
if b := (*mBucket)(atomic.LoadPointer(&n.buckets[i])); b != nil {
return b
}
p := (*mNode)(atomic.LoadPointer(&n.pred))
if p != nil {
var node []*Node
if n.mask > p.mask {
// Grow.
pb := (*mBucket)(atomic.LoadPointer(&p.buckets[i&p.mask]))
if pb == nil {
pb = p.initBucket(i & p.mask)
}
m := pb.freeze()
// Split nodes.
for _, x := range m {
if x.hash&n.mask == i {
node = append(node, x)
}
}
} else {
// Shrink.
pb0 := (*mBucket)(atomic.LoadPointer(&p.buckets[i]))
if pb0 == nil {
pb0 = p.initBucket(i)
}
pb1 := (*mBucket)(atomic.LoadPointer(&p.buckets[i+uint32(len(n.buckets))]))
if pb1 == nil {
pb1 = p.initBucket(i + uint32(len(n.buckets)))
}
m0 := pb0.freeze()
m1 := pb1.freeze()
// Merge nodes.
node = make([]*Node, 0, len(m0)+len(m1))
node = append(node, m0...)
node = append(node, m1...)
}
b := &mBucket{node: node}
if atomic.CompareAndSwapPointer(&n.buckets[i], nil, unsafe.Pointer(b)) {
if len(node) > mOverflowThreshold {
atomic.AddInt32(&n.overflow, int32(len(node)-mOverflowThreshold))
}
return b
}
}
return (*mBucket)(atomic.LoadPointer(&n.buckets[i]))
}
func (n *mNode) initBuckets() {
for i := range n.buckets {
n.initBucket(uint32(i))
}
atomic.StorePointer(&n.pred, nil)
}
// Cache is a 'cache map'.
type Cache struct {
mu sync.RWMutex
mHead unsafe.Pointer // *mNode
nodes int32
size int32
cacher Cacher
closed bool
}
// NewCache creates a new 'cache map'. The cacher is optional and
// may be nil.
func NewCache(cacher Cacher) *Cache {
h := &mNode{
buckets: make([]unsafe.Pointer, mInitialSize),
mask: mInitialSize - 1,
growThreshold: int32(mInitialSize * mOverflowThreshold),
shrinkThreshold: 0,
}
for i := range h.buckets {
h.buckets[i] = unsafe.Pointer(&mBucket{})
}
r := &Cache{
mHead: unsafe.Pointer(h),
cacher: cacher,
}
return r
}
func (r *Cache) getBucket(hash uint32) (*mNode, *mBucket) {
h := (*mNode)(atomic.LoadPointer(&r.mHead))
i := hash & h.mask
b := (*mBucket)(atomic.LoadPointer(&h.buckets[i]))
if b == nil {
b = h.initBucket(i)
}
return h, b
}
func (r *Cache) delete(n *Node) bool {
for {
h, b := r.getBucket(n.hash)
done, deleted := b.delete(r, h, n.hash, n.ns, n.key)
if done {
return deleted
}
}
return false
}
// Nodes returns number of 'cache node' in the map.
func (r *Cache) Nodes() int {
return int(atomic.LoadInt32(&r.nodes))
}
// Size returns sums of 'cache node' size in the map.
func (r *Cache) Size() int {
return int(atomic.LoadInt32(&r.size))
}
// Capacity returns cache capacity.
func (r *Cache) Capacity() int {
if r.cacher == nil {
return 0
}
return r.cacher.Capacity()
}
// SetCapacity sets cache capacity.
func (r *Cache) SetCapacity(capacity int) {
if r.cacher != nil {
r.cacher.SetCapacity(capacity)
}
}
// Get gets 'cache node' with the given namespace and key.
// If cache node is not found and setFunc is not nil, Get will atomically creates
// the 'cache node' by calling setFunc. Otherwise Get will returns nil.
//
// The returned 'cache handle' should be released after use by calling Release
// method.
func (r *Cache) Get(ns, key uint64, setFunc func() (size int, value Value)) *Handle {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return nil
}
hash := murmur32(ns, key, 0xf00)
for {
h, b := r.getBucket(hash)
done, _, n := b.get(r, h, hash, ns, key, setFunc == nil)
if done {
if n != nil {
n.mu.Lock()
if n.value == nil {
if setFunc == nil {
n.mu.Unlock()
n.unref()
return nil
}
n.size, n.value = setFunc()
if n.value == nil {
n.size = 0
n.mu.Unlock()
n.unref()
return nil
}
atomic.AddInt32(&r.size, int32(n.size))
}
n.mu.Unlock()
if r.cacher != nil {
r.cacher.Promote(n)
}
return &Handle{unsafe.Pointer(n)}
}
break
}
}
return nil
}
func (o *fakeObject) Release() {
if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
// Delete removes and ban 'cache node' with the given namespace and key.
// A banned 'cache node' will never inserted into the 'cache tree'. Ban
// only attributed to the particular 'cache node', so when a 'cache node'
// is recreated it will not be banned.
//
// If onDel is not nil, then it will be executed if such 'cache node'
// doesn't exist or once the 'cache node' is released.
//
// Delete return true is such 'cache node' exist.
func (r *Cache) Delete(ns, key uint64, onDel func()) bool {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return false
}
hash := murmur32(ns, key, 0xf00)
for {
h, b := r.getBucket(hash)
done, _, n := b.get(r, h, hash, ns, key, true)
if done {
if n != nil {
if onDel != nil {
n.mu.Lock()
n.onDel = append(n.onDel, onDel)
n.mu.Unlock()
}
if r.cacher != nil {
r.cacher.Ban(n)
}
n.unref()
return true
}
break
}
}
if onDel != nil {
onDel()
}
return false
}
// Evict evicts 'cache node' with the given namespace and key. This will
// simply call Cacher.Evict.
//
// Evict return true is such 'cache node' exist.
func (r *Cache) Evict(ns, key uint64) bool {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return false
}
hash := murmur32(ns, key, 0xf00)
for {
h, b := r.getBucket(hash)
done, _, n := b.get(r, h, hash, ns, key, true)
if done {
if n != nil {
if r.cacher != nil {
r.cacher.Evict(n)
}
n.unref()
return true
}
break
}
}
return false
}
// EvictNS evicts 'cache node' with the given namespace. This will
// simply call Cacher.EvictNS.
func (r *Cache) EvictNS(ns uint64) {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return
}
if o.fin != nil {
o.fin()
o.fin = nil
if r.cacher != nil {
r.cacher.EvictNS(ns)
}
}
// EvictAll evicts all 'cache node'. This will simply call Cacher.EvictAll.
func (r *Cache) EvictAll() {
r.mu.RLock()
defer r.mu.RUnlock()
if r.closed {
return
}
if r.cacher != nil {
r.cacher.EvictAll()
}
}
// Close closes the 'cache map' and releases all 'cache node'.
func (r *Cache) Close() error {
r.mu.Lock()
if !r.closed {
r.closed = true
if r.cacher != nil {
if err := r.cacher.Close(); err != nil {
return err
}
}
h := (*mNode)(r.mHead)
h.initBuckets()
for i := range h.buckets {
b := (*mBucket)(h.buckets[i])
for _, n := range b.node {
// Call releaser.
if n.value != nil {
if r, ok := n.value.(util.Releaser); ok {
r.Release()
}
n.value = nil
}
// Call OnDel.
for _, f := range n.onDel {
f()
}
}
}
}
r.mu.Unlock()
return nil
}
// Node is a 'cache node'.
type Node struct {
r *Cache
hash uint32
ns, key uint64
mu sync.Mutex
size int
value Value
ref int32
onDel []func()
CacheData unsafe.Pointer
}
// NS returns this 'cache node' namespace.
func (n *Node) NS() uint64 {
return n.ns
}
// Key returns this 'cache node' key.
func (n *Node) Key() uint64 {
return n.key
}
// Size returns this 'cache node' size.
func (n *Node) Size() int {
return n.size
}
// Value returns this 'cache node' value.
func (n *Node) Value() Value {
return n.value
}
// Ref returns this 'cache node' ref counter.
func (n *Node) Ref() int32 {
return atomic.LoadInt32(&n.ref)
}
// GetHandle returns an handle for this 'cache node'.
func (n *Node) GetHandle() *Handle {
if atomic.AddInt32(&n.ref, 1) <= 1 {
panic("BUG: Node.GetHandle on zero ref")
}
return &Handle{unsafe.Pointer(n)}
}
func (n *Node) unref() {
if atomic.AddInt32(&n.ref, -1) == 0 {
n.r.delete(n)
}
}
func (n *Node) unrefLocked() {
if atomic.AddInt32(&n.ref, -1) == 0 {
n.r.mu.RLock()
if !n.r.closed {
n.r.delete(n)
}
n.r.mu.RUnlock()
}
}
type Handle struct {
n unsafe.Pointer // *Node
}
func (h *Handle) Value() Value {
n := (*Node)(atomic.LoadPointer(&h.n))
if n != nil {
return n.value
}
return nil
}
func (h *Handle) Release() {
nPtr := atomic.LoadPointer(&h.n)
if nPtr != nil && atomic.CompareAndSwapPointer(&h.n, nPtr, nil) {
n := (*Node)(nPtr)
n.unrefLocked()
}
}
func murmur32(ns, key uint64, seed uint32) uint32 {
const (
m = uint32(0x5bd1e995)
r = 24
)
k1 := uint32(ns >> 32)
k2 := uint32(ns)
k3 := uint32(key >> 32)
k4 := uint32(key)
k1 *= m
k1 ^= k1 >> r
k1 *= m
k2 *= m
k2 ^= k2 >> r
k2 *= m
k3 *= m
k3 ^= k3 >> r
k3 *= m
k4 *= m
k4 ^= k4 >> r
k4 *= m
h := seed
h *= m
h ^= k1
h *= m
h ^= k2
h *= m
h ^= k3
h *= m
h ^= k4
h ^= h >> 13
h *= m
h ^= h >> 15
return h
}

576
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go generated vendored
View File

@ -8,17 +8,289 @@ package cache
import (
"math/rand"
"runtime"
"sync"
"sync/atomic"
"testing"
"time"
"unsafe"
)
func set(ns Namespace, key uint64, value interface{}, charge int, fin func()) Object {
obj, _ := ns.Get(key, func() (bool, interface{}, int, SetFin) {
return true, value, charge, fin
})
return obj
type int32o int32
func (o *int32o) acquire() {
if atomic.AddInt32((*int32)(o), 1) != 1 {
panic("BUG: invalid ref")
}
}
func TestCache_HitMiss(t *testing.T) {
func (o *int32o) Release() {
if atomic.AddInt32((*int32)(o), -1) != 0 {
panic("BUG: invalid ref")
}
}
type releaserFunc struct {
fn func()
value Value
}
func (r releaserFunc) Release() {
if r.fn != nil {
r.fn()
}
}
func set(c *Cache, ns, key uint64, value Value, charge int, relf func()) *Handle {
return c.Get(ns, key, func() (int, Value) {
if relf != nil {
return charge, releaserFunc{relf, value}
} else {
return charge, value
}
})
}
func TestCacheMap(t *testing.T) {
runtime.GOMAXPROCS(runtime.NumCPU())
nsx := []struct {
nobjects, nhandles, concurrent, repeat int
}{
{10000, 400, 50, 3},
{100000, 1000, 100, 10},
}
var (
objects [][]int32o
handles [][]unsafe.Pointer
)
for _, x := range nsx {
objects = append(objects, make([]int32o, x.nobjects))
handles = append(handles, make([]unsafe.Pointer, x.nhandles))
}
c := NewCache(nil)
wg := new(sync.WaitGroup)
var done int32
for ns, x := range nsx {
for i := 0; i < x.concurrent; i++ {
wg.Add(1)
go func(ns, i, repeat int, objects []int32o, handles []unsafe.Pointer) {
defer wg.Done()
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for j := len(objects) * repeat; j >= 0; j-- {
key := uint64(r.Intn(len(objects)))
h := c.Get(uint64(ns), key, func() (int, Value) {
o := &objects[key]
o.acquire()
return 1, o
})
if v := h.Value().(*int32o); v != &objects[key] {
t.Fatalf("#%d invalid value: want=%p got=%p", ns, &objects[key], v)
}
if objects[key] != 1 {
t.Fatalf("#%d invalid object %d: %d", ns, key, objects[key])
}
if !atomic.CompareAndSwapPointer(&handles[r.Intn(len(handles))], nil, unsafe.Pointer(h)) {
h.Release()
}
}
}(ns, i, x.repeat, objects[ns], handles[ns])
}
go func(handles []unsafe.Pointer) {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for atomic.LoadInt32(&done) == 0 {
i := r.Intn(len(handles))
h := (*Handle)(atomic.LoadPointer(&handles[i]))
if h != nil && atomic.CompareAndSwapPointer(&handles[i], unsafe.Pointer(h), nil) {
h.Release()
}
time.Sleep(time.Millisecond)
}
}(handles[ns])
}
go func() {
handles := make([]*Handle, 100000)
for atomic.LoadInt32(&done) == 0 {
for i := range handles {
handles[i] = c.Get(999999999, uint64(i), func() (int, Value) {
return 1, 1
})
}
for _, h := range handles {
h.Release()
}
}
}()
wg.Wait()
atomic.StoreInt32(&done, 1)
for _, handles0 := range handles {
for i := range handles0 {
h := (*Handle)(atomic.LoadPointer(&handles0[i]))
if h != nil && atomic.CompareAndSwapPointer(&handles0[i], unsafe.Pointer(h), nil) {
h.Release()
}
}
}
for ns, objects0 := range objects {
for i, o := range objects0 {
if o != 0 {
t.Fatalf("invalid object #%d.%d: ref=%d", ns, i, o)
}
}
}
}
func TestCacheMap_NodesAndSize(t *testing.T) {
c := NewCache(nil)
if c.Nodes() != 0 {
t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes())
}
if c.Size() != 0 {
t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size())
}
set(c, 0, 1, 1, 1, nil)
set(c, 0, 2, 2, 2, nil)
set(c, 1, 1, 3, 3, nil)
set(c, 2, 1, 4, 1, nil)
if c.Nodes() != 4 {
t.Errorf("invalid nodes counter: want=%d got=%d", 4, c.Nodes())
}
if c.Size() != 7 {
t.Errorf("invalid size counter: want=%d got=%d", 4, c.Size())
}
}
func TestLRUCache_Capacity(t *testing.T) {
c := NewCache(NewLRU(10))
if c.Capacity() != 10 {
t.Errorf("invalid capacity: want=%d got=%d", 10, c.Capacity())
}
set(c, 0, 1, 1, 1, nil).Release()
set(c, 0, 2, 2, 2, nil).Release()
set(c, 1, 1, 3, 3, nil).Release()
set(c, 2, 1, 4, 1, nil).Release()
set(c, 2, 2, 5, 1, nil).Release()
set(c, 2, 3, 6, 1, nil).Release()
set(c, 2, 4, 7, 1, nil).Release()
set(c, 2, 5, 8, 1, nil).Release()
if c.Nodes() != 7 {
t.Errorf("invalid nodes counter: want=%d got=%d", 7, c.Nodes())
}
if c.Size() != 10 {
t.Errorf("invalid size counter: want=%d got=%d", 10, c.Size())
}
c.SetCapacity(9)
if c.Capacity() != 9 {
t.Errorf("invalid capacity: want=%d got=%d", 9, c.Capacity())
}
if c.Nodes() != 6 {
t.Errorf("invalid nodes counter: want=%d got=%d", 6, c.Nodes())
}
if c.Size() != 8 {
t.Errorf("invalid size counter: want=%d got=%d", 8, c.Size())
}
}
func TestCacheMap_NilValue(t *testing.T) {
c := NewCache(NewLRU(10))
h := c.Get(0, 0, func() (size int, value Value) {
return 1, nil
})
if h != nil {
t.Error("cache handle is non-nil")
}
if c.Nodes() != 0 {
t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes())
}
if c.Size() != 0 {
t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size())
}
}
func TestLRUCache_GetLatency(t *testing.T) {
runtime.GOMAXPROCS(runtime.NumCPU())
const (
concurrentSet = 30
concurrentGet = 3
duration = 3 * time.Second
delay = 3 * time.Millisecond
maxkey = 100000
)
var (
set, getHit, getAll int32
getMaxLatency, getDuration int64
)
c := NewCache(NewLRU(5000))
wg := &sync.WaitGroup{}
until := time.Now().Add(duration)
for i := 0; i < concurrentSet; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for time.Now().Before(until) {
c.Get(0, uint64(r.Intn(maxkey)), func() (int, Value) {
time.Sleep(delay)
atomic.AddInt32(&set, 1)
return 1, 1
}).Release()
}
}(i)
}
for i := 0; i < concurrentGet; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for {
mark := time.Now()
if mark.Before(until) {
h := c.Get(0, uint64(r.Intn(maxkey)), nil)
latency := int64(time.Now().Sub(mark))
m := atomic.LoadInt64(&getMaxLatency)
if latency > m {
atomic.CompareAndSwapInt64(&getMaxLatency, m, latency)
}
atomic.AddInt64(&getDuration, latency)
if h != nil {
atomic.AddInt32(&getHit, 1)
h.Release()
}
atomic.AddInt32(&getAll, 1)
} else {
break
}
}
}(i)
}
wg.Wait()
getAvglatency := time.Duration(getDuration) / time.Duration(getAll)
t.Logf("set=%d getHit=%d getAll=%d getMaxLatency=%v getAvgLatency=%v",
set, getHit, getAll, time.Duration(getMaxLatency), getAvglatency)
if getAvglatency > delay/3 {
t.Errorf("get avg latency > %v: got=%v", delay/3, getAvglatency)
}
}
func TestLRUCache_HitMiss(t *testing.T) {
cases := []struct {
key uint64
value string
@ -36,36 +308,37 @@ func TestCache_HitMiss(t *testing.T) {
}
setfin := 0
c := NewLRUCache(1000)
ns := c.GetNamespace(0)
c := NewCache(NewLRU(1000))
for i, x := range cases {
set(ns, x.key, x.value, len(x.value), func() {
set(c, 0, x.key, x.value, len(x.value), func() {
setfin++
}).Release()
for j, y := range cases {
r, ok := ns.Get(y.key, nil)
h := c.Get(0, y.key, nil)
if j <= i {
// should hit
if !ok {
if h == nil {
t.Errorf("case '%d' iteration '%d' is miss", i, j)
} else if r.Value().(string) != y.value {
t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, r.Value().(string), y.value)
} else {
if x := h.Value().(releaserFunc).value.(string); x != y.value {
t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value)
}
}
} else {
// should miss
if ok {
t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, r.Value().(string))
if h != nil {
t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, h.Value().(releaserFunc).value.(string))
}
}
if ok {
r.Release()
if h != nil {
h.Release()
}
}
}
for i, x := range cases {
finalizerOk := false
ns.Delete(x.key, func(exist bool) {
c.Delete(0, x.key, func() {
finalizerOk = true
})
@ -74,22 +347,24 @@ func TestCache_HitMiss(t *testing.T) {
}
for j, y := range cases {
r, ok := ns.Get(y.key, nil)
h := c.Get(0, y.key, nil)
if j > i {
// should hit
if !ok {
if h == nil {
t.Errorf("case '%d' iteration '%d' is miss", i, j)
} else if r.Value().(string) != y.value {
t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, r.Value().(string), y.value)
} else {
if x := h.Value().(releaserFunc).value.(string); x != y.value {
t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value)
}
}
} else {
// should miss
if ok {
t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, r.Value().(string))
if h != nil {
t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, h.Value().(releaserFunc).value.(string))
}
}
if ok {
r.Release()
if h != nil {
h.Release()
}
}
}
@ -100,137 +375,180 @@ func TestCache_HitMiss(t *testing.T) {
}
func TestLRUCache_Eviction(t *testing.T) {
c := NewLRUCache(12)
ns := c.GetNamespace(0)
o1 := set(ns, 1, 1, 1, nil)
set(ns, 2, 2, 1, nil).Release()
set(ns, 3, 3, 1, nil).Release()
set(ns, 4, 4, 1, nil).Release()
set(ns, 5, 5, 1, nil).Release()
if r, ok := ns.Get(2, nil); ok { // 1,3,4,5,2
r.Release()
c := NewCache(NewLRU(12))
o1 := set(c, 0, 1, 1, 1, nil)
set(c, 0, 2, 2, 1, nil).Release()
set(c, 0, 3, 3, 1, nil).Release()
set(c, 0, 4, 4, 1, nil).Release()
set(c, 0, 5, 5, 1, nil).Release()
if h := c.Get(0, 2, nil); h != nil { // 1,3,4,5,2
h.Release()
}
set(ns, 9, 9, 10, nil).Release() // 5,2,9
set(c, 0, 9, 9, 10, nil).Release() // 5,2,9
for _, x := range []uint64{9, 2, 5, 1} {
r, ok := ns.Get(x, nil)
if !ok {
t.Errorf("miss for key '%d'", x)
for _, key := range []uint64{9, 2, 5, 1} {
h := c.Get(0, key, nil)
if h == nil {
t.Errorf("miss for key '%d'", key)
} else {
if r.Value().(int) != int(x) {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
if x := h.Value().(int); x != int(key) {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
}
r.Release()
h.Release()
}
}
o1.Release()
for _, x := range []uint64{1, 2, 5} {
r, ok := ns.Get(x, nil)
if !ok {
t.Errorf("miss for key '%d'", x)
for _, key := range []uint64{1, 2, 5} {
h := c.Get(0, key, nil)
if h == nil {
t.Errorf("miss for key '%d'", key)
} else {
if r.Value().(int) != int(x) {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
if x := h.Value().(int); x != int(key) {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
}
r.Release()
h.Release()
}
}
for _, x := range []uint64{3, 4, 9} {
r, ok := ns.Get(x, nil)
if ok {
t.Errorf("hit for key '%d'", x)
if r.Value().(int) != int(x) {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
for _, key := range []uint64{3, 4, 9} {
h := c.Get(0, key, nil)
if h != nil {
t.Errorf("hit for key '%d'", key)
if x := h.Value().(int); x != int(key) {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
}
r.Release()
h.Release()
}
}
}
func TestLRUCache_SetGet(t *testing.T) {
c := NewLRUCache(13)
ns := c.GetNamespace(0)
for i := 0; i < 200; i++ {
n := uint64(rand.Intn(99999) % 20)
set(ns, n, n, 1, nil).Release()
if p, ok := ns.Get(n, nil); ok {
if p.Value() == nil {
t.Errorf("key '%d' contains nil value", n)
func TestLRUCache_Evict(t *testing.T) {
c := NewCache(NewLRU(6))
set(c, 0, 1, 1, 1, nil).Release()
set(c, 0, 2, 2, 1, nil).Release()
set(c, 1, 1, 4, 1, nil).Release()
set(c, 1, 2, 5, 1, nil).Release()
set(c, 2, 1, 6, 1, nil).Release()
set(c, 2, 2, 7, 1, nil).Release()
for ns := 0; ns < 3; ns++ {
for key := 1; key < 3; key++ {
if h := c.Get(uint64(ns), uint64(key), nil); h != nil {
h.Release()
} else {
got := p.Value().(uint64)
if got != n {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", n, n, got)
t.Errorf("Cache.Get on #%d.%d return nil", ns, key)
}
}
p.Release()
}
if ok := c.Evict(0, 1); !ok {
t.Error("first Cache.Evict on #0.1 return false")
}
if ok := c.Evict(0, 1); ok {
t.Error("second Cache.Evict on #0.1 return true")
}
if h := c.Get(0, 1, nil); h != nil {
t.Errorf("Cache.Get on #0.1 return non-nil: %v", h.Value())
}
c.EvictNS(1)
if h := c.Get(1, 1, nil); h != nil {
t.Errorf("Cache.Get on #1.1 return non-nil: %v", h.Value())
}
if h := c.Get(1, 2, nil); h != nil {
t.Errorf("Cache.Get on #1.2 return non-nil: %v", h.Value())
}
c.EvictAll()
for ns := 0; ns < 3; ns++ {
for key := 1; key < 3; key++ {
if h := c.Get(uint64(ns), uint64(key), nil); h != nil {
t.Errorf("Cache.Get on #%d.%d return non-nil: %v", ns, key, h.Value())
}
}
}
}
func TestLRUCache_Delete(t *testing.T) {
delFuncCalled := 0
delFunc := func() {
delFuncCalled++
}
c := NewCache(NewLRU(2))
set(c, 0, 1, 1, 1, nil).Release()
set(c, 0, 2, 2, 1, nil).Release()
if ok := c.Delete(0, 1, delFunc); !ok {
t.Error("Cache.Delete on #1 return false")
}
if h := c.Get(0, 1, nil); h != nil {
t.Errorf("Cache.Get on #1 return non-nil: %v", h.Value())
}
if ok := c.Delete(0, 1, delFunc); ok {
t.Error("Cache.Delete on #1 return true")
}
h2 := c.Get(0, 2, nil)
if h2 == nil {
t.Error("Cache.Get on #2 return nil")
}
if ok := c.Delete(0, 2, delFunc); !ok {
t.Error("(1) Cache.Delete on #2 return false")
}
if ok := c.Delete(0, 2, delFunc); !ok {
t.Error("(2) Cache.Delete on #2 return false")
}
set(c, 0, 3, 3, 1, nil).Release()
set(c, 0, 4, 4, 1, nil).Release()
c.Get(0, 2, nil).Release()
for key := 2; key <= 4; key++ {
if h := c.Get(0, uint64(key), nil); h != nil {
h.Release()
} else {
t.Errorf("key '%d' doesn't exist", n)
t.Errorf("Cache.Get on #%d return nil", key)
}
}
h2.Release()
if h := c.Get(0, 2, nil); h != nil {
t.Errorf("Cache.Get on #2 return non-nil: %v", h.Value())
}
if delFuncCalled != 4 {
t.Errorf("delFunc isn't called 4 times: got=%d", delFuncCalled)
}
}
func TestLRUCache_Purge(t *testing.T) {
c := NewLRUCache(3)
ns1 := c.GetNamespace(0)
o1 := set(ns1, 1, 1, 1, nil)
o2 := set(ns1, 2, 2, 1, nil)
ns1.Purge(nil)
set(ns1, 3, 3, 1, nil).Release()
for _, x := range []uint64{1, 2, 3} {
r, ok := ns1.Get(x, nil)
if !ok {
t.Errorf("miss for key '%d'", x)
} else {
if r.Value().(int) != int(x) {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
func TestLRUCache_Close(t *testing.T) {
relFuncCalled := 0
relFunc := func() {
relFuncCalled++
}
r.Release()
delFuncCalled := 0
delFunc := func() {
delFuncCalled++
}
c := NewCache(NewLRU(2))
set(c, 0, 1, 1, 1, relFunc).Release()
set(c, 0, 2, 2, 1, relFunc).Release()
h3 := set(c, 0, 3, 3, 1, relFunc)
if h3 == nil {
t.Error("Cache.Get on #3 return nil")
}
o1.Release()
o2.Release()
for _, x := range []uint64{1, 2} {
r, ok := ns1.Get(x, nil)
if ok {
t.Errorf("hit for key '%d'", x)
if r.Value().(int) != int(x) {
t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
if ok := c.Delete(0, 3, delFunc); !ok {
t.Error("Cache.Delete on #3 return false")
}
r.Release()
c.Close()
if relFuncCalled != 3 {
t.Errorf("relFunc isn't called 3 times: got=%d", relFuncCalled)
}
}
}
func BenchmarkLRUCache_SetRelease(b *testing.B) {
capacity := b.N / 100
if capacity <= 0 {
capacity = 10
}
c := NewLRUCache(capacity)
ns := c.GetNamespace(0)
b.ResetTimer()
for i := uint64(0); i < uint64(b.N); i++ {
set(ns, i, nil, 1, nil).Release()
}
}
func BenchmarkLRUCache_SetReleaseTwice(b *testing.B) {
capacity := b.N / 100
if capacity <= 0 {
capacity = 10
}
c := NewLRUCache(capacity)
ns := c.GetNamespace(0)
b.ResetTimer()
na := b.N / 2
nb := b.N - na
for i := uint64(0); i < uint64(na); i++ {
set(ns, i, nil, 1, nil).Release()
}
for i := uint64(0); i < uint64(nb); i++ {
set(ns, i, nil, 1, nil).Release()
if delFuncCalled != 1 {
t.Errorf("delFunc isn't called 1 times: got=%d", delFuncCalled)
}
}

246
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/empty_cache.go generated vendored
View File

@ -1,246 +0,0 @@
// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package cache
import (
"sync"
"sync/atomic"
)
type emptyCache struct {
sync.Mutex
table map[uint64]*emptyNS
}
// NewEmptyCache creates a new initialized empty cache.
func NewEmptyCache() Cache {
return &emptyCache{
table: make(map[uint64]*emptyNS),
}
}
func (c *emptyCache) GetNamespace(id uint64) Namespace {
c.Lock()
defer c.Unlock()
if ns, ok := c.table[id]; ok {
return ns
}
ns := &emptyNS{
cache: c,
id: id,
table: make(map[uint64]*emptyNode),
}
c.table[id] = ns
return ns
}
func (c *emptyCache) Purge(fin PurgeFin) {
c.Lock()
for _, ns := range c.table {
ns.purgeNB(fin)
}
c.Unlock()
}
func (c *emptyCache) Zap(closed bool) {
c.Lock()
for _, ns := range c.table {
ns.zapNB(closed)
}
c.table = make(map[uint64]*emptyNS)
c.Unlock()
}
func (*emptyCache) SetCapacity(capacity int) {}
type emptyNS struct {
cache *emptyCache
id uint64
table map[uint64]*emptyNode
state nsState
}
func (ns *emptyNS) Get(key uint64, setf SetFunc) (o Object, ok bool) {
ns.cache.Lock()
switch ns.state {
case nsZapped:
ns.cache.Unlock()
if setf == nil {
return
}
var value interface{}
var fin func()
ok, value, _, fin = setf()
if ok {
o = &fakeObject{
value: value,
fin: fin,
}
}
return
case nsClosed:
ns.cache.Unlock()
return
}
n, ok := ns.table[key]
if ok {
n.ref++
} else {
if setf == nil {
ns.cache.Unlock()
return
}
var value interface{}
var fin func()
ok, value, _, fin = setf()
if !ok {
ns.cache.Unlock()
return
}
n = &emptyNode{
ns: ns,
key: key,
value: value,
setfin: fin,
ref: 1,
}
ns.table[key] = n
}
ns.cache.Unlock()
o = &emptyObject{node: n}
return
}
func (ns *emptyNS) Delete(key uint64, fin DelFin) bool {
ns.cache.Lock()
if ns.state != nsEffective {
ns.cache.Unlock()
if fin != nil {
fin(false)
}
return false
}
n, ok := ns.table[key]
if !ok {
ns.cache.Unlock()
if fin != nil {
fin(false)
}
return false
}
n.delfin = fin
ns.cache.Unlock()
return true
}
func (ns *emptyNS) purgeNB(fin PurgeFin) {
if ns.state != nsEffective {
return
}
for _, n := range ns.table {
n.purgefin = fin
}
}
func (ns *emptyNS) Purge(fin PurgeFin) {
ns.cache.Lock()
ns.purgeNB(fin)
ns.cache.Unlock()
}
func (ns *emptyNS) zapNB(closed bool) {
if ns.state != nsEffective {
return
}
for _, n := range ns.table {
n.execFin()
}
if closed {
ns.state = nsClosed
} else {
ns.state = nsZapped
}
ns.table = nil
}
func (ns *emptyNS) Zap(closed bool) {
ns.cache.Lock()
ns.zapNB(closed)
delete(ns.cache.table, ns.id)
ns.cache.Unlock()
}
type emptyNode struct {
ns *emptyNS
key uint64
value interface{}
ref int
setfin SetFin
delfin DelFin
purgefin PurgeFin
}
func (n *emptyNode) execFin() {
if n.setfin != nil {
n.setfin()
n.setfin = nil
}
if n.purgefin != nil {
n.purgefin(n.ns.id, n.key, n.delfin)
n.delfin = nil
n.purgefin = nil
} else if n.delfin != nil {
n.delfin(true)
n.delfin = nil
}
}
func (n *emptyNode) evict() {
n.ns.cache.Lock()
n.ref--
if n.ref == 0 {
if n.ns.state == nsEffective {
// Remove elem.
delete(n.ns.table, n.key)
// Execute finalizer.
n.execFin()
}
} else if n.ref < 0 {
panic("leveldb/cache: emptyNode: negative node reference")
}
n.ns.cache.Unlock()
}
type emptyObject struct {
node *emptyNode
once uint32
}
func (o *emptyObject) Value() interface{} {
if atomic.LoadUint32(&o.once) == 0 {
return o.node.value
}
return nil
}
func (o *emptyObject) Release() {
if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
return
}
o.node.evict()
o.node = nil
}

195
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru.go generated vendored Normal file
View File

@ -0,0 +1,195 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package cache
import (
"sync"
"unsafe"
)
type lruNode struct {
n *Node
h *Handle
ban bool
next, prev *lruNode
}
func (n *lruNode) insert(at *lruNode) {
x := at.next
at.next = n
n.prev = at
n.next = x
x.prev = n
}
func (n *lruNode) remove() {
if n.prev != nil {
n.prev.next = n.next
n.next.prev = n.prev
n.prev = nil
n.next = nil
} else {
panic("BUG: removing removed node")
}
}
type lru struct {
mu sync.Mutex
capacity int
used int
recent lruNode
}
func (r *lru) reset() {
r.recent.next = &r.recent
r.recent.prev = &r.recent
r.used = 0
}
func (r *lru) Capacity() int {
r.mu.Lock()
defer r.mu.Unlock()
return r.capacity
}
func (r *lru) SetCapacity(capacity int) {
var evicted []*lruNode
r.mu.Lock()
r.capacity = capacity
for r.used > r.capacity {
rn := r.recent.prev
if rn == nil {
panic("BUG: invalid LRU used or capacity counter")
}
rn.remove()
rn.n.CacheData = nil
r.used -= rn.n.Size()
evicted = append(evicted, rn)
}
r.mu.Unlock()
for _, rn := range evicted {
rn.h.Release()
}
}
func (r *lru) Promote(n *Node) {
var evicted []*lruNode
r.mu.Lock()
if n.CacheData == nil {
if n.Size() <= r.capacity {
rn := &lruNode{n: n, h: n.GetHandle()}
rn.insert(&r.recent)
n.CacheData = unsafe.Pointer(rn)
r.used += n.Size()
for r.used > r.capacity {
rn := r.recent.prev
if rn == nil {
panic("BUG: invalid LRU used or capacity counter")
}
rn.remove()
rn.n.CacheData = nil
r.used -= rn.n.Size()
evicted = append(evicted, rn)
}
}
} else {
rn := (*lruNode)(n.CacheData)
if !rn.ban {
rn.remove()
rn.insert(&r.recent)
}
}
r.mu.Unlock()
for _, rn := range evicted {
rn.h.Release()
}
}
func (r *lru) Ban(n *Node) {
r.mu.Lock()
if n.CacheData == nil {
n.CacheData = unsafe.Pointer(&lruNode{n: n, ban: true})
} else {
rn := (*lruNode)(n.CacheData)
if !rn.ban {
rn.remove()
rn.ban = true
r.used -= rn.n.Size()
r.mu.Unlock()
rn.h.Release()
rn.h = nil
return
}
}
r.mu.Unlock()
}
func (r *lru) Evict(n *Node) {
r.mu.Lock()
rn := (*lruNode)(n.CacheData)
if rn == nil || rn.ban {
r.mu.Unlock()
return
}
n.CacheData = nil
r.mu.Unlock()
rn.h.Release()
}
func (r *lru) EvictNS(ns uint64) {
var evicted []*lruNode
r.mu.Lock()
for e := r.recent.prev; e != &r.recent; {
rn := e
e = e.prev
if rn.n.NS() == ns {
rn.remove()
rn.n.CacheData = nil
r.used -= rn.n.Size()
evicted = append(evicted, rn)
}
}
r.mu.Unlock()
for _, rn := range evicted {
rn.h.Release()
}
}
func (r *lru) EvictAll() {
r.mu.Lock()
back := r.recent.prev
for rn := back; rn != &r.recent; rn = rn.prev {
rn.n.CacheData = nil
}
r.reset()
r.mu.Unlock()
for rn := back; rn != &r.recent; rn = rn.prev {
rn.h.Release()
}
}
func (r *lru) Close() error {
return nil
}
// NewLRU create a new LRU-cache.
func NewLRU(capacity int) Cacher {
r := &lru{capacity: capacity}
r.reset()
return r
}

354
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go generated vendored
View File

@ -1,354 +0,0 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package cache
import (
"sync"
"sync/atomic"
)
// lruCache represent a LRU cache state.
type lruCache struct {
sync.Mutex
recent lruNode
table map[uint64]*lruNs
capacity int
size int
}
// NewLRUCache creates a new initialized LRU cache with the given capacity.
func NewLRUCache(capacity int) Cache {
c := &lruCache{
table: make(map[uint64]*lruNs),
capacity: capacity,
}
c.recent.rNext = &c.recent
c.recent.rPrev = &c.recent
return c
}
// SetCapacity set cache capacity.
func (c *lruCache) SetCapacity(capacity int) {
c.Lock()
c.capacity = capacity
c.evict()
c.Unlock()
}
// GetNamespace return namespace object for given id.
func (c *lruCache) GetNamespace(id uint64) Namespace {
c.Lock()
defer c.Unlock()
if p, ok := c.table[id]; ok {
return p
}
p := &lruNs{
lru: c,
id: id,
table: make(map[uint64]*lruNode),
}
c.table[id] = p
return p
}
// Purge purge entire cache.
func (c *lruCache) Purge(fin PurgeFin) {
c.Lock()
for _, ns := range c.table {
ns.purgeNB(fin)
}
c.Unlock()
}
func (c *lruCache) Zap(closed bool) {
c.Lock()
for _, ns := range c.table {
ns.zapNB(closed)
}
c.table = make(map[uint64]*lruNs)
c.Unlock()
}
func (c *lruCache) evict() {
top := &c.recent
for n := c.recent.rPrev; c.size > c.capacity && n != top; {
n.state = nodeEvicted
n.rRemove()
n.evictNB()
c.size -= n.charge
n = c.recent.rPrev
}
}
type lruNs struct {
lru *lruCache
id uint64
table map[uint64]*lruNode
state nsState
}
func (ns *lruNs) Get(key uint64, setf SetFunc) (o Object, ok bool) {
lru := ns.lru
lru.Lock()
switch ns.state {
case nsZapped:
lru.Unlock()
if setf == nil {
return
}
var value interface{}
var fin func()
ok, value, _, fin = setf()
if ok {
o = &fakeObject{
value: value,
fin: fin,
}
}
return
case nsClosed:
lru.Unlock()
return
}
n, ok := ns.table[key]
if ok {
switch n.state {
case nodeEvicted:
// Insert to recent list.
n.state = nodeEffective
n.ref++
lru.size += n.charge
lru.evict()
fallthrough
case nodeEffective:
// Bump to front
n.rRemove()
n.rInsert(&lru.recent)
}
n.ref++
} else {
if setf == nil {
lru.Unlock()
return
}
var value interface{}
var charge int
var fin func()
ok, value, charge, fin = setf()
if !ok {
lru.Unlock()
return
}
n = &lruNode{
ns: ns,
key: key,
value: value,
charge: charge,
setfin: fin,
ref: 2,
}
ns.table[key] = n
n.rInsert(&lru.recent)
lru.size += charge
lru.evict()
}
lru.Unlock()
o = &lruObject{node: n}
return
}
func (ns *lruNs) Delete(key uint64, fin DelFin) bool {
lru := ns.lru
lru.Lock()
if ns.state != nsEffective {
lru.Unlock()
if fin != nil {
fin(false)
}
return false
}
n, ok := ns.table[key]
if !ok {
lru.Unlock()
if fin != nil {
fin(false)
}
return false
}
n.delfin = fin
switch n.state {
case nodeRemoved:
lru.Unlock()
return false
case nodeEffective:
lru.size -= n.charge
n.rRemove()
n.evictNB()
}
n.state = nodeRemoved
lru.Unlock()
return true
}
func (ns *lruNs) purgeNB(fin PurgeFin) {
lru := ns.lru
if ns.state != nsEffective {
return
}
for _, n := range ns.table {
n.purgefin = fin
if n.state == nodeEffective {
lru.size -= n.charge
n.rRemove()
n.evictNB()
}
n.state = nodeRemoved
}
}
func (ns *lruNs) Purge(fin PurgeFin) {
ns.lru.Lock()
ns.purgeNB(fin)
ns.lru.Unlock()
}
func (ns *lruNs) zapNB(closed bool) {
lru := ns.lru
if ns.state != nsEffective {
return
}
if closed {
ns.state = nsClosed
} else {
ns.state = nsZapped
}
for _, n := range ns.table {
if n.state == nodeEffective {
lru.size -= n.charge
n.rRemove()
}
n.state = nodeRemoved
n.execFin()
}
ns.table = nil
}
func (ns *lruNs) Zap(closed bool) {
ns.lru.Lock()
ns.zapNB(closed)
delete(ns.lru.table, ns.id)
ns.lru.Unlock()
}
type lruNode struct {
ns *lruNs
rNext, rPrev *lruNode
key uint64
value interface{}
charge int
ref int
state nodeState
setfin SetFin
delfin DelFin
purgefin PurgeFin
}
func (n *lruNode) rInsert(at *lruNode) {
x := at.rNext
at.rNext = n
n.rPrev = at
n.rNext = x
x.rPrev = n
}
func (n *lruNode) rRemove() bool {
// only remove if not already removed
if n.rPrev == nil {
return false
}
n.rPrev.rNext = n.rNext
n.rNext.rPrev = n.rPrev
n.rPrev = nil
n.rNext = nil
return true
}
func (n *lruNode) execFin() {
if n.setfin != nil {
n.setfin()
n.setfin = nil
}
if n.purgefin != nil {
n.purgefin(n.ns.id, n.key, n.delfin)
n.delfin = nil
n.purgefin = nil
} else if n.delfin != nil {
n.delfin(true)
n.delfin = nil
}
}
func (n *lruNode) evictNB() {
n.ref--
if n.ref == 0 {
if n.ns.state == nsEffective {
// remove elem
delete(n.ns.table, n.key)
// execute finalizer
n.execFin()
}
} else if n.ref < 0 {
panic("leveldb/cache: lruCache: negative node reference")
}
}
func (n *lruNode) evict() {
n.ns.lru.Lock()
n.evictNB()
n.ns.lru.Unlock()
}
type lruObject struct {
node *lruNode
once uint32
}
func (o *lruObject) Value() interface{} {
if atomic.LoadUint32(&o.once) == 0 {
return o.node.value
}
return nil
}
func (o *lruObject) Release() {
if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
return
}
o.node.evict()
o.node = nil
}

40
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/config.go generated vendored
View File

@ -1,40 +0,0 @@
// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package leveldb
const (
kNumLevels = 7
// Level-0 compaction is started when we hit this many files.
kL0_CompactionTrigger float64 = 4
// Soft limit on number of level-0 files. We slow down writes at this point.
kL0_SlowdownWritesTrigger = 8
// Maximum number of level-0 files. We stop writes at this point.
kL0_StopWritesTrigger = 12
// Maximum level to which a new compacted memdb is pushed if it
// does not create overlap. We try to push to level 2 to avoid the
// relatively expensive level 0=>1 compactions and to avoid some
// expensive manifest file operations. We do not push all the way to
// the largest level since that can generate a lot of wasted disk
// space if the same key space is being repeatedly overwritten.
kMaxMemCompactLevel = 2
// Maximum size of a table.
kMaxTableSize = 2 * 1048576
// Maximum bytes of overlaps in grandparent (i.e., level+2) before we
// stop building a single file in a level->level+1 compaction.
kMaxGrandParentOverlapBytes = 10 * kMaxTableSize
// Maximum number of bytes in all compacted files. We avoid expanding
// the lower level file set of a compaction if it would make the
// total compaction cover more than this many bytes.
kExpCompactionMaxBytes = 25 * kMaxTableSize
)

70
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go generated vendored
View File

@ -9,13 +9,12 @@ package leveldb
import (
"bytes"
"fmt"
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/storage"
"io"
"math/rand"
"testing"
"github.com/syndtr/goleveldb/leveldb/cache"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/storage"
)
const ctValSize = 1000
@ -32,7 +31,7 @@ func newDbCorruptHarnessWopt(t *testing.T, o *opt.Options) *dbCorruptHarness {
func newDbCorruptHarness(t *testing.T) *dbCorruptHarness {
return newDbCorruptHarnessWopt(t, &opt.Options{
BlockCache: cache.NewLRUCache(100),
BlockCacheCapacity: 100,
Strict: opt.StrictJournalChecksum,
})
}
@ -96,20 +95,21 @@ func (h *dbCorruptHarness) deleteRand(n, max int, rnd *rand.Rand) {
}
}
func (h *dbCorruptHarness) corrupt(ft storage.FileType, offset, n int) {
func (h *dbCorruptHarness) corrupt(ft storage.FileType, fi, offset, n int) {
p := &h.dbHarness
t := p.t
var file storage.File
ff, _ := p.stor.GetFiles(ft)
for _, f := range ff {
if file == nil || f.Num() > file.Num() {
file = f
sff := files(ff)
sff.sort()
if fi < 0 {
fi = len(sff) - 1
}
if fi >= len(sff) {
t.Fatalf("no such file with type %q with index %d", ft, fi)
}
if file == nil {
t.Fatalf("no such file with type %q", ft)
}
file := sff[fi]
r, err := file.Open()
if err != nil {
@ -225,8 +225,8 @@ func TestCorruptDB_Journal(t *testing.T) {
h.build(100)
h.check(100, 100)
h.closeDB()
h.corrupt(storage.TypeJournal, 19, 1)
h.corrupt(storage.TypeJournal, 32*1024+1000, 1)
h.corrupt(storage.TypeJournal, -1, 19, 1)
h.corrupt(storage.TypeJournal, -1, 32*1024+1000, 1)
h.openDB()
h.check(36, 36)
@ -242,7 +242,7 @@ func TestCorruptDB_Table(t *testing.T) {
h.compactRangeAt(0, "", "")
h.compactRangeAt(1, "", "")
h.closeDB()
h.corrupt(storage.TypeTable, 100, 1)
h.corrupt(storage.TypeTable, -1, 100, 1)
h.openDB()
h.check(99, 99)
@ -256,7 +256,7 @@ func TestCorruptDB_TableIndex(t *testing.T) {
h.build(10000)
h.compactMem()
h.closeDB()
h.corrupt(storage.TypeTable, -2000, 500)
h.corrupt(storage.TypeTable, -1, -2000, 500)
h.openDB()
h.check(5000, 9999)
@ -267,7 +267,7 @@ func TestCorruptDB_TableIndex(t *testing.T) {
func TestCorruptDB_MissingManifest(t *testing.T) {
rnd := rand.New(rand.NewSource(0x0badda7a))
h := newDbCorruptHarnessWopt(t, &opt.Options{
BlockCache: cache.NewLRUCache(100),
BlockCacheCapacity: 100,
Strict: opt.StrictJournalChecksum,
WriteBuffer: 1000 * 60,
})
@ -355,7 +355,7 @@ func TestCorruptDB_CorruptedManifest(t *testing.T) {
h.compactMem()
h.compactRange("", "")
h.closeDB()
h.corrupt(storage.TypeManifest, 0, 1000)
h.corrupt(storage.TypeManifest, -1, 0, 1000)
h.openAssert(false)
h.recover()
@ -370,7 +370,7 @@ func TestCorruptDB_CompactionInputError(t *testing.T) {
h.build(10)
h.compactMem()
h.closeDB()
h.corrupt(storage.TypeTable, 100, 1)
h.corrupt(storage.TypeTable, -1, 100, 1)
h.openDB()
h.check(9, 9)
@ -387,7 +387,7 @@ func TestCorruptDB_UnrelatedKeys(t *testing.T) {
h.build(10)
h.compactMem()
h.closeDB()
h.corrupt(storage.TypeTable, 100, 1)
h.corrupt(storage.TypeTable, -1, 100, 1)
h.openDB()
h.put(string(tkey(1000)), string(tval(1000, ctValSize)))
@ -470,3 +470,31 @@ func TestCorruptDB_MissingTableFiles(t *testing.T) {
h.close()
}
func TestCorruptDB_RecoverTable(t *testing.T) {
h := newDbCorruptHarnessWopt(t, &opt.Options{
WriteBuffer: 112 * opt.KiB,
CompactionTableSize: 90 * opt.KiB,
Filter: filter.NewBloomFilter(10),
})
h.build(1000)
h.compactMem()
h.compactRangeAt(0, "", "")
h.compactRangeAt(1, "", "")
seq := h.db.seq
h.closeDB()
h.corrupt(storage.TypeTable, 0, 1000, 1)
h.corrupt(storage.TypeTable, 3, 10000, 1)
// Corrupted filter shouldn't affect recovery.
h.corrupt(storage.TypeTable, 3, 113888, 10)
h.corrupt(storage.TypeTable, -1, 20000, 1)
h.recover()
if h.db.seq != seq {
t.Errorf("invalid seq, want=%d got=%d", seq, h.db.seq)
}
h.check(985, 985)
h.close()
}

548
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go generated vendored
View File

@ -7,15 +7,17 @@
package leveldb
import (
"errors"
"container/list"
"fmt"
"io"
"os"
"runtime"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/journal"
"github.com/syndtr/goleveldb/leveldb/memdb"
@ -30,41 +32,46 @@ type DB struct {
// Need 64-bit alignment.
seq uint64
// Session.
s *session
// MemDB
// MemDB.
memMu sync.RWMutex
mem *memdb.DB
frozenMem *memdb.DB
memPool chan *memdb.DB
mem, frozenMem *memDB
journal *journal.Writer
journalWriter storage.Writer
journalFile storage.File
frozenJournalFile storage.File
frozenSeq uint64
// Snapshot
// Snapshot.
snapsMu sync.Mutex
snapsRoot snapshotElement
snapsList *list.List
// Write
// Stats.
aliveSnaps, aliveIters int32
// Write.
writeC chan *Batch
writeMergedC chan bool
writeLockC chan struct{}
writeAckC chan error
writeDelay time.Duration
writeDelayN int
journalC chan *Batch
journalAckC chan error
// Compaction
// Compaction.
tcompCmdC chan cCmd
tcompPauseC chan chan<- struct{}
tcompTriggerC chan struct{}
mcompCmdC chan cCmd
mcompTriggerC chan struct{}
compErrC chan error
compPerErrC chan error
compErrSetC chan error
compStats [kNumLevels]cStats
compStats []cStats
// Close
// Close.
closeW sync.WaitGroup
closeC chan struct{}
closed uint32
@ -77,7 +84,11 @@ func openDB(s *session) (*DB, error) {
db := &DB{
s: s,
// Initial sequence
seq: s.stSeq,
seq: s.stSeqNum,
// MemDB
memPool: make(chan *memdb.DB, 1),
// Snapshot
snapsList: list.New(),
// Write
writeC: make(chan *Batch),
writeMergedC: make(chan bool),
@ -88,15 +99,14 @@ func openDB(s *session) (*DB, error) {
// Compaction
tcompCmdC: make(chan cCmd),
tcompPauseC: make(chan chan<- struct{}),
tcompTriggerC: make(chan struct{}, 1),
mcompCmdC: make(chan cCmd),
mcompTriggerC: make(chan struct{}, 1),
compErrC: make(chan error),
compPerErrC: make(chan error),
compErrSetC: make(chan error),
compStats: make([]cStats, s.o.GetNumLevel()),
// Close
closeC: make(chan struct{}),
}
db.initSnapshot()
if err := db.recoverJournal(); err != nil {
return nil, err
@ -112,8 +122,9 @@ func openDB(s *session) (*DB, error) {
return nil, err
}
// Don't include compaction error goroutine into wait group.
// Doesn't need to be included in the wait group.
go db.compactionError()
go db.mpoolDrain()
db.closeW.Add(3)
go db.tCompaction()
@ -135,9 +146,10 @@ func openDB(s *session) (*DB, error) {
// detected in the DB. Corrupted DB can be recovered with Recover
// function.
//
// The returned DB instance is goroutine-safe.
// The DB must be closed after use, by calling Close method.
func Open(p storage.Storage, o *opt.Options) (db *DB, err error) {
s, err := newSession(p, o)
func Open(stor storage.Storage, o *opt.Options) (db *DB, err error) {
s, err := newSession(stor, o)
if err != nil {
return
}
@ -177,6 +189,7 @@ func Open(p storage.Storage, o *opt.Options) (db *DB, err error) {
// detected in the DB. Corrupted DB can be recovered with Recover
// function.
//
// The returned DB instance is goroutine-safe.
// The DB must be closed after use, by calling Close method.
func OpenFile(path string, o *opt.Options) (db *DB, err error) {
stor, err := storage.OpenFile(path)
@ -197,9 +210,10 @@ func OpenFile(path string, o *opt.Options) (db *DB, err error) {
// The DB must already exist or it will returns an error.
// Also, Recover will ignore ErrorIfMissing and ErrorIfExist options.
//
// The returned DB instance is goroutine-safe.
// The DB must be closed after use, by calling Close method.
func Recover(p storage.Storage, o *opt.Options) (db *DB, err error) {
s, err := newSession(p, o)
func Recover(stor storage.Storage, o *opt.Options) (db *DB, err error) {
s, err := newSession(stor, o)
if err != nil {
return
}
@ -225,6 +239,7 @@ func Recover(p storage.Storage, o *opt.Options) (db *DB, err error) {
// RecoverFile uses standard file-system backed storage implementation as desribed
// in the leveldb/storage package.
//
// The returned DB instance is goroutine-safe.
// The DB must be closed after use, by calling Close method.
func RecoverFile(path string, o *opt.Options) (db *DB, err error) {
stor, err := storage.OpenFile(path)
@ -241,16 +256,28 @@ func RecoverFile(path string, o *opt.Options) (db *DB, err error) {
}
func recoverTable(s *session, o *opt.Options) error {
ff0, err := s.getFiles(storage.TypeTable)
o = dupOptions(o)
// Mask StrictReader, lets StrictRecovery doing its job.
o.Strict &= ^opt.StrictReader
// Get all tables and sort it by file number.
tableFiles_, err := s.getFiles(storage.TypeTable)
if err != nil {
return err
}
ff1 := files(ff0)
ff1.sort()
tableFiles := files(tableFiles_)
tableFiles.sort()
var mSeq uint64
var good, corrupted int
rec := new(sessionRecord)
var (
maxSeq uint64
recoveredKey, goodKey, corruptedKey, corruptedBlock, droppedTable int
// We will drop corrupted table.
strict = o.GetStrict(opt.StrictRecovery)
rec = &sessionRecord{numLevel: o.GetNumLevel()}
bpool = util.NewBufferPool(o.GetBlockSize() + 5)
)
buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) {
tmp = s.newTemp()
writer, err := tmp.Create()
@ -264,8 +291,9 @@ func recoverTable(s *session, o *opt.Options) error {
tmp = nil
}
}()
// Copy entries.
tw := table.NewWriter(writer, o)
// Copy records.
for iter.Next() {
key := iter.Key()
if validIkey(key) {
@ -296,45 +324,73 @@ func recoverTable(s *session, o *opt.Options) error {
if err != nil {
return err
}
defer reader.Close()
var closed bool
defer func() {
if !closed {
reader.Close()
}
}()
// Get file size.
size, err := reader.Seek(0, 2)
if err != nil {
return err
}
var tSeq uint64
var tgood, tcorrupted, blockerr int
var min, max []byte
tr := table.NewReader(reader, size, nil, o)
var (
tSeq uint64
tgoodKey, tcorruptedKey, tcorruptedBlock int
imin, imax []byte
)
tr, err := table.NewReader(reader, size, storage.NewFileInfo(file), nil, bpool, o)
if err != nil {
return err
}
iter := tr.NewIterator(nil, nil)
iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) {
s.logf("table@recovery found error @%d %q", file.Num(), err)
blockerr++
if itererr, ok := iter.(iterator.ErrorCallbackSetter); ok {
itererr.SetErrorCallback(func(err error) {
if errors.IsCorrupted(err) {
s.logf("table@recovery block corruption @%d %q", file.Num(), err)
tcorruptedBlock++
}
})
}
// Scan the table.
for iter.Next() {
key := iter.Key()
_, seq, _, ok := parseIkey(key)
if !ok {
tcorrupted++
_, seq, _, kerr := parseIkey(key)
if kerr != nil {
tcorruptedKey++
continue
}
tgood++
tgoodKey++
if seq > tSeq {
tSeq = seq
}
if min == nil {
min = append([]byte{}, key...)
if imin == nil {
imin = append([]byte{}, key...)
}
max = append(max[:0], key...)
imax = append(imax[:0], key...)
}
if err := iter.Error(); err != nil {
iter.Release()
return err
}
iter.Release()
if tgood > 0 {
if tcorrupted > 0 || blockerr > 0 {
goodKey += tgoodKey
corruptedKey += tcorruptedKey
corruptedBlock += tcorruptedBlock
if strict && (tcorruptedKey > 0 || tcorruptedBlock > 0) {
droppedTable++
s.logf("table@recovery dropped @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
return nil
}
if tgoodKey > 0 {
if tcorruptedKey > 0 || tcorruptedBlock > 0 {
// Rebuild the table.
s.logf("table@recovery rebuilding @%d", file.Num())
iter := tr.NewIterator(nil, nil)
@ -343,62 +399,77 @@ func recoverTable(s *session, o *opt.Options) error {
if err != nil {
return err
}
closed = true
reader.Close()
if err := file.Replace(tmp); err != nil {
return err
}
size = newSize
}
if tSeq > mSeq {
mSeq = tSeq
if tSeq > maxSeq {
maxSeq = tSeq
}
recoveredKey += tgoodKey
// Add table to level 0.
rec.addTable(0, file.Num(), uint64(size), min, max)
s.logf("table@recovery recovered @%d N·%d C·%d B·%d S·%d Q·%d", file.Num(), tgood, tcorrupted, blockerr, size, tSeq)
rec.addTable(0, file.Num(), uint64(size), imin, imax)
s.logf("table@recovery recovered @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
} else {
s.logf("table@recovery unrecoverable @%d C·%d B·%d S·%d", file.Num(), tcorrupted, blockerr, size)
droppedTable++
s.logf("table@recovery unrecoverable @%d Ck·%d Cb·%d S·%d", file.Num(), tcorruptedKey, tcorruptedBlock, size)
}
good += tgood
corrupted += tcorrupted
return nil
}
// Recover all tables.
if len(ff1) > 0 {
s.logf("table@recovery F·%d", len(ff1))
s.markFileNum(ff1[len(ff1)-1].Num())
for _, file := range ff1 {
if len(tableFiles) > 0 {
s.logf("table@recovery F·%d", len(tableFiles))
// Mark file number as used.
s.markFileNum(tableFiles[len(tableFiles)-1].Num())
for _, file := range tableFiles {
if err := recoverTable(file); err != nil {
return err
}
}
s.logf("table@recovery recovered F·%d N·%d C·%d Q·%d", len(ff1), good, corrupted, mSeq)
s.logf("table@recovery recovered F·%d N·%d Gk·%d Ck·%d Q·%d", len(tableFiles), recoveredKey, goodKey, corruptedKey, maxSeq)
}
// Set sequence number.
rec.setSeq(mSeq + 1)
rec.setSeqNum(maxSeq)
// Create new manifest.
if err := s.create(); err != nil {
return err
}
// Commit.
return s.commit(rec)
}
func (d *DB) recoverJournal() error {
s := d.s
ff0, err := s.getFiles(storage.TypeJournal)
func (db *DB) recoverJournal() error {
// Get all tables and sort it by file number.
journalFiles_, err := db.s.getFiles(storage.TypeJournal)
if err != nil {
return err
}
ff1 := files(ff0)
ff1.sort()
ff2 := make([]storage.File, 0, len(ff1))
for _, file := range ff1 {
if file.Num() >= s.stJournalNum || file.Num() == s.stPrevJournalNum {
s.markFileNum(file.Num())
ff2 = append(ff2, file)
journalFiles := files(journalFiles_)
journalFiles.sort()
// Discard older journal.
prev := -1
for i, file := range journalFiles {
if file.Num() >= db.s.stJournalNum {
if prev >= 0 {
i--
journalFiles[i] = journalFiles[prev]
}
journalFiles = journalFiles[i:]
break
} else if file.Num() == db.s.stPrevJournalNum {
prev = i
}
}
@ -406,38 +477,43 @@ func (d *DB) recoverJournal() error {
var of storage.File
var mem *memdb.DB
batch := new(Batch)
cm := newCMem(s)
cm := newCMem(db.s)
buf := new(util.Buffer)
// Options.
strict := s.o.GetStrict(opt.StrictJournal)
checksum := s.o.GetStrict(opt.StrictJournalChecksum)
writeBuffer := s.o.GetWriteBuffer()
strict := db.s.o.GetStrict(opt.StrictJournal)
checksum := db.s.o.GetStrict(opt.StrictJournalChecksum)
writeBuffer := db.s.o.GetWriteBuffer()
recoverJournal := func(file storage.File) error {
s.logf("journal@recovery recovering @%d", file.Num())
db.logf("journal@recovery recovering @%d", file.Num())
reader, err := file.Open()
if err != nil {
return err
}
defer reader.Close()
// Create/reset journal reader instance.
if jr == nil {
jr = journal.NewReader(reader, dropper{s, file}, strict, checksum)
jr = journal.NewReader(reader, dropper{db.s, file}, strict, checksum)
} else {
jr.Reset(reader, dropper{s, file}, strict, checksum)
jr.Reset(reader, dropper{db.s, file}, strict, checksum)
}
// Flush memdb and remove obsolete journal file.
if of != nil {
if mem.Len() > 0 {
if err := cm.flush(mem, 0); err != nil {
return err
}
}
if err := cm.commit(file.Num(), d.seq); err != nil {
if err := cm.commit(file.Num(), db.seq); err != nil {
return err
}
cm.reset()
of.Remove()
of = nil
}
// Reset memdb.
// Replay journal to memdb.
mem.Reset()
for {
r, err := jr.Next()
@ -445,43 +521,58 @@ func (d *DB) recoverJournal() error {
if err == io.EOF {
break
}
return err
return errors.SetFile(err, file)
}
buf.Reset()
if _, err := buf.ReadFrom(r); err != nil {
if strict {
return err
if err == io.ErrUnexpectedEOF {
// This is error returned due to corruption, with strict == false.
continue
} else {
return errors.SetFile(err, file)
}
}
if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mem); err != nil {
if strict || !errors.IsCorrupted(err) {
return errors.SetFile(err, file)
} else {
db.s.logf("journal error: %v (skipped)", err)
// We won't apply sequence number as it might be corrupted.
continue
}
if err := batch.decode(buf.Bytes()); err != nil {
return err
}
if err := batch.memReplay(mem); err != nil {
return err
}
d.seq = batch.seq + uint64(batch.len())
// Save sequence number.
db.seq = batch.seq + uint64(batch.Len())
// Flush it if large enough.
if mem.Size() >= writeBuffer {
// Large enough, flush it.
if err := cm.flush(mem, 0); err != nil {
return err
}
// Reset memdb.
mem.Reset()
}
}
of = file
return nil
}
// Recover all journals.
if len(ff2) > 0 {
s.logf("journal@recovery F·%d", len(ff2))
mem = memdb.New(s.icmp, writeBuffer)
for _, file := range ff2 {
if len(journalFiles) > 0 {
db.logf("journal@recovery F·%d", len(journalFiles))
// Mark file number as used.
db.s.markFileNum(journalFiles[len(journalFiles)-1].Num())
mem = memdb.New(db.s.icmp, writeBuffer)
for _, file := range journalFiles {
if err := recoverJournal(file); err != nil {
return err
}
}
// Flush the last journal.
if mem.Len() > 0 {
if err := cm.flush(mem, 0); err != nil {
@ -489,72 +580,140 @@ func (d *DB) recoverJournal() error {
}
}
}
// Create a new journal.
if _, err := d.newMem(0); err != nil {
if _, err := db.newMem(0); err != nil {
return err
}
// Commit.
if err := cm.commit(d.journalFile.Num(), d.seq); err != nil {
if err := cm.commit(db.journalFile.Num(), db.seq); err != nil {
// Close journal.
if d.journal != nil {
d.journal.Close()
d.journalWriter.Close()
if db.journal != nil {
db.journal.Close()
db.journalWriter.Close()
}
return err
}
// Remove the last journal.
// Remove the last obsolete journal file.
if of != nil {
of.Remove()
}
return nil
}
func (d *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) {
s := d.s
func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) {
ikey := newIkey(key, seq, ktSeek)
ikey := newIKey(key, seq, tSeek)
em, fm := d.getMems()
for _, m := range [...]*memdb.DB{em, fm} {
em, fm := db.getMems()
for _, m := range [...]*memDB{em, fm} {
if m == nil {
continue
}
mk, mv, me := m.Find(ikey)
defer m.decref()
mk, mv, me := m.mdb.Find(ikey)
if me == nil {
ukey, _, t, ok := parseIkey(mk)
if ok && s.icmp.uCompare(ukey, key) == 0 {
if t == tDel {
ukey, _, kt, kerr := parseIkey(mk)
if kerr != nil {
// Shouldn't have had happen.
panic(kerr)
}
if db.s.icmp.uCompare(ukey, key) == 0 {
if kt == ktDel {
return nil, ErrNotFound
}
return mv, nil
return append([]byte{}, mv...), nil
}
} else if me != ErrNotFound {
return nil, me
}
}
v := s.version()
value, cSched, err := v.get(ikey, ro)
v := db.s.version()
value, cSched, err := v.get(ikey, ro, false)
v.release()
if cSched {
// Trigger table compaction.
d.compTrigger(d.tcompTriggerC)
db.compSendTrigger(db.tcompCmdC)
}
return
}
func (db *DB) has(key []byte, seq uint64, ro *opt.ReadOptions) (ret bool, err error) {
ikey := newIkey(key, seq, ktSeek)
em, fm := db.getMems()
for _, m := range [...]*memDB{em, fm} {
if m == nil {
continue
}
defer m.decref()
mk, _, me := m.mdb.Find(ikey)
if me == nil {
ukey, _, kt, kerr := parseIkey(mk)
if kerr != nil {
// Shouldn't have had happen.
panic(kerr)
}
if db.s.icmp.uCompare(ukey, key) == 0 {
if kt == ktDel {
return false, nil
}
return true, nil
}
} else if me != ErrNotFound {
return false, me
}
}
v := db.s.version()
_, cSched, err := v.get(ikey, ro, true)
v.release()
if cSched {
// Trigger table compaction.
db.compSendTrigger(db.tcompCmdC)
}
if err == nil {
ret = true
} else if err == ErrNotFound {
err = nil
}
return
}
// Get gets the value for the given key. It returns ErrNotFound if the
// DB does not contain the key.
// DB does not contains the key.
//
// The caller should not modify the contents of the returned slice, but
// it is safe to modify the contents of the argument after Get returns.
func (d *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
err = d.ok()
// The returned slice is its own copy, it is safe to modify the contents
// of the returned slice.
// It is safe to modify the contents of the argument after Get returns.
func (db *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
err = db.ok()
if err != nil {
return
}
return d.get(key, d.getSeq(), ro)
se := db.acquireSnapshot()
defer db.releaseSnapshot(se)
return db.get(key, se.seq, ro)
}
// Has returns true if the DB does contains the given key.
//
// It is safe to modify the contents of the argument after Get returns.
func (db *DB) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) {
err = db.ok()
if err != nil {
return
}
se := db.acquireSnapshot()
defer db.releaseSnapshot(se)
return db.has(key, se.seq, ro)
}
// NewIterator returns an iterator for the latest snapshot of the
@ -573,14 +732,16 @@ func (d *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
// The iterator must be released after use, by calling Release method.
//
// Also read Iterator documentation of the leveldb/iterator package.
func (d *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
if err := d.ok(); err != nil {
func (db *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
if err := db.ok(); err != nil {
return iterator.NewEmptyIterator(err)
}
p := d.newSnapshot()
defer p.Release()
return p.NewIterator(slice, ro)
se := db.acquireSnapshot()
defer db.releaseSnapshot(se)
// Iterator holds 'version' lock, 'version' is immutable so snapshot
// can be released after iterator created.
return db.newIterator(se.seq, slice, ro)
}
// GetSnapshot returns a latest snapshot of the underlying DB. A snapshot
@ -588,25 +749,35 @@ func (d *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterat
// content of snapshot are guaranteed to be consistent.
//
// The snapshot must be released after use, by calling Release method.
func (d *DB) GetSnapshot() (*Snapshot, error) {
if err := d.ok(); err != nil {
func (db *DB) GetSnapshot() (*Snapshot, error) {
if err := db.ok(); err != nil {
return nil, err
}
return d.newSnapshot(), nil
return db.newSnapshot(), nil
}
// GetProperty returns value of the given property name.
//
// Property names:
// leveldb.num-files-at-level{n}
// Returns the number of filer at level 'n'.
// Returns the number of files at level 'n'.
// leveldb.stats
// Returns statistics of the underlying DB.
// leveldb.sstables
// Returns sstables list for each level.
func (d *DB) GetProperty(name string) (value string, err error) {
err = d.ok()
// leveldb.blockpool
// Returns block pool stats.
// leveldb.cachedblock
// Returns size of cached block.
// leveldb.openedtables
// Returns number of opened tables.
// leveldb.alivesnaps
// Returns number of alive snapshots.
// leveldb.aliveiters
// Returns number of alive iterators.
func (db *DB) GetProperty(name string) (value string, err error) {
err = db.ok()
if err != nil {
return
}
@ -615,19 +786,18 @@ func (d *DB) GetProperty(name string) (value string, err error) {
if !strings.HasPrefix(name, prefix) {
return "", errors.New("leveldb: GetProperty: unknown property: " + name)
}
p := name[len(prefix):]
s := d.s
v := s.version()
v := db.s.version()
defer v.release()
numFilesPrefix := "num-files-at-level"
switch {
case strings.HasPrefix(p, "num-files-at-level"):
case strings.HasPrefix(p, numFilesPrefix):
var level uint
var rest string
n, _ := fmt.Scanf("%d%s", &level, &rest)
if n != 1 || level >= kNumLevels {
n, _ := fmt.Sscanf(p[len(numFilesPrefix):], "%d%s", &level, &rest)
if n != 1 || int(level) >= db.s.o.GetNumLevel() {
err = errors.New("leveldb: GetProperty: invalid property: " + name)
} else {
value = fmt.Sprint(v.tLen(int(level)))
@ -636,22 +806,36 @@ func (d *DB) GetProperty(name string) (value string, err error) {
value = "Compactions\n" +
" Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB)\n" +
"-------+------------+---------------+---------------+---------------+---------------\n"
for level, tt := range v.tables {
duration, read, write := d.compStats[level].get()
if len(tt) == 0 && duration == 0 {
for level, tables := range v.tables {
duration, read, write := db.compStats[level].get()
if len(tables) == 0 && duration == 0 {
continue
}
value += fmt.Sprintf(" %3d | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n",
level, len(tt), float64(tt.size())/1048576.0, duration.Seconds(),
level, len(tables), float64(tables.size())/1048576.0, duration.Seconds(),
float64(read)/1048576.0, float64(write)/1048576.0)
}
case p == "sstables":
for level, tt := range v.tables {
for level, tables := range v.tables {
value += fmt.Sprintf("--- level %d ---\n", level)
for _, t := range tt {
value += fmt.Sprintf("%d:%d[%q .. %q]\n", t.file.Num(), t.size, t.min, t.max)
for _, t := range tables {
value += fmt.Sprintf("%d:%d[%q .. %q]\n", t.file.Num(), t.size, t.imin, t.imax)
}
}
case p == "blockpool":
value = fmt.Sprintf("%v", db.s.tops.bpool)
case p == "cachedblock":
if db.s.tops.bcache != nil {
value = fmt.Sprintf("%d", db.s.tops.bcache.Size())
} else {
value = "<nil>"
}
case p == "openedtables":
value = fmt.Sprintf("%d", db.s.tops.cache.Size())
case p == "alivesnaps":
value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveSnaps))
case p == "aliveiters":
value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveIters))
default:
err = errors.New("leveldb: GetProperty: unknown property: " + name)
}
@ -665,23 +849,23 @@ func (d *DB) GetProperty(name string) (value string, err error) {
// data compresses by a factor of ten, the returned sizes will be one-tenth
// the size of the corresponding user data size.
// The results may not include the sizes of recently written data.
func (d *DB) SizeOf(ranges []util.Range) (Sizes, error) {
if err := d.ok(); err != nil {
func (db *DB) SizeOf(ranges []util.Range) (Sizes, error) {
if err := db.ok(); err != nil {
return nil, err
}
v := d.s.version()
v := db.s.version()
defer v.release()
sizes := make(Sizes, 0, len(ranges))
for _, r := range ranges {
min := newIKey(r.Start, kMaxSeq, tSeek)
max := newIKey(r.Limit, kMaxSeq, tSeek)
start, err := v.offsetOf(min)
imin := newIkey(r.Start, kMaxSeq, ktSeek)
imax := newIkey(r.Limit, kMaxSeq, ktSeek)
start, err := v.offsetOf(imin)
if err != nil {
return nil, err
}
limit, err := v.offsetOf(max)
limit, err := v.offsetOf(imax)
if err != nil {
return nil, err
}
@ -695,61 +879,67 @@ func (d *DB) SizeOf(ranges []util.Range) (Sizes, error) {
return sizes, nil
}
// Close closes the DB. This will also releases any outstanding snapshot.
// Close closes the DB. This will also releases any outstanding snapshot and
// abort any in-flight compaction.
//
// It is not safe to close a DB until all outstanding iterators are released.
// It is valid to call Close multiple times. Other methods should not be
// called after the DB has been closed.
func (d *DB) Close() error {
if !d.setClosed() {
func (db *DB) Close() error {
if !db.setClosed() {
return ErrClosed
}
s := d.s
start := time.Now()
s.log("db@close closing")
db.log("db@close closing")
// Clear the finalizer.
runtime.SetFinalizer(d, nil)
runtime.SetFinalizer(db, nil)
// Get compaction error.
var err error
select {
case err = <-d.compErrC:
case err = <-db.compErrC:
default:
}
close(d.closeC)
// Signal all goroutines.
close(db.closeC)
// Wait for the close WaitGroup.
d.closeW.Wait()
// Wait for all gorotines to exit.
db.closeW.Wait()
// Close journal.
if d.journal != nil {
d.journal.Close()
d.journalWriter.Close()
// Lock writer and closes journal.
db.writeLockC <- struct{}{}
if db.journal != nil {
db.journal.Close()
db.journalWriter.Close()
}
if db.writeDelayN > 0 {
db.logf("db@write was delayed N·%d T·%v", db.writeDelayN, db.writeDelay)
}
// Close session.
s.close()
s.logf("db@close done T·%v", time.Since(start))
s.release()
db.s.close()
db.logf("db@close done T·%v", time.Since(start))
db.s.release()
if d.closer != nil {
if err1 := d.closer.Close(); err == nil {
if db.closer != nil {
if err1 := db.closer.Close(); err == nil {
err = err1
}
}
d.s = nil
d.mem = nil
d.frozenMem = nil
d.journal = nil
d.journalWriter = nil
d.journalFile = nil
d.frozenJournalFile = nil
d.snapsRoot = snapshotElement{}
d.closer = nil
// NIL'ing pointers.
db.s = nil
db.mem = nil
db.frozenMem = nil
db.journal = nil
db.journalWriter = nil
db.journalFile = nil
db.frozenJournalFile = nil
db.closer = nil
return err
}

677
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go generated vendored
View File

@ -7,11 +7,12 @@
package leveldb
import (
"errors"
"sync"
"time"
"github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/memdb"
"github.com/syndtr/goleveldb/leveldb/opt"
)
var (
@ -68,13 +69,13 @@ type cMem struct {
}
func newCMem(s *session) *cMem {
return &cMem{s: s, rec: new(sessionRecord)}
return &cMem{s: s, rec: &sessionRecord{numLevel: s.o.GetNumLevel()}}
}
func (c *cMem) flush(mem *memdb.DB, level int) error {
s := c.s
// Write memdb to table
// Write memdb to table.
iter := mem.NewIterator(nil)
defer iter.Release()
t, n, err := s.tops.createFrom(iter)
@ -82,51 +83,85 @@ func (c *cMem) flush(mem *memdb.DB, level int) error {
return err
}
// Pick level.
if level < 0 {
level = s.version_NB().pickLevel(t.min.ukey(), t.max.ukey())
v := s.version()
level = v.pickLevel(t.imin.ukey(), t.imax.ukey())
v.release()
}
c.rec.addTableFile(level, t)
s.logf("mem@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.min, t.max)
s.logf("mem@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.imin, t.imax)
c.level = level
return nil
}
func (c *cMem) reset() {
c.rec = new(sessionRecord)
c.rec = &sessionRecord{numLevel: c.s.o.GetNumLevel()}
}
func (c *cMem) commit(journal, seq uint64) error {
c.rec.setJournalNum(journal)
c.rec.setSeq(seq)
// Commit changes
c.rec.setSeqNum(seq)
// Commit changes.
return c.s.commit(c.rec)
}
func (d *DB) compactionError() {
var err error
func (db *DB) compactionError() {
var (
err error
wlocked bool
)
noerr:
// No error.
for {
select {
case _, _ = <-d.closeC:
return
case err = <-d.compErrSetC:
if err != nil {
case err = <-db.compErrSetC:
switch {
case err == nil:
case errors.IsCorrupted(err):
goto hasperr
default:
goto haserr
}
case _, _ = <-db.closeC:
return
}
}
haserr:
// Transient error.
for {
select {
case _, _ = <-d.closeC:
return
case err = <-d.compErrSetC:
if err == nil {
case db.compErrC <- err:
case err = <-db.compErrSetC:
switch {
case err == nil:
goto noerr
case errors.IsCorrupted(err):
goto hasperr
default:
}
case d.compErrC <- err:
case _, _ = <-db.closeC:
return
}
}
hasperr:
// Persistent error.
for {
select {
case db.compErrC <- err:
case db.compPerErrC <- err:
case db.writeLockC <- struct{}{}:
// Hold write lock, so that write won't pass-through.
wlocked = true
case _, _ = <-db.closeC:
if wlocked {
// We should release the lock or Close will hang.
<-db.writeLockC
}
return
}
}
}
@ -137,51 +172,72 @@ func (cnt *compactionTransactCounter) incr() {
*cnt++
}
func (d *DB) compactionTransact(name string, exec func(cnt *compactionTransactCounter) error, rollback func() error) {
s := d.s
type compactionTransactInterface interface {
run(cnt *compactionTransactCounter) error
revert() error
}
func (db *DB) compactionTransact(name string, t compactionTransactInterface) {
defer func() {
if x := recover(); x != nil {
if x == errCompactionTransactExiting && rollback != nil {
if err := rollback(); err != nil {
s.logf("%s rollback error %q", name, err)
if x == errCompactionTransactExiting {
if err := t.revert(); err != nil {
db.logf("%s revert error %q", name, err)
}
}
panic(x)
}
}()
const (
backoffMin = 1 * time.Second
backoffMax = 8 * time.Second
backoffMul = 2 * time.Second
)
backoff := backoffMin
backoffT := time.NewTimer(backoff)
lastCnt := compactionTransactCounter(0)
var (
backoff = backoffMin
backoffT = time.NewTimer(backoff)
lastCnt = compactionTransactCounter(0)
disableBackoff = db.s.o.GetDisableCompactionBackoff()
)
for n := 0; ; n++ {
// Check wether the DB is closed.
if d.isClosed() {
s.logf("%s exiting", name)
d.compactionExitTransact()
if db.isClosed() {
db.logf("%s exiting", name)
db.compactionExitTransact()
} else if n > 0 {
s.logf("%s retrying N·%d", name, n)
db.logf("%s retrying N·%d", name, n)
}
// Execute.
cnt := compactionTransactCounter(0)
err := exec(&cnt)
err := t.run(&cnt)
if err != nil {
db.logf("%s error I·%d %q", name, cnt, err)
}
// Set compaction error status.
select {
case d.compErrSetC <- err:
case _, _ = <-d.closeC:
s.logf("%s exiting", name)
d.compactionExitTransact()
case db.compErrSetC <- err:
case perr := <-db.compPerErrC:
if err != nil {
db.logf("%s exiting (persistent error %q)", name, perr)
db.compactionExitTransact()
}
case _, _ = <-db.closeC:
db.logf("%s exiting", name)
db.compactionExitTransact()
}
if err == nil {
return
}
s.logf("%s error I·%d %q", name, cnt, err)
if errors.IsCorrupted(err) {
db.logf("%s exiting (corruption detected)", name)
db.compactionExitTransact()
}
if !disableBackoff {
// Reset backoff duration if counter is advancing.
if cnt > lastCnt {
backoff = backoffMin
@ -198,53 +254,77 @@ func (d *DB) compactionTransact(name string, exec func(cnt *compactionTransactCo
}
select {
case <-backoffT.C:
case _, _ = <-d.closeC:
s.logf("%s exiting", name)
d.compactionExitTransact()
case _, _ = <-db.closeC:
db.logf("%s exiting", name)
db.compactionExitTransact()
}
}
}
}
func (d *DB) compactionExitTransact() {
type compactionTransactFunc struct {
runFunc func(cnt *compactionTransactCounter) error
revertFunc func() error
}
func (t *compactionTransactFunc) run(cnt *compactionTransactCounter) error {
return t.runFunc(cnt)
}
func (t *compactionTransactFunc) revert() error {
if t.revertFunc != nil {
return t.revertFunc()
}
return nil
}
func (db *DB) compactionTransactFunc(name string, run func(cnt *compactionTransactCounter) error, revert func() error) {
db.compactionTransact(name, &compactionTransactFunc{run, revert})
}
func (db *DB) compactionExitTransact() {
panic(errCompactionTransactExiting)
}
func (d *DB) memCompaction() {
mem := d.getFrozenMem()
func (db *DB) memCompaction() {
mem := db.getFrozenMem()
if mem == nil {
return
}
defer mem.decref()
s := d.s
c := newCMem(s)
c := newCMem(db.s)
stats := new(cStatsStaging)
s.logf("mem@flush N·%d S·%s", mem.Len(), shortenb(mem.Size()))
db.logf("mem@flush N·%d S·%s", mem.mdb.Len(), shortenb(mem.mdb.Size()))
// Don't compact empty memdb.
if mem.Len() == 0 {
s.logf("mem@flush skipping")
if mem.mdb.Len() == 0 {
db.logf("mem@flush skipping")
// drop frozen mem
d.dropFrozenMem()
db.dropFrozenMem()
return
}
// Pause table compaction.
ch := make(chan struct{})
resumeC := make(chan struct{})
select {
case d.tcompPauseC <- (chan<- struct{})(ch):
case _, _ = <-d.closeC:
case db.tcompPauseC <- (chan<- struct{})(resumeC):
case <-db.compPerErrC:
close(resumeC)
resumeC = nil
case _, _ = <-db.closeC:
return
}
d.compactionTransact("mem@flush", func(cnt *compactionTransactCounter) (err error) {
db.compactionTransactFunc("mem@flush", func(cnt *compactionTransactCounter) (err error) {
stats.startTimer()
defer stats.stopTimer()
return c.flush(mem, -1)
return c.flush(mem.mdb, -1)
}, func() error {
for _, r := range c.rec.addedTables {
s.logf("mem@flush rollback @%d", r.num)
f := s.getTableFile(r.num)
db.logf("mem@flush revert @%d", r.num)
f := db.s.getTableFile(r.num)
if err := f.Remove(); err != nil {
return err
}
@ -252,147 +332,176 @@ func (d *DB) memCompaction() {
return nil
})
d.compactionTransact("mem@commit", func(cnt *compactionTransactCounter) (err error) {
db.compactionTransactFunc("mem@commit", func(cnt *compactionTransactCounter) (err error) {
stats.startTimer()
defer stats.stopTimer()
return c.commit(d.journalFile.Num(), d.frozenSeq)
return c.commit(db.journalFile.Num(), db.frozenSeq)
}, nil)
s.logf("mem@flush commited F·%d T·%v", len(c.rec.addedTables), stats.duration)
db.logf("mem@flush committed F·%d T·%v", len(c.rec.addedTables), stats.duration)
for _, r := range c.rec.addedTables {
stats.write += r.size
}
d.compStats[c.level].add(stats)
db.compStats[c.level].add(stats)
// Drop frozen mem.
d.dropFrozenMem()
db.dropFrozenMem()
// Resume table compaction.
if resumeC != nil {
select {
case <-ch:
case _, _ = <-d.closeC:
case <-resumeC:
close(resumeC)
case _, _ = <-db.closeC:
return
}
}
// Trigger table compaction.
d.compTrigger(d.mcompTriggerC)
db.compSendTrigger(db.tcompCmdC)
}
func (d *DB) tableCompaction(c *compaction, noTrivial bool) {
s := d.s
type tableCompactionBuilder struct {
db *DB
s *session
c *compaction
rec *sessionRecord
stat0, stat1 *cStatsStaging
rec := new(sessionRecord)
rec.addCompactionPointer(c.level, c.max)
snapHasLastUkey bool
snapLastUkey []byte
snapLastSeq uint64
snapIter int
snapKerrCnt int
snapDropCnt int
if !noTrivial && c.trivial() {
t := c.tables[0][0]
s.logf("table@move L%d@%d -> L%d", c.level, t.file.Num(), c.level+1)
rec.deleteTable(c.level, t.file.Num())
rec.addTableFile(c.level+1, t)
d.compactionTransact("table@move", func(cnt *compactionTransactCounter) (err error) {
return s.commit(rec)
}, nil)
return
}
kerrCnt int
dropCnt int
var stats [2]cStatsStaging
for i, tt := range c.tables {
for _, t := range tt {
stats[i].read += t.size
// Insert deleted tables into record
rec.deleteTable(c.level+i, t.file.Num())
minSeq uint64
strict bool
tableSize int
tw *tWriter
}
func (b *tableCompactionBuilder) appendKV(key, value []byte) error {
// Create new table if not already.
if b.tw == nil {
// Check for pause event.
if b.db != nil {
select {
case ch := <-b.db.tcompPauseC:
b.db.pauseCompaction(ch)
case _, _ = <-b.db.closeC:
b.db.compactionExitTransact()
default:
}
}
sourceSize := int(stats[0].read + stats[1].read)
minSeq := d.minSeq()
s.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.level, len(c.tables[0]), c.level+1, len(c.tables[1]), shortenb(sourceSize), minSeq)
var snapUkey []byte
var snapHasUkey bool
var snapSeq uint64
var snapIter int
var snapDropCnt int
var dropCnt int
d.compactionTransact("table@build", func(cnt *compactionTransactCounter) (err error) {
ukey := append([]byte{}, snapUkey...)
hasUkey := snapHasUkey
lseq := snapSeq
dropCnt = snapDropCnt
snapSched := snapIter == 0
var tw *tWriter
finish := func() error {
t, err := tw.finish()
// Create new table.
var err error
b.tw, err = b.s.tops.create()
if err != nil {
return err
}
rec.addTableFile(c.level+1, t)
stats[1].write += t.size
s.logf("table@build created L%d@%d N·%d S·%s %q:%q", c.level+1, t.file.Num(), tw.tw.EntriesLen(), shortenb(int(t.size)), t.min, t.max)
}
// Write key/value into table.
return b.tw.append(key, value)
}
func (b *tableCompactionBuilder) needFlush() bool {
return b.tw.tw.BytesLen() >= b.tableSize
}
func (b *tableCompactionBuilder) flush() error {
t, err := b.tw.finish()
if err != nil {
return err
}
b.rec.addTableFile(b.c.level+1, t)
b.stat1.write += t.size
b.s.logf("table@build created L%d@%d N·%d S·%s %q:%q", b.c.level+1, t.file.Num(), b.tw.tw.EntriesLen(), shortenb(int(t.size)), t.imin, t.imax)
b.tw = nil
return nil
}
}
defer func() {
stats[1].stopTimer()
if tw != nil {
tw.drop()
tw = nil
func (b *tableCompactionBuilder) cleanup() {
if b.tw != nil {
b.tw.drop()
b.tw = nil
}
}()
}
stats[1].startTimer()
iter := c.newIterator()
func (b *tableCompactionBuilder) run(cnt *compactionTransactCounter) error {
snapResumed := b.snapIter > 0
hasLastUkey := b.snapHasLastUkey // The key might has zero length, so this is necessary.
lastUkey := append([]byte{}, b.snapLastUkey...)
lastSeq := b.snapLastSeq
b.kerrCnt = b.snapKerrCnt
b.dropCnt = b.snapDropCnt
// Restore compaction state.
b.c.restore()
defer b.cleanup()
b.stat1.startTimer()
defer b.stat1.stopTimer()
iter := b.c.newIterator()
defer iter.Release()
for i := 0; iter.Next(); i++ {
// Incr transact counter.
cnt.incr()
// Skip until last state.
if i < snapIter {
if i < b.snapIter {
continue
}
key := iKey(iter.Key())
if c.shouldStopBefore(key) && tw != nil {
err = finish()
if err != nil {
return
}
snapSched = true
tw = nil
resumed := false
if snapResumed {
resumed = true
snapResumed = false
}
// Scheduled for snapshot, snapshot will used to retry compaction
// if error occured.
if snapSched {
snapUkey = append(snapUkey[:0], ukey...)
snapHasUkey = hasUkey
snapSeq = lseq
snapIter = i
snapDropCnt = dropCnt
snapSched = false
ikey := iter.Key()
ukey, seq, kt, kerr := parseIkey(ikey)
if kerr == nil {
shouldStop := !resumed && b.c.shouldStopBefore(ikey)
if !hasLastUkey || b.s.icmp.uCompare(lastUkey, ukey) != 0 {
// First occurrence of this user key.
// Only rotate tables if ukey doesn't hop across.
if b.tw != nil && (shouldStop || b.needFlush()) {
if err := b.flush(); err != nil {
return err
}
if seq, t, ok := key.parseNum(); !ok {
// Don't drop error keys
ukey = ukey[:0]
hasUkey = false
lseq = kMaxSeq
} else {
if !hasUkey || s.icmp.uCompare(key.ukey(), ukey) != 0 {
// First occurrence of this user key
ukey = append(ukey[:0], key.ukey()...)
hasUkey = true
lseq = kMaxSeq
// Creates snapshot of the state.
b.c.save()
b.snapHasLastUkey = hasLastUkey
b.snapLastUkey = append(b.snapLastUkey[:0], lastUkey...)
b.snapLastSeq = lastSeq
b.snapIter = i
b.snapKerrCnt = b.kerrCnt
b.snapDropCnt = b.dropCnt
}
drop := false
if lseq <= minSeq {
hasLastUkey = true
lastUkey = append(lastUkey[:0], ukey...)
lastSeq = kMaxSeq
}
switch {
case lastSeq <= b.minSeq:
// Dropped because newer entry for same user key exist
drop = true // (A)
} else if t == tDel && seq <= minSeq && c.isBaseLevelForKey(ukey) {
fallthrough // (A)
case kt == ktDel && seq <= b.minSeq && b.c.baseLevelForKey(lastUkey):
// For this user key:
// (1) there is no data in higher levels
// (2) data in lower levels will have larger seq numbers
@ -400,131 +509,150 @@ func (d *DB) tableCompaction(c *compaction, noTrivial bool) {
// smaller seq numbers will be dropped in the next
// few iterations of this loop (by rule (A) above).
// Therefore this deletion marker is obsolete and can be dropped.
drop = true
}
lseq = seq
if drop {
dropCnt++
lastSeq = seq
b.dropCnt++
continue
}
}
// Create new table if not already
if tw == nil {
// Check for pause event.
select {
case ch := <-d.tcompPauseC:
d.pauseCompaction(ch)
case _, _ = <-d.closeC:
d.compactionExitTransact()
default:
lastSeq = seq
}
} else {
if b.strict {
return kerr
}
// Create new table.
tw, err = s.tops.create()
if err != nil {
return
// Don't drop corrupted keys.
hasLastUkey = false
lastUkey = lastUkey[:0]
lastSeq = kMaxSeq
b.kerrCnt++
}
if err := b.appendKV(ikey, iter.Value()); err != nil {
return err
}
}
// Write key/value into table
err = tw.add(key, iter.Value())
if err != nil {
return
if err := iter.Error(); err != nil {
return err
}
// Finish table if it is big enough
if tw.tw.BytesLen() >= kMaxTableSize {
err = finish()
if err != nil {
return
}
snapSched = true
tw = nil
}
// Finish last table.
if b.tw != nil && !b.tw.empty() {
return b.flush()
}
return nil
}
err = iter.Error()
if err != nil {
return
}
// Finish last table
if tw != nil && !tw.empty() {
err = finish()
if err != nil {
return
}
tw = nil
}
return
}, func() error {
for _, r := range rec.addedTables {
s.logf("table@build rollback @%d", r.num)
f := s.getTableFile(r.num)
func (b *tableCompactionBuilder) revert() error {
for _, at := range b.rec.addedTables {
b.s.logf("table@build revert @%d", at.num)
f := b.s.getTableFile(at.num)
if err := f.Remove(); err != nil {
return err
}
}
return nil
})
}
func (db *DB) tableCompaction(c *compaction, noTrivial bool) {
defer c.release()
rec := &sessionRecord{numLevel: db.s.o.GetNumLevel()}
rec.addCompPtr(c.level, c.imax)
if !noTrivial && c.trivial() {
t := c.tables[0][0]
db.logf("table@move L%d@%d -> L%d", c.level, t.file.Num(), c.level+1)
rec.delTable(c.level, t.file.Num())
rec.addTableFile(c.level+1, t)
db.compactionTransactFunc("table@move", func(cnt *compactionTransactCounter) (err error) {
return db.s.commit(rec)
}, nil)
return
}
var stats [2]cStatsStaging
for i, tables := range c.tables {
for _, t := range tables {
stats[i].read += t.size
// Insert deleted tables into record
rec.delTable(c.level+i, t.file.Num())
}
}
sourceSize := int(stats[0].read + stats[1].read)
minSeq := db.minSeq()
db.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.level, len(c.tables[0]), c.level+1, len(c.tables[1]), shortenb(sourceSize), minSeq)
b := &tableCompactionBuilder{
db: db,
s: db.s,
c: c,
rec: rec,
stat1: &stats[1],
minSeq: minSeq,
strict: db.s.o.GetStrict(opt.StrictCompaction),
tableSize: db.s.o.GetCompactionTableSize(c.level + 1),
}
db.compactionTransact("table@build", b)
// Commit changes
d.compactionTransact("table@commit", func(cnt *compactionTransactCounter) (err error) {
db.compactionTransactFunc("table@commit", func(cnt *compactionTransactCounter) (err error) {
stats[1].startTimer()
defer stats[1].stopTimer()
return s.commit(rec)
return db.s.commit(rec)
}, nil)
resultSize := int(int(stats[1].write))
s.logf("table@compaction commited F%s S%s D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), dropCnt, stats[1].duration)
resultSize := int(stats[1].write)
db.logf("table@compaction committed F%s S%s Ke·%d D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), b.kerrCnt, b.dropCnt, stats[1].duration)
// Save compaction stats
for i := range stats {
d.compStats[c.level+1].add(&stats[i])
db.compStats[c.level+1].add(&stats[i])
}
}
func (d *DB) tableRangeCompaction(level int, min, max []byte) {
s := d.s
s.logf("table@compaction range L%d %q:%q", level, min, max)
func (db *DB) tableRangeCompaction(level int, umin, umax []byte) {
db.logf("table@compaction range L%d %q:%q", level, umin, umax)
if level >= 0 {
if c := s.getCompactionRange(level, min, max); c != nil {
d.tableCompaction(c, true)
if c := db.s.getCompactionRange(level, umin, umax); c != nil {
db.tableCompaction(c, true)
}
} else {
v := s.version_NB()
v := db.s.version()
m := 1
for i, t := range v.tables[1:] {
if t.isOverlaps(min, max, true, s.icmp) {
if t.overlaps(db.s.icmp, umin, umax, false) {
m = i + 1
}
}
v.release()
for level := 0; level < m; level++ {
if c := s.getCompactionRange(level, min, max); c != nil {
d.tableCompaction(c, true)
if c := db.s.getCompactionRange(level, umin, umax); c != nil {
db.tableCompaction(c, true)
}
}
}
}
func (d *DB) tableAutoCompaction() {
if c := d.s.pickCompaction(); c != nil {
d.tableCompaction(c, false)
func (db *DB) tableAutoCompaction() {
if c := db.s.pickCompaction(); c != nil {
db.tableCompaction(c, false)
}
}
func (d *DB) tableNeedCompaction() bool {
return d.s.version_NB().needCompaction()
func (db *DB) tableNeedCompaction() bool {
v := db.s.version()
defer v.release()
return v.needCompaction()
}
func (d *DB) pauseCompaction(ch chan<- struct{}) {
func (db *DB) pauseCompaction(ch chan<- struct{}) {
select {
case ch <- struct{}{}:
case _, _ = <-d.closeC:
d.compactionExitTransact()
case _, _ = <-db.closeC:
db.compactionExitTransact()
}
}
@ -537,7 +665,12 @@ type cIdle struct {
}
func (r cIdle) ack(err error) {
if r.ackC != nil {
defer func() {
recover()
}()
r.ackC <- err
}
}
type cRange struct {
@ -547,56 +680,67 @@ type cRange struct {
}
func (r cRange) ack(err error) {
if r.ackC != nil {
defer func() {
recover()
}()
if r.ackC != nil {
r.ackC <- err
}
}
func (d *DB) compSendIdle(compC chan<- cCmd) error {
// This will trigger auto compation and/or wait for all compaction to be done.
func (db *DB) compSendIdle(compC chan<- cCmd) (err error) {
ch := make(chan error)
defer close(ch)
// Send cmd.
select {
case compC <- cIdle{ch}:
case err := <-d.compErrC:
return err
case _, _ = <-d.closeC:
case err = <-db.compErrC:
return
case _, _ = <-db.closeC:
return ErrClosed
}
// Wait cmd.
return <-ch
select {
case err = <-ch:
case err = <-db.compErrC:
case _, _ = <-db.closeC:
return ErrClosed
}
return err
}
func (d *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err error) {
// This will trigger auto compaction but will not wait for it.
func (db *DB) compSendTrigger(compC chan<- cCmd) {
select {
case compC <- cIdle{}:
default:
}
}
// Send range compaction request.
func (db *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err error) {
ch := make(chan error)
defer close(ch)
// Send cmd.
select {
case compC <- cRange{level, min, max, ch}:
case err := <-d.compErrC:
case err := <-db.compErrC:
return err
case _, _ = <-d.closeC:
case _, _ = <-db.closeC:
return ErrClosed
}
// Wait cmd.
select {
case err = <-d.compErrC:
case err = <-ch:
case err = <-db.compErrC:
case _, _ = <-db.closeC:
return ErrClosed
}
return err
}
func (d *DB) compTrigger(compTriggerC chan struct{}) {
select {
case compTriggerC <- struct{}{}:
default:
}
}
func (d *DB) mCompaction() {
func (db *DB) mCompaction() {
var x cCmd
defer func() {
@ -608,24 +752,27 @@ func (d *DB) mCompaction() {
if x != nil {
x.ack(ErrClosed)
}
d.closeW.Done()
db.closeW.Done()
}()
for {
select {
case _, _ = <-d.closeC:
return
case x = <-d.mcompCmdC:
d.memCompaction()
case x = <-db.mcompCmdC:
switch x.(type) {
case cIdle:
db.memCompaction()
x.ack(nil)
x = nil
case <-d.mcompTriggerC:
d.memCompaction()
default:
panic("leveldb: unknown command")
}
case _, _ = <-db.closeC:
return
}
}
}
func (d *DB) tCompaction() {
func (db *DB) tCompaction() {
var x cCmd
var ackQ []cCmd
@ -642,19 +789,18 @@ func (d *DB) tCompaction() {
if x != nil {
x.ack(ErrClosed)
}
d.closeW.Done()
db.closeW.Done()
}()
for {
if d.tableNeedCompaction() {
if db.tableNeedCompaction() {
select {
case x = <-d.tcompCmdC:
case <-d.tcompTriggerC:
case _, _ = <-d.closeC:
return
case ch := <-d.tcompPauseC:
d.pauseCompaction(ch)
case x = <-db.tcompCmdC:
case ch := <-db.tcompPauseC:
db.pauseCompaction(ch)
continue
case _, _ = <-db.closeC:
return
default:
}
} else {
@ -664,12 +810,11 @@ func (d *DB) tCompaction() {
}
ackQ = ackQ[:0]
select {
case x = <-d.tcompCmdC:
case <-d.tcompTriggerC:
case ch := <-d.tcompPauseC:
d.pauseCompaction(ch)
case x = <-db.tcompCmdC:
case ch := <-db.tcompPauseC:
db.pauseCompaction(ch)
continue
case _, _ = <-d.closeC:
case _, _ = <-db.closeC:
return
}
}
@ -678,11 +823,13 @@ func (d *DB) tCompaction() {
case cIdle:
ackQ = append(ackQ, x)
case cRange:
d.tableRangeCompaction(cmd.level, cmd.min, cmd.max)
db.tableRangeCompaction(cmd.level, cmd.min, cmd.max)
x.ack(nil)
default:
panic("leveldb: unknown command")
}
x = nil
}
d.tableAutoCompaction()
db.tableAutoCompaction()
}
}

98
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go generated vendored
View File

@ -8,7 +8,10 @@ package leveldb
import (
"errors"
"math/rand"
"runtime"
"sync"
"sync/atomic"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
@ -19,50 +22,69 @@ var (
errInvalidIkey = errors.New("leveldb: Iterator: invalid internal key")
)
func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
s := db.s
type memdbReleaser struct {
once sync.Once
m *memDB
}
func (mr *memdbReleaser) Release() {
mr.once.Do(func() {
mr.m.decref()
})
}
func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
em, fm := db.getMems()
v := s.version()
v := db.s.version()
ti := v.getIterators(slice, ro)
n := len(ti) + 2
i := make([]iterator.Iterator, 0, n)
i = append(i, em.NewIterator(slice))
emi := em.mdb.NewIterator(slice)
emi.SetReleaser(&memdbReleaser{m: em})
i = append(i, emi)
if fm != nil {
i = append(i, fm.NewIterator(slice))
fmi := fm.mdb.NewIterator(slice)
fmi.SetReleaser(&memdbReleaser{m: fm})
i = append(i, fmi)
}
i = append(i, ti...)
strict := s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator)
mi := iterator.NewMergedIterator(i, s.icmp, strict)
strict := opt.GetStrict(db.s.o.Options, ro, opt.StrictReader)
mi := iterator.NewMergedIterator(i, db.s.icmp, strict)
mi.SetReleaser(&versionReleaser{v: v})
return mi
}
func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *dbIter {
var slice_ *util.Range
var islice *util.Range
if slice != nil {
slice_ = &util.Range{}
islice = &util.Range{}
if slice.Start != nil {
slice_.Start = newIKey(slice.Start, kMaxSeq, tSeek)
islice.Start = newIkey(slice.Start, kMaxSeq, ktSeek)
}
if slice.Limit != nil {
slice_.Limit = newIKey(slice.Limit, kMaxSeq, tSeek)
islice.Limit = newIkey(slice.Limit, kMaxSeq, ktSeek)
}
}
rawIter := db.newRawIterator(slice_, ro)
rawIter := db.newRawIterator(islice, ro)
iter := &dbIter{
db: db,
icmp: db.s.icmp,
iter: rawIter,
seq: seq,
strict: db.s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator),
strict: opt.GetStrict(db.s.o.Options, ro, opt.StrictReader),
key: make([]byte, 0),
value: make([]byte, 0),
}
atomic.AddInt32(&db.aliveIters, 1)
runtime.SetFinalizer(iter, (*dbIter).Release)
return iter
}
func (db *DB) iterSamplingRate() int {
return rand.Intn(2 * db.s.o.GetIteratorSamplingRate())
}
type dir int
const (
@ -75,11 +97,13 @@ const (
// dbIter represent an interator states over a database session.
type dbIter struct {
db *DB
icmp *iComparer
iter iterator.Iterator
seq uint64
strict bool
smaplingGap int
dir dir
key []byte
value []byte
@ -87,6 +111,15 @@ type dbIter struct {
releaser util.Releaser
}
func (i *dbIter) sampleSeek() {
ikey := i.iter.Key()
i.smaplingGap -= len(ikey) + len(i.iter.Value())
for i.smaplingGap < 0 {
i.smaplingGap += i.db.iterSamplingRate()
i.db.sampleSeek(ikey)
}
}
func (i *dbIter) setErr(err error) {
i.err = err
i.key = nil
@ -144,7 +177,7 @@ func (i *dbIter) Seek(key []byte) bool {
return false
}
ikey := newIKey(key, i.seq, tSeek)
ikey := newIkey(key, i.seq, ktSeek)
if i.iter.Seek(ikey) {
i.dir = dirSOI
return i.next()
@ -156,15 +189,15 @@ func (i *dbIter) Seek(key []byte) bool {
func (i *dbIter) next() bool {
for {
ukey, seq, t, ok := parseIkey(i.iter.Key())
if ok {
if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil {
i.sampleSeek()
if seq <= i.seq {
switch t {
case tDel:
switch kt {
case ktDel:
// Skip deleted key.
i.key = append(i.key[:0], ukey...)
i.dir = dirForward
case tVal:
case ktVal:
if i.dir == dirSOI || i.icmp.uCompare(ukey, i.key) > 0 {
i.key = append(i.key[:0], ukey...)
i.value = append(i.value[:0], i.iter.Value()...)
@ -174,7 +207,7 @@ func (i *dbIter) next() bool {
}
}
} else if i.strict {
i.setErr(errInvalidIkey)
i.setErr(kerr)
break
}
if !i.iter.Next() {
@ -207,20 +240,20 @@ func (i *dbIter) prev() bool {
del := true
if i.iter.Valid() {
for {
ukey, seq, t, ok := parseIkey(i.iter.Key())
if ok {
if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil {
i.sampleSeek()
if seq <= i.seq {
if !del && i.icmp.uCompare(ukey, i.key) < 0 {
return true
}
del = (t == tDel)
del = (kt == ktDel)
if !del {
i.key = append(i.key[:0], ukey...)
i.value = append(i.value[:0], i.iter.Value()...)
}
}
} else if i.strict {
i.setErr(errInvalidIkey)
i.setErr(kerr)
return false
}
if !i.iter.Prev() {
@ -249,13 +282,13 @@ func (i *dbIter) Prev() bool {
return i.Last()
case dirForward:
for i.iter.Prev() {
ukey, _, _, ok := parseIkey(i.iter.Key())
if ok {
if ukey, _, _, kerr := parseIkey(i.iter.Key()); kerr == nil {
i.sampleSeek()
if i.icmp.uCompare(ukey, i.key) < 0 {
goto cont
}
} else if i.strict {
i.setErr(errInvalidIkey)
i.setErr(kerr)
return false
}
}
@ -289,6 +322,7 @@ func (i *dbIter) Release() {
if i.releaser != nil {
i.releaser.Release()
i.releaser = nil
}
i.dir = dirReleased
@ -296,13 +330,19 @@ func (i *dbIter) Release() {
i.value = nil
i.iter.Release()
i.iter = nil
atomic.AddInt32(&i.db.aliveIters, -1)
i.db = nil
}
}
func (i *dbIter) SetReleaser(releaser util.Releaser) {
if i.dir != dirReleased {
i.releaser = releaser
if i.dir == dirReleased {
panic(util.ErrReleased)
}
if i.releaser != nil && releaser != nil {
panic(util.ErrHasReleaser)
}
i.releaser = releaser
}
func (i *dbIter) Error() error {

146
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_snapshot.go generated vendored
View File

@ -7,8 +7,11 @@
package leveldb
import (
"container/list"
"fmt"
"runtime"
"sync"
"sync/atomic"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
@ -18,62 +21,53 @@ import (
type snapshotElement struct {
seq uint64
ref int
// Next and previous pointers in the doubly-linked list of elements.
next, prev *snapshotElement
}
// Initialize the snapshot.
func (db *DB) initSnapshot() {
db.snapsRoot.next = &db.snapsRoot
db.snapsRoot.prev = &db.snapsRoot
e *list.Element
}
// Acquires a snapshot, based on latest sequence.
func (db *DB) acquireSnapshot() *snapshotElement {
db.snapsMu.Lock()
defer db.snapsMu.Unlock()
seq := db.getSeq()
elem := db.snapsRoot.prev
if elem == &db.snapsRoot || elem.seq != seq {
at := db.snapsRoot.prev
next := at.next
elem = &snapshotElement{
seq: seq,
prev: at,
next: next,
if e := db.snapsList.Back(); e != nil {
se := e.Value.(*snapshotElement)
if se.seq == seq {
se.ref++
return se
} else if seq < se.seq {
panic("leveldb: sequence number is not increasing")
}
at.next = elem
next.prev = elem
}
elem.ref++
db.snapsMu.Unlock()
return elem
se := &snapshotElement{seq: seq, ref: 1}
se.e = db.snapsList.PushBack(se)
return se
}
// Releases given snapshot element.
func (db *DB) releaseSnapshot(elem *snapshotElement) {
if !db.isClosed() {
func (db *DB) releaseSnapshot(se *snapshotElement) {
db.snapsMu.Lock()
elem.ref--
if elem.ref == 0 {
elem.prev.next = elem.next
elem.next.prev = elem.prev
elem.next = nil
elem.prev = nil
} else if elem.ref < 0 {
defer db.snapsMu.Unlock()
se.ref--
if se.ref == 0 {
db.snapsList.Remove(se.e)
se.e = nil
} else if se.ref < 0 {
panic("leveldb: Snapshot: negative element reference")
}
db.snapsMu.Unlock()
}
}
// Gets minimum sequence that not being snapshoted.
func (db *DB) minSeq() uint64 {
db.snapsMu.Lock()
defer db.snapsMu.Unlock()
elem := db.snapsRoot.prev
if elem != &db.snapsRoot {
return elem.seq
if e := db.snapsList.Front(); e != nil {
return e.Value.(*snapshotElement).seq
}
return db.getSeq()
}
@ -81,38 +75,59 @@ func (db *DB) minSeq() uint64 {
type Snapshot struct {
db *DB
elem *snapshotElement
mu sync.Mutex
mu sync.RWMutex
released bool
}
// Creates new snapshot object.
func (db *DB) newSnapshot() *Snapshot {
p := &Snapshot{
snap := &Snapshot{
db: db,
elem: db.acquireSnapshot(),
}
runtime.SetFinalizer(p, (*Snapshot).Release)
return p
atomic.AddInt32(&db.aliveSnaps, 1)
runtime.SetFinalizer(snap, (*Snapshot).Release)
return snap
}
func (snap *Snapshot) String() string {
return fmt.Sprintf("leveldb.Snapshot{%d}", snap.elem.seq)
}
// Get gets the value for the given key. It returns ErrNotFound if
// the DB does not contain the key.
// the DB does not contains the key.
//
// The caller should not modify the contents of the returned slice, but
// it is safe to modify the contents of the argument after Get returns.
func (p *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
db := p.db
err = db.ok()
func (snap *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
err = snap.db.ok()
if err != nil {
return
}
p.mu.Lock()
defer p.mu.Unlock()
if p.released {
snap.mu.RLock()
defer snap.mu.RUnlock()
if snap.released {
err = ErrSnapshotReleased
return
}
return db.get(key, p.elem.seq, ro)
return snap.db.get(key, snap.elem.seq, ro)
}
// Has returns true if the DB does contains the given key.
//
// It is safe to modify the contents of the argument after Get returns.
func (snap *Snapshot) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) {
err = snap.db.ok()
if err != nil {
return
}
snap.mu.RLock()
defer snap.mu.RUnlock()
if snap.released {
err = ErrSnapshotReleased
return
}
return snap.db.has(key, snap.elem.seq, ro)
}
// NewIterator returns an iterator for the snapshot of the uderlying DB.
@ -132,17 +147,18 @@ func (p *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error
// iterator would be still valid until released.
//
// Also read Iterator documentation of the leveldb/iterator package.
func (p *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
db := p.db
if err := db.ok(); err != nil {
func (snap *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
if err := snap.db.ok(); err != nil {
return iterator.NewEmptyIterator(err)
}
p.mu.Lock()
defer p.mu.Unlock()
if p.released {
snap.mu.Lock()
defer snap.mu.Unlock()
if snap.released {
return iterator.NewEmptyIterator(ErrSnapshotReleased)
}
return db.newIterator(p.elem.seq, slice, ro)
// Since iterator already hold version ref, it doesn't need to
// hold snapshot ref.
return snap.db.newIterator(snap.elem.seq, slice, ro)
}
// Release releases the snapshot. This will not release any returned
@ -150,16 +166,18 @@ func (p *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.
// underlying DB is closed.
//
// Other methods should not be called after the snapshot has been released.
func (p *Snapshot) Release() {
p.mu.Lock()
if !p.released {
// Clear the finalizer.
runtime.SetFinalizer(p, nil)
func (snap *Snapshot) Release() {
snap.mu.Lock()
defer snap.mu.Unlock()
p.released = true
p.db.releaseSnapshot(p.elem)
p.db = nil
p.elem = nil
if !snap.released {
// Clear the finalizer.
runtime.SetFinalizer(snap, nil)
snap.released = true
snap.db.releaseSnapshot(snap.elem)
atomic.AddInt32(&snap.db.aliveSnaps, -1)
snap.db = nil
snap.elem = nil
}
p.mu.Unlock()
}

207
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go generated vendored
View File

@ -8,106 +8,203 @@ package leveldb
import (
"sync/atomic"
"time"
"github.com/syndtr/goleveldb/leveldb/journal"
"github.com/syndtr/goleveldb/leveldb/memdb"
)
type memDB struct {
db *DB
mdb *memdb.DB
ref int32
}
func (m *memDB) incref() {
atomic.AddInt32(&m.ref, 1)
}
func (m *memDB) decref() {
if ref := atomic.AddInt32(&m.ref, -1); ref == 0 {
// Only put back memdb with std capacity.
if m.mdb.Capacity() == m.db.s.o.GetWriteBuffer() {
m.mdb.Reset()
m.db.mpoolPut(m.mdb)
}
m.db = nil
m.mdb = nil
} else if ref < 0 {
panic("negative memdb ref")
}
}
// Get latest sequence number.
func (d *DB) getSeq() uint64 {
return atomic.LoadUint64(&d.seq)
func (db *DB) getSeq() uint64 {
return atomic.LoadUint64(&db.seq)
}
// Atomically adds delta to seq.
func (d *DB) addSeq(delta uint64) {
atomic.AddUint64(&d.seq, delta)
func (db *DB) addSeq(delta uint64) {
atomic.AddUint64(&db.seq, delta)
}
func (db *DB) sampleSeek(ikey iKey) {
v := db.s.version()
if v.sampleSeek(ikey) {
// Trigger table compaction.
db.compSendTrigger(db.tcompCmdC)
}
v.release()
}
func (db *DB) mpoolPut(mem *memdb.DB) {
defer func() {
recover()
}()
select {
case db.memPool <- mem:
default:
}
}
func (db *DB) mpoolGet() *memdb.DB {
select {
case mem := <-db.memPool:
return mem
default:
return nil
}
}
func (db *DB) mpoolDrain() {
ticker := time.NewTicker(30 * time.Second)
for {
select {
case <-ticker.C:
select {
case <-db.memPool:
default:
}
case _, _ = <-db.closeC:
close(db.memPool)
return
}
}
}
// Create new memdb and froze the old one; need external synchronization.
// newMem only called synchronously by the writer.
func (d *DB) newMem(n int) (mem *memdb.DB, err error) {
s := d.s
num := s.allocFileNum()
file := s.getJournalFile(num)
func (db *DB) newMem(n int) (mem *memDB, err error) {
num := db.s.allocFileNum()
file := db.s.getJournalFile(num)
w, err := file.Create()
if err != nil {
s.reuseFileNum(num)
db.s.reuseFileNum(num)
return
}
d.memMu.Lock()
if d.journal == nil {
d.journal = journal.NewWriter(w)
} else {
d.journal.Reset(w)
d.journalWriter.Close()
d.frozenJournalFile = d.journalFile
db.memMu.Lock()
defer db.memMu.Unlock()
if db.frozenMem != nil {
panic("still has frozen mem")
}
d.journalWriter = w
d.journalFile = file
d.frozenMem = d.mem
d.mem = memdb.New(s.icmp, maxInt(d.s.o.GetWriteBuffer(), n))
mem = d.mem
// The seq only incremented by the writer.
d.frozenSeq = d.seq
d.memMu.Unlock()
if db.journal == nil {
db.journal = journal.NewWriter(w)
} else {
db.journal.Reset(w)
db.journalWriter.Close()
db.frozenJournalFile = db.journalFile
}
db.journalWriter = w
db.journalFile = file
db.frozenMem = db.mem
mdb := db.mpoolGet()
if mdb == nil || mdb.Capacity() < n {
mdb = memdb.New(db.s.icmp, maxInt(db.s.o.GetWriteBuffer(), n))
}
mem = &memDB{
db: db,
mdb: mdb,
ref: 2,
}
db.mem = mem
// The seq only incremented by the writer. And whoever called newMem
// should hold write lock, so no need additional synchronization here.
db.frozenSeq = db.seq
return
}
// Get all memdbs.
func (d *DB) getMems() (e *memdb.DB, f *memdb.DB) {
d.memMu.RLock()
defer d.memMu.RUnlock()
return d.mem, d.frozenMem
func (db *DB) getMems() (e, f *memDB) {
db.memMu.RLock()
defer db.memMu.RUnlock()
if db.mem == nil {
panic("nil effective mem")
}
db.mem.incref()
if db.frozenMem != nil {
db.frozenMem.incref()
}
return db.mem, db.frozenMem
}
// Get frozen memdb.
func (d *DB) getEffectiveMem() *memdb.DB {
d.memMu.RLock()
defer d.memMu.RUnlock()
return d.mem
func (db *DB) getEffectiveMem() *memDB {
db.memMu.RLock()
defer db.memMu.RUnlock()
if db.mem == nil {
panic("nil effective mem")
}
db.mem.incref()
return db.mem
}
// Check whether we has frozen memdb.
func (d *DB) hasFrozenMem() bool {
d.memMu.RLock()
defer d.memMu.RUnlock()
return d.frozenMem != nil
func (db *DB) hasFrozenMem() bool {
db.memMu.RLock()
defer db.memMu.RUnlock()
return db.frozenMem != nil
}
// Get frozen memdb.
func (d *DB) getFrozenMem() *memdb.DB {
d.memMu.RLock()
defer d.memMu.RUnlock()
return d.frozenMem
func (db *DB) getFrozenMem() *memDB {
db.memMu.RLock()
defer db.memMu.RUnlock()
if db.frozenMem != nil {
db.frozenMem.incref()
}
return db.frozenMem
}
// Drop frozen memdb; assume that frozen memdb isn't nil.
func (d *DB) dropFrozenMem() {
d.memMu.Lock()
if err := d.frozenJournalFile.Remove(); err != nil {
d.s.logf("journal@remove removing @%d %q", d.frozenJournalFile.Num(), err)
func (db *DB) dropFrozenMem() {
db.memMu.Lock()
if err := db.frozenJournalFile.Remove(); err != nil {
db.logf("journal@remove removing @%d %q", db.frozenJournalFile.Num(), err)
} else {
d.s.logf("journal@remove removed @%d", d.frozenJournalFile.Num())
db.logf("journal@remove removed @%d", db.frozenJournalFile.Num())
}
d.frozenJournalFile = nil
d.frozenMem = nil
d.memMu.Unlock()
db.frozenJournalFile = nil
db.frozenMem.decref()
db.frozenMem = nil
db.memMu.Unlock()
}
// Set closed flag; return true if not already closed.
func (d *DB) setClosed() bool {
return atomic.CompareAndSwapUint32(&d.closed, 0, 1)
func (db *DB) setClosed() bool {
return atomic.CompareAndSwapUint32(&db.closed, 0, 1)
}
// Check whether DB was closed.
func (d *DB) isClosed() bool {
return atomic.LoadUint32(&d.closed) != 0
func (db *DB) isClosed() bool {
return atomic.LoadUint32(&db.closed) != 0
}
// Check read ok status.
func (d *DB) ok() error {
if d.isClosed() {
func (db *DB) ok() error {
if db.isClosed() {
return ErrClosed
}
return nil

957
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go generated vendored
View File

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More