From 34aa67efc6c79e1d40e51bf0f0f3a9e019998bcd Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 21 Jul 2014 16:21:20 +0200 Subject: [PATCH] Sync script for V's serpent impl. Synced with V's serpent. --- alethzero/MainWin.cpp | 2 +- libpyserpent/CMakeLists.txt | 3 - libpyserpent/pyserpent.cpp | 259 ++++++++++++++++++++---------------- libserpent/compiler.cpp | 3 +- libserpent/funcs.cpp | 19 +-- libserpent/funcs.h | 7 +- libserpent/lllparser.cpp | 2 +- libserpent/parser.cpp | 17 ++- libserpent/rewriter.cpp | 9 +- libserpent/tokenize.cpp | 5 +- libserpent/tokenize.h | 4 +- libserpent/util.cpp | 2 +- pullSerpent.sh | 11 ++ sc/cmdline.cpp | 13 +- 14 files changed, 185 insertions(+), 171 deletions(-) create mode 100755 pullSerpent.sh diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index 3488b4f50..810fea580 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -1119,7 +1119,7 @@ void Main::on_data_textChanged() { try { - m_data = compile(src); + m_data = eth::asBytes(::compile(src)); for (auto& i: errors) i = "(LLL " + i + ")"; } diff --git a/libpyserpent/CMakeLists.txt b/libpyserpent/CMakeLists.txt index 8c986c77a..6b483ece2 100644 --- a/libpyserpent/CMakeLists.txt +++ b/libpyserpent/CMakeLists.txt @@ -29,16 +29,13 @@ if("${TARGET_PLATFORM}" STREQUAL "w64") target_link_libraries(${EXECUTABLE} shlwapi) elseif (APPLE) # Latest mavericks boost libraries only come with -mt - target_link_libraries(${EXECUTABLE} boost_python-mt) target_link_libraries(${EXECUTABLE} boost_thread-mt) find_package(Threads REQUIRED) target_link_libraries(${EXECUTABLE} ${CMAKE_THREAD_LIBS_INIT}) elseif (UNIX) - target_link_libraries(${EXECUTABLE} ${Boost_PYTHON_LIBRARY}) target_link_libraries(${EXECUTABLE} ${Boost_THREAD_LIBRARY}) target_link_libraries(${EXECUTABLE} ${CMAKE_THREAD_LIBS_INIT}) else () - target_link_libraries(${EXECUTABLE} boost_python) target_link_libraries(${EXECUTABLE} boost_thread) find_package(Threads REQUIRED) target_link_libraries(${EXECUTABLE} ${CMAKE_THREAD_LIBS_INIT}) diff --git a/libpyserpent/pyserpent.cpp b/libpyserpent/pyserpent.cpp index 93d9c8917..6caa7033a 100644 --- a/libpyserpent/pyserpent.cpp +++ b/libpyserpent/pyserpent.cpp @@ -1,132 +1,155 @@ -#include -#include #include +#include + +#include +#include +#include #include -// Provide a python wrapper for the C++ functions +#define PYMETHOD(name, FROM, method, TO) \ + static PyObject * name(PyObject *self, PyObject *args) { \ + FROM(med) \ + return TO(method(med)); \ + } -using namespace boost::python; +#define FROMSTR(v) \ + const char *command; \ + int len; \ + if (!PyArg_ParseTuple(args, "s#", &command, &len)) \ + return NULL; \ + std::string v = std::string(command, len); \ -//std::vector to python list converter -//http://stackoverflow.com/questions/5314319/how-to-export-stdvector -template -struct VecToList -{ - static PyObject* convert(const std::vector& vec) - { - boost::python::list* l = new boost::python::list(); - for(size_t i = 0; i < vec.size(); i++) - (*l).append(vec[i]); - - return l->ptr(); - } -}; +#define FROMNODE(v) \ + PyObject *node; \ + if (!PyArg_ParseTuple(args, "O", &node)) \ + return NULL; \ + Node v = cppifyNode(node); -// python list to std::vector converter -//http://code.activestate.com/lists/python-cplusplus-sig/16463/ -template -struct Vector_from_python_list -{ +#define FROMLIST(v) \ + PyObject *node; \ + if (!PyArg_ParseTuple(args, "O", &node)) \ + return NULL; \ + std::vector v = cppifyNodeList(node); - Vector_from_python_list() - { - using namespace boost::python; - using namespace boost::python::converter; - registry::push_back(&Vector_from_python_list::convertible, - &Vector_from_python_list::construct, - type_id ->()); +// Convert metadata into python wrapper form [file, ln, ch] +PyObject* pyifyMetadata(Metadata m) { + PyObject* a = PyList_New(0); + PyList_Append(a, Py_BuildValue("s#", m.file.c_str(), m.file.length())); + PyList_Append(a, Py_BuildValue("i", m.ln)); + PyList_Append(a, Py_BuildValue("i", m.ch)); + return a; +} +// Convert node into python wrapper form +// [token=0/astnode=1, val, metadata, args] +PyObject* pyifyNode(Node n) { + PyObject* a = PyList_New(0); + PyList_Append(a, Py_BuildValue("i", n.type == ASTNODE)); + PyList_Append(a, Py_BuildValue("s#", n.val.c_str(), n.val.length())); + PyList_Append(a, pyifyMetadata(n.metadata)); + for (unsigned i = 0; i < n.args.size(); i++) + PyList_Append(a, pyifyNode(n.args[i])); + return a; +} + +// Convert string into python wrapper form +PyObject* pyifyString(std::string s) { + return Py_BuildValue("s#", s.c_str(), s.length()); +} + +// Convert list of nodes into python wrapper form +PyObject* pyifyNodeList(std::vector n) { + PyObject* a = PyList_New(0); + for (unsigned i = 0; i < n.size(); i++) + PyList_Append(a, pyifyNode(n[i])); + return a; +} + +// Convert pyobject int into normal form +int cppifyInt(PyObject* o) { + int out; + if (!PyArg_Parse(o, "i", &out)) + err("Argument should be integer", Metadata()); + return out; +} + +// Convert pyobject string into normal form +std::string cppifyString(PyObject* o) { + const char *command; + if (!PyArg_Parse(o, "s", &command)) + err("Argument should be string", Metadata()); + return std::string(command); +} + +// Convert metadata from python wrapper form +Metadata cppifyMetadata(PyObject* o) { + std::string file = cppifyString(PyList_GetItem(o, 0)); + int ln = cppifyInt(PyList_GetItem(o, 1)); + int ch = cppifyInt(PyList_GetItem(o, 2)); + return Metadata(file, ln, ch); +} + +// Convert node from python wrapper form +Node cppifyNode(PyObject* o) { + Node n; + int isAstNode = cppifyInt(PyList_GetItem(o, 0)); + n.type = isAstNode ? ASTNODE : TOKEN; + n.val = cppifyString(PyList_GetItem(o, 1)); + n.metadata = cppifyMetadata(PyList_GetItem(o, 2)); + std::vector args; + for (int i = 3; i < PyList_Size(o); i++) { + args.push_back(cppifyNode(PyList_GetItem(o, i))); } - - // Determine if obj_ptr can be converted in a std::vector - static void* convertible(PyObject* obj_ptr) - { - if (!PyList_Check(obj_ptr)){ - return 0; - } - return obj_ptr; - } - - // Convert obj_ptr into a std::vector - static void construct( - PyObject* obj_ptr, - boost::python::converter::rvalue_from_python_stage1_data* data) - { - using namespace boost::python; - // Extract the character data from the python string - // const char* value = PyString_AsString(obj_ptr); - list l(handle<>(borrowed(obj_ptr))); - - // // Verify that obj_ptr is a string (should be ensured by convertible()) - // assert(value); - - // Grab pointer to memory into which to construct the new std::vector - void* storage = ( - (boost::python::converter::rvalue_from_python_storage ->*) - - data)->storage.bytes; - - // in-place construct the new std::vector using the character data - // extraced from the python object - std::vector& v = *(new (storage) std::vector()); - - // populate the vector from list contains !!! - int le = len(l); - v.resize(le); - for(int i = 0;i!=le;++i){ - v[i] = extract(l[i]); - } - - // Stash the memory chunk pointer for later use by boost.python - data->convertible = storage; - } -}; + n.args = args; + return n; +} -std::string printMetadata(Metadata m) { - return "["+m.file+" "+intToDecimal(m.ln)+" "+intToDecimal(m.ch)+"]"; +//Convert list of nodes into normal form +std::vector cppifyNodeList(PyObject* o) { + std::vector out; + for (int i = 0; i < PyList_Size(o); i++) { + out.push_back(cppifyNode(PyList_GetItem(o,i))); + } + return out; } -BOOST_PYTHON_FUNCTION_OVERLOADS(tokenize_overloads, tokenize, 1, 2); -BOOST_PYTHON_FUNCTION_OVERLOADS(printast_overloads, printAST, 1, 2); -BOOST_PYTHON_FUNCTION_OVERLOADS(parselll_overloads, parseLLL, 1, 2); -//BOOST_PYTHON_FUNCTION_OVERLOADS(metadata_overloads, Metadata, 0, 3); -BOOST_PYTHON_MODULE(pyserpent) +PYMETHOD(ps_compile, FROMSTR, compile, pyifyString) +PYMETHOD(ps_compile_to_lll, FROMSTR, compileToLLL, pyifyNode) +PYMETHOD(ps_compile_lll, FROMNODE, compileLLL, pyifyString) +PYMETHOD(ps_parse, FROMSTR, parseSerpent, pyifyNode) +PYMETHOD(ps_rewrite, FROMNODE, rewrite, pyifyNode) +PYMETHOD(ps_pretty_compile, FROMSTR, prettyCompile, pyifyNodeList) +PYMETHOD(ps_pretty_compile_lll, FROMNODE, prettyCompileLLL, pyifyNodeList) +PYMETHOD(ps_serialize, FROMLIST, serialize, pyifyString) +PYMETHOD(ps_deserialize, FROMSTR, deserialize, pyifyNodeList) +PYMETHOD(ps_parse_lll, FROMSTR, parseLLL, pyifyNode) + + +static PyMethodDef PyextMethods[] = { + {"compile", ps_compile, METH_VARARGS, + "Compile code."}, + {"compile_to_lll", ps_parse, METH_VARARGS, + "Compile code to LLL."}, + {"compile_lll", ps_compile_lll, METH_VARARGS, + "Compile LLL to EVM."}, + {"parse", ps_parse, METH_VARARGS, + "Parse serpent"}, + {"rewrite", ps_rewrite, METH_VARARGS, + "Rewrite parsed serpent to LLL"}, + {"pretty_compile", ps_pretty_compile, METH_VARARGS, + "Compile to EVM opcodes"}, + {"pretty_compile_lll", ps_pretty_compile_lll, METH_VARARGS, + "Compile LLL to EVM opcodes"}, + {"serialize", ps_serialize, METH_VARARGS, + "Convert EVM opcodes to bin"}, + {"deserialize", ps_deserialize, METH_VARARGS, + "Convert EVM bin to opcodes"}, + {"parse_lll", ps_parse_lll, METH_VARARGS, + "Parse LLL"}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +PyMODINIT_FUNC initpyext(void) { - def("tokenize", tokenize, tokenize_overloads()); - def("parse", parseSerpent); - def("parseLLL", parseLLL, parselll_overloads()); - def("rewrite", rewrite); - def("compile_to_lll", compileToLLL); - def("encode_datalist", encodeDatalist); - def("decode_datalist", decodeDatalist); - def("compile_lll", compileLLL); - def("assemble", assemble); - def("deserialize", deserialize); - def("dereference", dereference); - def("flatten", flatten); - def("serialize", serialize); - def("compile", compile); - def("pretty_compile", prettyCompile); - def("pretty_assemble", prettyAssemble); - //class_("Node",init<>()) - to_python_converter >, - VecToList >(); - to_python_converter >, - VecToList >(); - Vector_from_python_list(); - Vector_from_python_list(); - class_("Metadata",init<>()) - .def(init()) - .def("__str__", printMetadata) - .def("__repr__", printMetadata) - ; - class_("Node",init<>()) - .def(init<>()) - .def("__str__", printAST, printast_overloads()) - .def("__repr__", printAST, printast_overloads()) - ; - //class_("Vector",init<>()) - // .def(init<>()); + PyObject *m = Py_InitModule( "pyext", PyextMethods ); } diff --git a/libserpent/compiler.cpp b/libserpent/compiler.cpp index f83fa5481..959d2993b 100644 --- a/libserpent/compiler.cpp +++ b/libserpent/compiler.cpp @@ -86,8 +86,7 @@ programData opcodeify(Node node, programAux aux=Aux()) { Node code = astnode("____CODE", o, m); Node nodelist[] = { token("$begincode"+symb+".endcode"+symb, m), token("DUP", m), - sub.code, - token("$begincode"+symb, m), token("CODECOPY", m), + token("$begincode"+symb, m), sub.code, token("CODECOPY", m), token("$endcode"+symb, m), token("JUMP", m), token("~begincode"+symb, m), code, token("~endcode"+symb, m) }; diff --git a/libserpent/funcs.cpp b/libserpent/funcs.cpp index 3a5573f23..e80fcff06 100644 --- a/libserpent/funcs.cpp +++ b/libserpent/funcs.cpp @@ -9,28 +9,15 @@ #include "compiler.h" #include "rewriter.h" #include "tokenize.h" -#include -#include Node compileToLLL(std::string input) { return rewrite(parseSerpent(input)); } -std::vector compile(std::string input) { - return eth::compileLLL(printSimple(compileToLLL(input))); +std::string compile(std::string input) { + return compileLLL(compileToLLL(input)); } std::vector prettyCompile(std::string input) { - return deserialize(bytesToString( - eth::compileLLL(printSimple(compileToLLL(input))))); -} - -std::string bytesToString(std::vector input) { - std::string o; - for (unsigned i = 0; i < input.size(); i++) o += (char)input[i]; - return o; -} - -std::string bytesToHex(std::vector input) { - return binToHex(bytesToString(input)); + return prettyCompileLLL(compileToLLL(input)); } diff --git a/libserpent/funcs.h b/libserpent/funcs.h index c33a38226..0f3355ec8 100644 --- a/libserpent/funcs.h +++ b/libserpent/funcs.h @@ -1,5 +1,4 @@ #include -#include #include #include #include "bignum.h" @@ -25,10 +24,6 @@ Node compileToLLL(std::string input); -std::vector compile(std::string input); +std::string compile(std::string input); std::vector prettyCompile(std::string input); - -std::string bytesToString(std::vector input); - -std::string bytesToHex(std::vector input); diff --git a/libserpent/lllparser.cpp b/libserpent/lllparser.cpp index 812ef4a48..ad4fbd52d 100644 --- a/libserpent/lllparser.cpp +++ b/libserpent/lllparser.cpp @@ -66,5 +66,5 @@ Node parseLLL(std::string s, bool allowFileRead) { file = s; input = get_file_contents(s); } - return parseLLLTokenStream(tokenize(s, Metadata(file, 0, 0))); + return parseLLLTokenStream(tokenize(s, Metadata(file, 0, 0), true)); } diff --git a/libserpent/parser.cpp b/libserpent/parser.cpp index 74b7efc26..38fdca6ed 100644 --- a/libserpent/parser.cpp +++ b/libserpent/parser.cpp @@ -18,6 +18,7 @@ int precedence(Node tok) { else if (v=="&" || v=="|" || v=="xor" || v=="==" || v == "!=") return 5; else if (v=="&&" || v=="and") return 6; else if (v=="||" || v=="or") return 7; + else if (v==":") return 8; else if (v=="=") return 10; else if (v=="+=" || v=="-=" || v=="*=" || v=="/=" || v=="%=") return 10; else if (v=="@/=" || v=="@%=") return 10; @@ -28,10 +29,9 @@ int precedence(Node tok) { int toktype(Node tok) { if (tok.type == ASTNODE) return COMPOUND; std::string v = tok.val; - if (v == "(" || v == "[") return LPAREN; - else if (v == ")" || v == "]") return RPAREN; + if (v == "(" || v == "[" || v == "{") return LPAREN; + else if (v == ")" || v == "]" || v == "}") return RPAREN; else if (v == ",") return COMMA; - else if (v == ":") return COLON; else if (v == "!" || v == "not") return UNARY_OP; else if (precedence(tok) >= 0) return BINARY_OP; if (tok.val[0] != '"' && tok.val[0] != '\'') { @@ -54,7 +54,7 @@ std::vector shuntingYard(std::vector tokens) { std::vector oq; std::vector stack; Node prev, tok; - int prevtyp, toktyp = 0; + int prevtyp = 0, toktyp = 0; while (iq.size()) { prev = tok; @@ -100,13 +100,12 @@ std::vector shuntingYard(std::vector tokens) { } stack.push_back(tok); } - // Comma and colon mean finish evaluating the argument - else if (toktyp == COMMA || toktyp == COLON) { + // Comma means finish evaluating the argument + else if (toktyp == COMMA) { while (stack.size() && toktype(stack.back()) != LPAREN) { oq.push_back(stack.back()); stack.pop_back(); } - if (toktyp == COLON) oq.push_back(tok); } } while (stack.size()) { @@ -151,10 +150,10 @@ Node treefy(std::vector stream) { else if (typ == RPAREN) { std::vector args; while (1) { + if (toktype(oq.back()) == LPAREN) break; args.push_back(oq.back()); oq.pop_back(); if (!oq.size()) err("Bracket without matching", tok.metadata); - if (toktype(oq.back()) == LPAREN) break; } oq.pop_back(); args.push_back(oq.back()); @@ -180,7 +179,7 @@ Node treefy(std::vector stream) { // into 2 ( id 3 5 * ) +, effectively putting "id" as a dummy // function where the algo was expecting a function to call the // thing inside the brackets. This reverses that step - if (fun == "id" && args2.size()) { + if (fun == "id" && args2.size() == 1) { oq.push_back(args2[0]); } else { diff --git a/libserpent/rewriter.cpp b/libserpent/rewriter.cpp index 0192607bd..00712b871 100644 --- a/libserpent/rewriter.cpp +++ b/libserpent/rewriter.cpp @@ -236,7 +236,7 @@ std::string macros[][2] = { { "tx.gasprice", "(gasprice)" }, { "tx.origin", "(origin)" }, { "tx.gas", "(gas)" }, - { "contract.balance", "(balance)" }, + { "contract.balance", "(balance (address))" }, { "contract.address", "(address)" }, { "block.prevhash", "(prevhash)" }, { "block.coinbase", "(coinbase)" }, @@ -271,6 +271,7 @@ std::string synonyms[][2] = { { ">", "sgt" }, { "=", "set" }, { "==", "eq" }, + { ":", "kv" }, { "---END---", "" } //Keep this line at the end of the list }; @@ -350,7 +351,7 @@ Node array_lit_transform(Node node) { o2.push_back(astnode("alloc", o1, node.metadata)); std::vector o3; o3.push_back(astnode("set", o2, node.metadata)); - for (unsigned i = 0; i < node.args.size(); i++) { + for (unsigned i = 0; i < node.args.size(); i++) { // (mstore (add (get symb) i*32) v) std::vector o5; o5.push_back(token(symb, node.metadata)); @@ -404,9 +405,9 @@ Node apply_rules(Node node) { // Array_lit special instruction if (node.val == "array_lit") node = array_lit_transform(node); - if (node.type == ASTNODE && node.val != "ref" && node.val != "get") { + if (node.type == ASTNODE) { unsigned i = 0; - if (node.val == "set") { + if (node.val == "set" || node.val == "ref" || node.val == "get") { node.args[0].val = "'" + node.args[0].val; i = 1; } diff --git a/libserpent/tokenize.cpp b/libserpent/tokenize.cpp index 0fc3fb2fa..fc33d5dad 100644 --- a/libserpent/tokenize.cpp +++ b/libserpent/tokenize.cpp @@ -22,7 +22,7 @@ int chartype(char c) { } // "y = f(45,124)/3" -> [ "y", "f", "(", "45", ",", "124", ")", "/", "3"] -std::vector tokenize(std::string inp, Metadata metadata) { +std::vector tokenize(std::string inp, Metadata metadata, bool lispMode) { int curtype = SPACE; unsigned pos = 0; int lastNewline = 0; @@ -33,6 +33,9 @@ std::vector tokenize(std::string inp, Metadata metadata) { inp += " "; while (pos < inp.length()) { int headtype = chartype(inp[pos]); + if (lispMode) { + if (inp[pos] == '\'') headtype = ALPHANUM; + } // Are we inside a quote? if (curtype == SQUOTE || curtype == DQUOTE) { // Close quote diff --git a/libserpent/tokenize.h b/libserpent/tokenize.h index 7ded48cc5..04a42f3c6 100644 --- a/libserpent/tokenize.h +++ b/libserpent/tokenize.h @@ -9,6 +9,8 @@ int chartype(char c); -std::vector tokenize(std::string inp, Metadata meta=Metadata()); +std::vector tokenize(std::string inp, + Metadata meta=Metadata(), + bool lispMode=false); #endif diff --git a/libserpent/util.cpp b/libserpent/util.cpp index 512e53317..6ca39de9d 100644 --- a/libserpent/util.cpp +++ b/libserpent/util.cpp @@ -205,7 +205,7 @@ std::string get_file_contents(std::string filename) { std::string contents; in.seekg(0, std::ios::end); - contents.resize((size_t)in.tellg()); + contents.resize(in.tellg()); in.seekg(0, std::ios::beg); in.read(&contents[0], contents.size()); in.close(); diff --git a/pullSerpent.sh b/pullSerpent.sh new file mode 100755 index 000000000..eb0664e90 --- /dev/null +++ b/pullSerpent.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +opwd="$PWD" +cd ../serpent +git pull +cp bignum.* compiler.* funcs.* lllparser.* opcodes.h parser.* rewriter.* tokenize.* util.* ../cpp-ethereum/libserpent/ +cp cmdline.* "$opwd/sc/" +cp pyserpent.* "$opwd/libpyserpent/" +cd "$opwd" +perl -i -p -e 's:include "(.*)":include :gc' sc/* libpyserpent/* + diff --git a/sc/cmdline.cpp b/sc/cmdline.cpp index e8a0a130c..71d9928b2 100644 --- a/sc/cmdline.cpp +++ b/sc/cmdline.cpp @@ -3,14 +3,12 @@ #include #include #include -#include -#include int main(int argv, char** argc) { if (argv == 1) { std::cerr << "Must provide a command and arguments! Try parse, rewrite, compile, assemble\n"; return 0; - } + } std::string flag = ""; std::string command = argc[1]; std::string input; @@ -47,7 +45,7 @@ int main(int argv, char** argc) { std::cout << printAST(buildFragmentTree(parseLLL(input, true))) << "\n"; } else if (command == "compile_lll") { - std::cout << bytesToHex(eth::compileLLL(input)) << "\n"; + std::cout << binToHex(compileLLL(parseLLL(input, true))) << "\n"; } else if (command == "dereference") { std::cout << printAST(dereference(parseLLL(input, true)), haveSec) <<"\n"; @@ -56,11 +54,10 @@ int main(int argv, char** argc) { std::cout << printTokens(prettyAssemble(parseLLL(input, true))) <<"\n"; } else if (command == "pretty_compile_lll") { - std::cout << printTokens(deserialize(bytesToString( - eth::compileLLL(input)))) << "\n"; + std::cout << printTokens(prettyCompileLLL(parseLLL(input, true))) << "\n"; } else if (command == "pretty_compile") { - std::cout << printTokens(deserialize(bytesToString(compile(input)))) << "\n"; + std::cout << printTokens(prettyCompile(input)) << "\n"; } else if (command == "assemble") { std::cout << assemble(parseLLL(input, true)) << "\n"; @@ -75,7 +72,7 @@ int main(int argv, char** argc) { std::cout << printTokens(deserialize(hexToBin(input))) << "\n"; } else if (command == "compile") { - std::cout << bytesToHex(compile(input)) << "\n"; + std::cout << binToHex(compile(input)) << "\n"; } else if (command == "encode_datalist") { std::vector tokens = tokenize(input);