Merge remote-tracking branch 'upstream/develop' into addTests

Conflicts:
	test/ttTransactionTestFiller.json

CMakeLists.txt

@@ -160,7 +160,6 @@ if (EVMJIT)
 endif()
 
 add_subdirectory(libdevcore)
-add_subdirectory(rlp)
 add_subdirectory(libevmcore)
 add_subdirectory(liblll)
 
@@ -178,6 +177,7 @@ endif()
 
 if (JSONRPC)
 	add_subdirectory(libweb3jsonrpc)
+	add_subdirectory(ethrpctest)
 endif()
 
 add_subdirectory(secp256k1)
@@ -195,10 +195,13 @@ add_subdirectory(libevm)
 add_subdirectory(libethereum)
 
 add_subdirectory(libwebthree)
+add_subdirectory(libtestutils)
 add_subdirectory(test)
 
 if (NOT JUSTTESTS)
 
+	add_subdirectory(rlp)
+	add_subdirectory(abi)
 	add_subdirectory(eth)
 
 	if("x${CMAKE_BUILD_TYPE}" STREQUAL "xDebug")
@@ -226,9 +229,6 @@ if (NOT JUSTTESTS)
 
 endif()
 
-enable_testing()
-add_test(NAME alltests WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test COMMAND testeth)
-
 #unset(TARGET_PLATFORM CACHE)
 
 if (WIN32)

CodingStandards.txt

@@ -13,7 +13,7 @@ c. Don't use braces for condition-body one-liners.
 d. Never place condition bodies on same line as condition.
 e. Space between first paren and keyword, but *not* following first paren or preceeding final paren.
 f. No spaces when fewer than intra-expression three parens together; when three or more, space according to clarity.
-g. No spaces for subscripting.
+g. No spaces for subscripting or unary operators.
 h. No space before ':' but one after it, except in the ternary operator: one on both sides.
 i. Space all other operators.
 j. Braces, when used, always have their own lines and are at same indentation level as "parent" scope.

abi/CMakeLists.txt

@@ -0,0 +1,16 @@
+cmake_policy(SET CMP0015 NEW)
+set(CMAKE_AUTOMOC OFF)
+
+aux_source_directory(. SRC_LIST)
+
+include_directories(BEFORE ..)
+include_directories(${LEVELDB_INCLUDE_DIRS})
+
+set(EXECUTABLE abi)
+
+add_executable(${EXECUTABLE} ${SRC_LIST})
+
+target_link_libraries(${EXECUTABLE} ethereum)
+
+install( TARGETS ${EXECUTABLE} DESTINATION bin) 
+

abi/main.cpp

@@ -0,0 +1,769 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file main.cpp
+ * @author Gav Wood <i@gavwood.com>
+ * @date 2014
+ * RLP tool.
+ */
+#include <fstream>
+#include <iostream>
+#include <boost/regex.hpp>
+#include <boost/algorithm/string.hpp>
+#include "../test/JsonSpiritHeaders.h"
+#include <libdevcore/CommonIO.h>
+#include <libdevcore/RLP.h>
+#include <libdevcrypto/SHA3.h>
+#include <libethereum/Client.h>
+using namespace std;
+using namespace dev;
+namespace js = json_spirit;
+
+void help()
+{
+	cout
+		<< "Usage abi enc <method_name> (<arg1>, (<arg2>, ... ))" << endl
+		<< "      abi enc -a <abi.json> <method_name> (<arg1>, (<arg2>, ... ))" << endl
+		<< "      abi dec -a <abi.json> [ <signature> | <unique_method_name> ]" << endl
+		<< "Options:" << endl
+		<< "    -a,--abi-file <filename>  Specify the JSON ABI file." << endl
+		<< "    -h,--help  Print this help message and exit." << endl
+		<< "    -V,--version  Show the version and exit." << endl
+		<< "Input options:" << endl
+		<< "    -f,--format-prefix  Require all input formats to be prefixed e.g. 0x for hex, . for decimal, @ for binary." << endl
+		<< "    -F,--no-format-prefix  Require no input format to be prefixed." << endl
+		<< "    -t,--typing  Require all arguments to be typed e.g. b32: (bytes32), u64: (uint64), b[]: (byte[]), i: (int256)." << endl
+		<< "    -T,--no-typing  Require no arguments to be typed." << endl
+		<< "Output options:" << endl
+		<< "    -i,--index <n>  Output only the nth (counting from 0) return value." << endl
+		<< "    -d,--decimal  All data should be displayed as decimal." << endl
+		<< "    -x,--hex  Display all data as hex." << endl
+		<< "    -b,--binary  Display all data as binary." << endl
+		<< "    -p,--prefix  Prefix by a base identifier." << endl
+		;
+	exit(0);
+}
+
+void version()
+{
+	cout << "abi version " << dev::Version << endl;
+	exit(0);
+}
+
+enum class Mode
+{
+	Encode,
+	Decode
+};
+
+enum class Encoding
+{
+	Auto,
+	Decimal,
+	Hex,
+	Binary,
+};
+
+enum class Tristate
+{
+	False = false,
+	True = true,
+	Mu
+};
+
+enum class Format
+{
+	Binary,
+	Hex,
+	Decimal,
+	Open,
+	Close
+};
+
+struct InvalidUserString: public Exception {};
+struct InvalidFormat: public Exception {};
+
+enum class Base
+{
+	Unknown,
+	Bytes,
+	Address,
+	Int,
+	Uint,
+	Fixed
+};
+
+static const map<Base, string> s_bases =
+{
+	{ Base::Bytes, "bytes" },
+	{ Base::Address, "address" },
+	{ Base::Int, "int" },
+	{ Base::Uint, "uint" },
+	{ Base::Fixed, "fixed" }
+};
+
+struct EncodingPrefs
+{
+	Encoding e = Encoding::Auto;
+	bool prefix = true;
+};
+
+struct ABIType
+{
+	Base base = Base::Unknown;
+	unsigned size = 32;
+	unsigned ssize = 0;
+	vector<int> dims;
+	string name;
+	ABIType() = default;
+	ABIType(std::string const& _type, std::string const& _name):
+		name(_name)
+	{
+		string rest;
+		for (auto const& i: s_bases)
+			if (boost::algorithm::starts_with(_type, i.second))
+			{
+				base = i.first;
+				rest = _type.substr(i.second.size());
+			}
+		if (base == Base::Unknown)
+			throw InvalidFormat();
+		boost::regex r("(\\d*)(x(\\d+))?((\\[\\d*\\])*)");
+		boost::smatch res;
+		boost::regex_match(rest, res, r);
+		size = res[1].length() > 0 ? stoi(res[1]) : 0;
+		ssize = res[3].length() > 0 ? stoi(res[3]) : 0;
+		boost::regex r2("\\[(\\d*)\\](.*)");
+		for (rest = res[4]; boost::regex_match(rest, res, r2); rest = res[2])
+			dims.push_back(!res[1].length() ? -1 : stoi(res[1]));
+	}
+
+	ABIType(std::string const& _s)
+	{
+		if (_s.size() < 1)
+			return;
+		switch (_s[0])
+		{
+		case 'b': base = Base::Bytes; break;
+		case 'a': base = Base::Address; break;
+		case 'i': base = Base::Int; break;
+		case 'u': base = Base::Uint; break;
+		case 'f': base = Base::Fixed; break;
+		default: throw InvalidFormat();
+		}
+		if (_s.size() < 2)
+		{
+			if (base == Base::Fixed)
+				size = ssize = 16;
+			else if (base == Base::Address || base == Base::Bytes)
+				size = 0;
+			else
+				size = 32;
+			return;
+		}
+		strings d;
+		boost::algorithm::split(d, _s, boost::is_any_of("*"));
+		string s = d[0];
+		if (s.find_first_of('x') == string::npos)
+			size = stoi(s.substr(1));
+		else
+		{
+			size = stoi(s.substr(1, s.find_first_of('x') - 1));
+			ssize = stoi(s.substr(s.find_first_of('x') + 1));
+		}
+		for (unsigned i = 1; i < d.size(); ++i)
+			if (d[i].empty())
+				dims.push_back(-1);
+			else
+				dims.push_back(stoi(d[i]));
+	}
+
+	string canon() const
+	{
+		string ret;
+		switch (base)
+		{
+		case Base::Bytes: ret = "bytes" + (size > 0 ? toString(size) : ""); break;
+		case Base::Address: ret = "address"; break;
+		case Base::Int: ret = "int" + toString(size); break;
+		case Base::Uint: ret = "uint" + toString(size); break;
+		case Base::Fixed: ret = "fixed" + toString(size) + "x" + toString(ssize); break;
+		default: throw InvalidFormat();
+		}
+		for (int i: dims)
+			ret += "[" + ((i > -1) ? toString(i) : "") + "]";
+		return ret;
+	}
+
+	bool isBytes() const { return base == Base::Bytes && !size; }
+
+	string render(bytes const& _data, EncodingPrefs _e) const
+	{
+		if (base == Base::Uint || base == Base::Int)
+		{
+			if (_e.e == Encoding::Hex)
+				return (_e.prefix ? "0x" : "") + toHex(toCompactBigEndian(fromBigEndian<bigint>(bytesConstRef(&_data).cropped(32 - size / 8))));
+			else
+			{
+				bigint i = fromBigEndian<bigint>(bytesConstRef(&_data).cropped(32 - size / 8));
+				if (base == Base::Int && i > (bigint(1) << (size - 1)))
+					i -= (bigint(1) << size);
+				return toString(i);
+			}
+		}
+		else if (base == Base::Address)
+		{
+			Address a = Address(h256(_data), Address::AlignRight);
+			return _e.e == Encoding::Binary ? asString(a.asBytes()) : ((_e.prefix ? "0x" : "") + toString(a));
+		}
+		else if (isBytes())
+		{
+			return _e.e == Encoding::Binary ? asString(_data) : ((_e.prefix ? "0x" : "") + toHex(_data));
+		}
+		else if (base == Base::Bytes)
+		{
+			bytesConstRef b(&_data);
+			b = b.cropped(0, size);
+			return _e.e == Encoding::Binary ? asString(b) : ((_e.prefix ? "0x" : "") + toHex(b));
+		}
+		else
+			throw InvalidFormat();
+	}
+
+	bytes unrender(bytes const& _data, Format _f) const
+	{
+		if (isBytes())
+		{
+			auto ret = _data;
+			while (ret.size() % 32 != 0)
+				ret.push_back(0);
+			return ret;
+		}
+		else
+			return aligned(_data, _f, 32);
+	}
+
+	void noteHexInput(unsigned _nibbles) { if (base == Base::Unknown) { if (_nibbles == 40) base = Base::Address; else { base = Base::Bytes; size = _nibbles / 2; } } }
+	void noteBinaryInput() { if (base == Base::Unknown) { base = Base::Bytes; size = 32; } }
+	void noteDecimalInput() { if (base == Base::Unknown) { base = Base::Uint; size = 32; } }
+
+	bytes aligned(bytes const& _b, Format _f, unsigned _length) const
+	{
+		bytes ret = _b;
+		while (ret.size() < _length)
+			if (base == Base::Bytes || (base == Base::Unknown && _f == Format::Binary))
+				ret.push_back(0);
+			else
+				ret.insert(ret.begin(), 0);
+		while (ret.size() > _length)
+			if (base == Base::Bytes || (base == Base::Unknown && _f == Format::Binary))
+				ret.pop_back();
+			else
+				ret.erase(ret.begin());
+		return ret;
+	}
+};
+
+tuple<bytes, ABIType, Format> fromUser(std::string const& _arg, Tristate _prefix, Tristate _typing)
+{
+	ABIType type;
+	string val;
+	if (_typing == Tristate::True || (_typing == Tristate::Mu && _arg.find(':') != string::npos))
+	{
+		if (_arg.find(':') == string::npos)
+			throw InvalidUserString();
+		type = ABIType(_arg.substr(0, _arg.find(':')));
+		val = _arg.substr(_arg.find(':') + 1);
+	}
+	else
+		val = _arg;
+
+	if (_prefix != Tristate::False)
+	{
+		if (val.substr(0, 2) == "0x")
+		{
+			type.noteHexInput(val.size() - 2);
+			return make_tuple(fromHex(val), type, Format::Hex);
+		}
+		if (val.substr(0, 1) == "+")
+		{
+			type.noteDecimalInput();
+			return make_tuple(toCompactBigEndian(bigint(val.substr(1))), type, Format::Decimal);
+		}
+		if (val.substr(0, 1) == "'")
+		{
+			type.noteBinaryInput();
+			return make_tuple(asBytes(val.substr(1)), type, Format::Binary);
+		}
+		if (val == "[")
+			return make_tuple(bytes(), type, Format::Open);
+		if (val == "]")
+			return make_tuple(bytes(), type, Format::Close);
+	}
+	if (_prefix != Tristate::True)
+	{
+		if (val.find_first_not_of("0123456789") == string::npos)
+		{
+			type.noteDecimalInput();
+			return make_tuple(toCompactBigEndian(bigint(val)), type, Format::Decimal);
+		}
+		if (val.find_first_not_of("0123456789abcdefABCDEF") == string::npos)
+		{
+			type.noteHexInput(val.size());
+			return make_tuple(fromHex(val), type, Format::Hex);
+		}
+		if (val == "[")
+			return make_tuple(bytes(), type, Format::Open);
+		if (val == "]")
+			return make_tuple(bytes(), type, Format::Close);
+		type.noteBinaryInput();
+		return make_tuple(asBytes(val), type, Format::Binary);
+	}
+	throw InvalidUserString();
+}
+
+struct ExpectedAdditionalParameter: public Exception {};
+struct ExpectedOpen: public Exception {};
+struct ExpectedClose: public Exception {};
+
+struct ABIMethod
+{
+	string name;
+	vector<ABIType> ins;
+	vector<ABIType> outs;
+	bool isConstant = false;
+
+	// isolation *IS* documentation.
+
+	ABIMethod() = default;
+
+	ABIMethod(js::mObject _o)
+	{
+		name = _o["name"].get_str();
+		isConstant = _o["constant"].get_bool();
+		if (_o.count("inputs"))
+			for (auto const& i: _o["inputs"].get_array())
+			{
+				js::mObject a = i.get_obj();
+				ins.push_back(ABIType(a["type"].get_str(), a["name"].get_str()));
+			}
+		if (_o.count("outputs"))
+			for (auto const& i: _o["outputs"].get_array())
+			{
+				js::mObject a = i.get_obj();
+				outs.push_back(ABIType(a["type"].get_str(), a["name"].get_str()));
+			}
+	}
+
+	ABIMethod(string const& _name, vector<ABIType> const& _args)
+	{
+		name = _name;
+		ins = _args;
+	}
+
+	string sig() const
+	{
+		string methodArgs;
+		for (auto const& arg: ins)
+			methodArgs += (methodArgs.empty() ? "" : ",") + arg.canon();
+		return name + "(" + methodArgs + ")";
+	}
+	FixedHash<4> id() const { return FixedHash<4>(sha3(sig())); }
+
+	std::string solidityDeclaration() const
+	{
+		ostringstream ss;
+		ss << "function " << name << "(";
+		int f = 0;
+		for (ABIType const& i: ins)
+			ss << (f++ ? ", " : "") << i.canon() << " " << i.name;
+		ss << ") ";
+		if (isConstant)
+			ss << "constant ";
+		if (!outs.empty())
+		{
+			ss << "returns(";
+			f = 0;
+			for (ABIType const& i: outs)
+				ss << (f++ ? ", " : "") << i.canon() << " " << i.name;
+			ss << ")";
+		}
+		return ss.str();
+	}
+
+	bytes encode(vector<pair<bytes, Format>> const& _params) const
+	{
+		bytes ret = name.empty() ? bytes() : id().asBytes();
+		bytes suffix;
+
+		// int int[] int
+		// example: 42 [ 1 2 3 ] 69
+		// int[2][][3]
+		// example: [ [ [ 1 2 3 ] [ 4 5 6 ] ] [ ] ]
+
+		unsigned pi = 0;
+
+		for (ABIType const& a: ins)
+		{
+			if (pi >= _params.size())
+				throw ExpectedAdditionalParameter();
+			auto put = [&]() {
+				if (a.isBytes())
+					ret += h256(u256(_params[pi].first.size())).asBytes();
+				suffix += a.unrender(_params[pi].first, _params[pi].second);
+				pi++;
+				if (pi >= _params.size())
+					throw ExpectedAdditionalParameter();
+			};
+			function<void(vector<int>, unsigned)> putDim = [&](vector<int> addr, unsigned q) {
+				if (addr.size() == a.dims.size())
+					put();
+				else
+				{
+					if (_params[pi].second != Format::Open)
+						throw ExpectedOpen();
+					++pi;
+					int l = a.dims[addr.size()];
+					if (l == -1)
+					{
+						// read ahead in params and discover the arity.
+						unsigned depth = 0;
+						l = 0;
+						for (unsigned pi2 = pi; depth || _params[pi2].second != Format::Close;)
+						{
+							if (_params[pi2].second == Format::Open)
+								++depth;
+							if (_params[pi2].second == Format::Close)
+								--depth;
+							if (!depth)
+								++l;
+							if (++pi2 == _params.size())
+								throw ExpectedClose();
+						}
+						ret += h256(u256(l)).asBytes();
+					}
+					q *= l;
+					for (addr.push_back(0); addr.back() < l; ++addr.back())
+						putDim(addr, q);
+					if (_params[pi].second != Format::Close)
+						throw ExpectedClose();
+					++pi;
+				}
+			};
+			putDim(vector<int>(), 1);
+		}
+		return ret + suffix;
+	}
+	string decode(bytes const& _data, int _index, EncodingPrefs _ep)
+	{
+		stringstream out;
+		unsigned di = 0;
+		vector<unsigned> catDims;
+		for (ABIType const& a: outs)
+		{
+			auto put = [&]() {
+				if (a.isBytes())
+				{
+					catDims.push_back(fromBigEndian<unsigned>(bytesConstRef(&_data).cropped(di, 32)));
+					di += 32;
+				}
+			};
+			function<void(vector<int>, unsigned)> putDim = [&](vector<int> addr, unsigned q) {
+				if (addr.size() == a.dims.size())
+					put();
+				else
+				{
+					int l = a.dims[addr.size()];
+					if (l == -1)
+					{
+						l = fromBigEndian<unsigned>(bytesConstRef(&_data).cropped(di, 32));
+						catDims.push_back(l);
+						di += 32;
+					}
+					q *= l;
+					for (addr.push_back(0); addr.back() < l; ++addr.back())
+						putDim(addr, q);
+				}
+			};
+			putDim(vector<int>(), 1);
+		}
+		unsigned d = 0;
+		for (ABIType const& a: outs)
+		{
+			if (_index == -1 && out.tellp() > 0)
+				out << ", ";
+			auto put = [&]() {
+				if (a.isBytes())
+				{
+					out << a.render(bytesConstRef(&_data).cropped(di, catDims[d]).toBytes(), _ep);
+					di += ((catDims[d] + 31) / 32) * 32;
+					d++;
+				}
+				else
+				{
+					out << a.render(bytesConstRef(&_data).cropped(di, 32).toBytes(), _ep);
+					di += 32;
+				}
+			};
+			function<void(vector<int>)> putDim = [&](vector<int> addr) {
+				if (addr.size() == a.dims.size())
+					put();
+				else
+				{
+					out << "[";
+					addr.push_back(0);
+					int l = a.dims[addr.size() - 1];
+					if (l == -1)
+						l = catDims[d++];
+					for (addr.back() = 0; addr.back() < l; ++addr.back())
+					{
+						if (addr.back())
+							out << ", ";
+						putDim(addr);
+					}
+					out << "]";
+				}
+			};
+			putDim(vector<int>());
+		}
+		return out.str();
+	}
+};
+
+string canonSig(string const& _name, vector<ABIType> const& _args)
+{
+	try {
+		string methodArgs;
+		for (auto const& arg: _args)
+			methodArgs += (methodArgs.empty() ? "" : ",") + arg.canon();
+		return _name + "(" + methodArgs + ")";
+	}
+	catch (...) {
+		return string();
+	}
+}
+
+struct UnknownMethod: public Exception {};
+struct OverloadedMethod: public Exception {};
+
+class ABI
+{
+public:
+	ABI() = default;
+	ABI(std::string const& _json)
+	{
+		js::mValue v;
+		js::read_string(_json, v);
+		for (auto const& i: v.get_array())
+		{
+			js::mObject o = i.get_obj();
+			if (o["type"].get_str() != "function")
+				continue;
+			ABIMethod m(o);
+			m_methods[m.id()] = m;
+		}
+	}
+
+	ABIMethod method(string _nameOrSig, vector<ABIType> const& _args) const
+	{
+		auto id = FixedHash<4>(sha3(_nameOrSig));
+		if (!m_methods.count(id))
+			id = FixedHash<4>(sha3(canonSig(_nameOrSig, _args)));
+		if (!m_methods.count(id))
+			for (auto const& m: m_methods)
+				if (m.second.name == _nameOrSig)
+				{
+					if (m_methods.count(id))
+						throw OverloadedMethod();
+					id = m.first;
+				}
+		if (m_methods.count(id))
+			return m_methods.at(id);
+		throw UnknownMethod();
+	}
+
+	friend ostream& operator<<(ostream& _out, ABI const& _abi);
+
+private:
+	map<FixedHash<4>, ABIMethod> m_methods;
+};
+
+ostream& operator<<(ostream& _out, ABI const& _abi)
+{
+	_out << "contract {" << endl;
+	for (auto const& i: _abi.m_methods)
+		_out << "  " << i.second.solidityDeclaration() << "; // " << i.first.abridged() << endl;
+	_out << "}" << endl;
+	return _out;
+}
+
+void userOutput(ostream& _out, bytes const& _data, Encoding _e)
+{
+	switch (_e)
+	{
+	case Encoding::Binary:
+		_out.write((char const*)_data.data(), _data.size());
+		break;
+	default:
+		_out << toHex(_data) << endl;
+	}
+}
+
+template <unsigned n, class T> vector<typename std::remove_reference<decltype(get<n>(T()))>::type> retrieve(vector<T> const& _t)
+{
+	vector<typename std::remove_reference<decltype(get<n>(T()))>::type> ret;
+	for (T const& i: _t)
+		ret.push_back(get<n>(i));
+	return ret;
+}
+
+int main(int argc, char** argv)
+{
+	Encoding encoding = Encoding::Auto;
+	Mode mode = Mode::Encode;
+	string abiFile;
+	string method;
+	Tristate formatPrefix = Tristate::Mu;
+	Tristate typePrefix = Tristate::Mu;
+	EncodingPrefs prefs;
+	bool verbose = false;
+	int outputIndex = -1;
+	vector<pair<bytes, Format>> params;
+	vector<ABIType> args;
+	string incoming;
+
+	for (int i = 1; i < argc; ++i)
+	{
+		string arg = argv[i];
+		if (arg == "-h" || arg == "--help")
+			help();
+		else if (arg == "enc" && i == 1)
+			mode = Mode::Encode;
+		else if (arg == "dec" && i == 1)
+			mode = Mode::Decode;
+		else if ((arg == "-a" || arg == "--abi") && argc > i)
+			abiFile = argv[++i];
+		else if ((arg == "-i" || arg == "--index") && argc > i)
+			outputIndex = atoi(argv[++i]);
+		else if (arg == "-p" || arg == "--prefix")
+			prefs.prefix = true;
+		else if (arg == "-f" || arg == "--format-prefix")
+			formatPrefix = Tristate::True;
+		else if (arg == "-F" || arg == "--no-format-prefix")
+			formatPrefix = Tristate::False;
+		else if (arg == "-t" || arg == "--typing")
+			typePrefix = Tristate::True;
+		else if (arg == "-T" || arg == "--no-typing")
+			typePrefix = Tristate::False;
+		else if (arg == "-x" || arg == "--hex")
+			prefs.e = Encoding::Hex;
+		else if (arg == "-d" || arg == "--decimal" || arg == "--dec")
+			prefs.e = Encoding::Decimal;
+		else if (arg == "-b" || arg == "--binary" || arg == "--bin")
+			prefs.e = Encoding::Binary;
+		else if (arg == "-v" || arg == "--verbose")
+			verbose = true;
+		else if (arg == "-V" || arg == "--version")
+			version();
+		else if (method.empty())
+			method = arg;
+		else if (mode == Mode::Encode)
+		{
+			auto u = fromUser(arg, formatPrefix, typePrefix);
+			args.push_back(get<1>(u));
+			params.push_back(make_pair(get<0>(u), get<2>(u)));
+		}
+		else if (mode == Mode::Decode)
+			incoming += arg;
+	}
+
+	string abiData;
+	if (!abiFile.empty())
+		abiData = contentsString(abiFile);
+
+	if (mode == Mode::Encode)
+	{
+		ABIMethod m;
+		if (abiData.empty())
+			m = ABIMethod(method, args);
+		else
+		{
+			ABI abi(abiData);
+			if (verbose)
+				cerr << "ABI:" << endl << abi;
+			try {
+				m = abi.method(method, args);
+			}
+			catch (...)
+			{
+				cerr << "Unknown method in ABI." << endl;
+				exit(-1);
+			}
+		}
+		try {
+			userOutput(cout, m.encode(params), encoding);
+		}
+		catch (ExpectedAdditionalParameter const&)
+		{
+			cerr << "Expected additional parameter in input." << endl;
+			exit(-1);
+		}
+		catch (ExpectedOpen const&)
+		{
+			cerr << "Expected open-bracket '[' in input." << endl;
+			exit(-1);
+		}
+		catch (ExpectedClose const&)
+		{
+			cerr << "Expected close-bracket ']' in input." << endl;
+			exit(-1);
+		}
+	}
+	else if (mode == Mode::Decode)
+	{
+		if (abiData.empty())
+		{
+			cerr << "Please specify an ABI file." << endl;
+			exit(-1);
+		}
+		else
+		{
+			ABI abi(abiData);
+			ABIMethod m;
+			if (verbose)
+				cerr << "ABI:" << endl << abi;
+			try {
+				m = abi.method(method, args);
+			}
+			catch(...)
+			{
+				cerr << "Unknown method in ABI." << endl;
+				exit(-1);
+			}
+			string encoded;
+			if (incoming == "--" || incoming.empty())
+				for (int i = cin.get(); i != -1; i = cin.get())
+					encoded.push_back((char)i);
+			else
+			{
+				encoded = contentsString(incoming);
+			}
+			cout << m.decode(fromHex(boost::trim_copy(encoded)), outputIndex, prefs) << endl;
+		}
+	}
+
+	return 0;
+}

alethzero/CMakeLists.txt

@@ -19,6 +19,7 @@ find_package (Qt5WebEngine QUIET)
 find_package (Qt5WebEngineWidgets QUIET)
 
 qt5_wrap_ui(ui_Main.h Main.ui)
+qt5_wrap_ui(ui_Connect.h Connect.ui)
 qt5_wrap_ui(ui_Debugger.h Debugger.ui)
 qt5_wrap_ui(ui_Transact.h Transact.ui)
 
@@ -33,7 +34,7 @@ endif ()
 # eth_add_executable is defined in cmake/EthExecutableHelper.cmake
 eth_add_executable(${EXECUTABLE} 
 	ICON alethzero
-	UI_RESOURCES alethzero.icns Main.ui Debugger.ui Transact.ui
+	UI_RESOURCES alethzero.icns Main.ui Connect.ui Debugger.ui Transact.ui
 	WIN_RESOURCES alethzero.rc
 )
 

alethzero/Connect.cpp

@@ -0,0 +1,63 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file Connect.cpp
+ * @author Alex Leverington <nessence@gmail.com>
+ * @date 2015
+ */
+
+#include "Connect.h"
+
+#include <libp2p/Host.h>
+#include "ui_Connect.h"
+
+Connect::Connect(QWidget *parent) :
+	QDialog(parent),
+	ui(new Ui::Connect)
+{
+	ui->setupUi(this);
+}
+
+Connect::~Connect()
+{
+	delete ui;
+}
+
+void Connect::setEnvironment(QStringList const& _nodes)
+{
+	ui->host->addItems(_nodes);
+}
+
+void Connect::reset()
+{
+	ui->nodeId->clear();
+	ui->required->setChecked(false);
+}
+
+QString Connect::host()
+{
+	return ui->host->currentText();
+}
+
+QString Connect::nodeId()
+{
+	return ui->nodeId->text();
+}
+
+bool Connect::required()
+{
+	return ui->required->isChecked();
+}

alethzero/Connect.h

@@ -0,0 +1,55 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file Connect.h
+ * @author Alex Leverington <nessence@gmail.com>
+ * @date 2015
+ */
+
+#pragma once
+
+#include <QDialog>
+#include <QList>
+
+namespace Ui { class Connect; }
+namespace dev { namespace p2p { class Host; } }
+
+class Connect : public QDialog
+{
+	Q_OBJECT
+
+public:
+	explicit Connect(QWidget* _parent = 0);
+	~Connect();
+
+	/// Populate host chooser with default host entries.
+	void setEnvironment(QStringList const& _nodes);
+
+	/// Clear dialogue inputs.
+	void reset();
+
+	/// @returns the host string, as chosen or entered by the user. Assumed to be "hostOrIP:port" (:port is optional).
+	QString host();
+	
+	/// @returns the identity of the node, as entered by the user. Assumed to be a 64-character hex string.
+	QString nodeId();
+	
+	/// @returns true if Required is checked by the user, indicating that the host is a required Peer.
+	bool required();
+
+private:
+	Ui::Connect* ui;
+};

alethzero/Connect.ui

@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Connect</class>
+ <widget class="QDialog" name="Connect">
+  <property name="windowModality">
+   <enum>Qt::WindowModal</enum>
+  </property>
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>343</width>
+    <height>178</height>
+   </rect>
+  </property>
+  <property name="sizePolicy">
+   <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
+    <horstretch>0</horstretch>
+    <verstretch>0</verstretch>
+   </sizepolicy>
+  </property>
+  <property name="maximumSize">
+   <size>
+    <width>343</width>
+    <height>178</height>
+   </size>
+  </property>
+  <property name="windowTitle">
+   <string>Connect to Peer</string>
+  </property>
+  <property name="modal">
+   <bool>true</bool>
+  </property>
+  <layout class="QFormLayout" name="formLayout_2">
+   <item row="1" column="0">
+    <layout class="QFormLayout" name="formLayout">
+     <item row="1" column="0" colspan="2">
+      <widget class="QComboBox" name="host">
+       <property name="minimumSize">
+        <size>
+         <width>311</width>
+         <height>0</height>
+        </size>
+       </property>
+       <property name="editable">
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
+     <item row="2" column="0" colspan="2">
+      <widget class="QLineEdit" name="nodeId">
+       <property name="placeholderText">
+        <string>Node Id</string>
+       </property>
+      </widget>
+     </item>
+     <item row="3" column="0" colspan="2">
+      <widget class="QCheckBox" name="required">
+       <property name="text">
+        <string>Required (Always Connect to this Peer)</string>
+       </property>
+       <property name="tristate">
+        <bool>false</bool>
+       </property>
+      </widget>
+     </item>
+     <item row="4" column="0" colspan="2">
+      <widget class="QDialogButtonBox" name="buttonBox">
+       <property name="orientation">
+        <enum>Qt::Horizontal</enum>
+       </property>
+       <property name="standardButtons">
+        <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="0" column="0">
+    <widget class="QLabel" name="formLabel">
+     <property name="text">
+      <string>Enter a peer to which a connection may be made:</string>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>accepted()</signal>
+   <receiver>Connect</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>248</x>
+     <y>254</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>157</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>Connect</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>316</x>
+     <y>260</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>286</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

alethzero/Debugger.cpp

@@ -67,7 +67,8 @@ void Debugger::populate(dev::eth::Executive& _executive, dev::eth::Transaction c
 bool DebugSession::populate(dev::eth::Executive& _executive, dev::eth::Transaction const& _transaction)
 {
 	try {
-		if (_executive.setup(_transaction))
+		_executive.initialize(_transaction);
+		if (_executive.execute())
 			return false;
 	}
 	catch (...)

alethzero/Main.ui

@@ -118,7 +118,7 @@
      <x>0</x>
      <y>0</y>
      <width>1617</width>
-     <height>24</height>
+     <height>22</height>
     </rect>
    </property>
    <widget class="QMenu" name="menu_File">
@@ -134,8 +134,7 @@
     <addaction name="go"/>
     <addaction name="separator"/>
     <addaction name="upnp"/>
-    <addaction name="usePast"/>
+    <addaction name="dropPeers"/>
-    <addaction name="localNetworking"/>
     <addaction name="net"/>
     <addaction name="connect"/>
    </widget>
@@ -208,7 +207,7 @@
    <addaction name="menu_Debug"/>
   </widget>
   <widget class="QStatusBar" name="statusbar"/>
-  <widget class="QDockWidget" name="dockWidget_2">
+  <widget class="QDockWidget" name="dockWidget_accounts">
    <property name="features">
     <set>QDockWidget::DockWidgetFeatureMask</set>
    </property>
@@ -284,7 +283,27 @@
        </widget>
        <widget class="QWidget" name="layoutWidget">
         <layout class="QGridLayout" name="gridLayout_2">
-         <item row="1" column="2">
+         <item row="3" column="0">
+          <widget class="QLabel" name="label">
+           <property name="text">
+            <string>&amp;Listen on</string>
+           </property>
+           <property name="buddy">
+            <cstring>port</cstring>
+           </property>
+          </widget>
+         </item>
+         <item row="0" column="1" colspan="2">
+          <widget class="QSpinBox" name="idealPeers">
+           <property name="minimum">
+            <number>1</number>
+           </property>
+           <property name="value">
+            <number>5</number>
+           </property>
+          </widget>
+         </item>
+         <item row="3" column="2">
           <widget class="QSpinBox" name="port">
            <property name="minimum">
             <number>1024</number>
@@ -297,8 +316,8 @@
            </property>
           </widget>
          </item>
-         <item row="1" column="1">
+         <item row="3" column="1">
-          <widget class="QLineEdit" name="forceAddress">
+          <widget class="QLineEdit" name="listenIP">
            <property name="inputMask">
             <string/>
            </property>
@@ -310,16 +329,6 @@
            </property>
           </widget>
          </item>
-         <item row="1" column="0">
-          <widget class="QLabel" name="label">
-           <property name="text">
-            <string>&amp;Listen on</string>
-           </property>
-           <property name="buddy">
-            <cstring>port</cstring>
-           </property>
-          </widget>
-         </item>
          <item row="0" column="0">
           <widget class="QLabel" name="label_3">
            <property name="text">
@@ -330,20 +339,17 @@
            </property>
           </widget>
          </item>
-         <item row="0" column="1" colspan="2">
+         <item row="5" column="1">
-          <widget class="QSpinBox" name="idealPeers">
+          <widget class="QLineEdit" name="forcePublicIP">
-           <property name="minimum">
+           <property name="placeholderText">
-            <number>1</number>
+            <string>Automatic</string>
-           </property>
-           <property name="value">
-            <number>5</number>
            </property>
           </widget>
          </item>
-         <item row="2" column="1" colspan="2">
+         <item row="5" column="0">
-          <widget class="QLineEdit" name="clientName">
+          <widget class="QLabel" name="label_2">
-           <property name="placeholderText">
+           <property name="text">
-            <string>Anonymous</string>
+            <string>Public IP</string>
            </property>
           </widget>
          </item>
@@ -357,6 +363,13 @@
            </property>
           </widget>
          </item>
+         <item row="2" column="1">
+          <widget class="QLineEdit" name="clientName">
+           <property name="placeholderText">
+            <string>Anonymous</string>
+           </property>
+          </widget>
+         </item>
         </layout>
        </widget>
       </widget>
@@ -566,7 +579,7 @@
     </layout>
    </widget>
   </widget>
-  <widget class="QDockWidget" name="dockWidget_6">
+  <widget class="QDockWidget" name="dockWidget_contracts">
    <property name="features">
     <set>QDockWidget::DockWidgetFeatureMask</set>
    </property>
@@ -1443,12 +1456,12 @@ font-size: 14pt</string>
     <string>Show &amp;Anonymous Accounts</string>
    </property>
   </action>
-  <action name="usePast">
+  <action name="dropPeers">
    <property name="checkable">
     <bool>true</bool>
    </property>
    <property name="text">
-    <string>Use &amp;Past Peers</string>
+    <string>&amp;Drop Past Peers</string>
    </property>
   </action>
   <action name="loadJS">
@@ -1694,9 +1707,8 @@ font-size: 14pt</string>
   <tabstop>tabWidget</tabstop>
   <tabstop>urlEdit</tabstop>
   <tabstop>idealPeers</tabstop>
-  <tabstop>forceAddress</tabstop>
+  <tabstop>listenIP</tabstop>
   <tabstop>port</tabstop>
-  <tabstop>clientName</tabstop>
   <tabstop>transactionQueue</tabstop>
   <tabstop>pendingInfo</tabstop>
   <tabstop>blockChainFilter</tabstop>

alethzero/MainWin.cpp

@@ -28,6 +28,7 @@
 //pragma GCC diagnostic ignored "-Werror=pedantic"
 #include <QtNetwork/QNetworkReply>
 #include <QtWidgets/QFileDialog>
+#include <QtWidgets/QDialog>
 #include <QtWidgets/QMessageBox>
 #include <QtWidgets/QInputDialog>
 #include <QtWebEngine/QtWebEngine>
@@ -135,8 +136,13 @@ Main::Main(QWidget *parent) :
 //		ui->log->addItem(QString::fromStdString(s));
 	};
 
+#if !ETH_FATDB
+	delete ui->dockWidget_accounts;
+	delete ui->dockWidget_contracts;
+#endif
+
 #if ETH_DEBUG
-	m_servers.append("localhost:30300");
+	m_servers.append("127.0.0.1:30300");
 #endif
 	m_servers.append(QString::fromStdString(Host::pocHost() + ":30303"));
 
@@ -164,7 +170,7 @@ Main::Main(QWidget *parent) :
 	bytesConstRef network((byte*)m_networkConfig.data(), m_networkConfig.size());
 	m_webThree.reset(new WebThreeDirect(string("AlethZero/v") + dev::Version + "/" DEV_QUOTED(ETH_BUILD_TYPE) "/" DEV_QUOTED(ETH_BUILD_PLATFORM), getDataDir(), false, {"eth", "shh"}, p2p::NetworkPreferences(), network));
 
-	m_httpConnector.reset(new jsonrpc::HttpServer(8080));
+	m_httpConnector.reset(new jsonrpc::HttpServer(SensibleHttpPort, "", "", dev::SensibleHttpThreads));
 	m_server.reset(new OurWebThreeStubServer(*m_httpConnector, *web3(), keysAsVector(m_myKeys), this));
 	connect(&*m_server, SIGNAL(onNewId(QString)), SLOT(addNewId(QString)));
 	m_server->setIdentities(keysAsVector(owned()));
@@ -246,7 +252,30 @@ void Main::addNewId(QString _ids)
 
 NetworkPreferences Main::netPrefs() const
 {
-	return NetworkPreferences(ui->port->value(), ui->forceAddress->text().toStdString(), ui->upnp->isChecked(), ui->localNetworking->isChecked());
+	auto listenIP = ui->listenIP->text().toStdString();
+	try
+	{
+		listenIP = bi::address::from_string(listenIP).to_string();
+	}
+	catch (...)
+	{
+		listenIP.clear();
+	}
+	
+	auto publicIP = ui->forcePublicIP->text().toStdString();
+	try
+	{
+		publicIP = bi::address::from_string(publicIP).to_string();
+	}
+	catch (...)
+	{
+		publicIP.clear();
+	}
+	
+	if (isPublicAddress(publicIP))
+		return NetworkPreferences(publicIP, listenIP, ui->port->value(), ui->upnp->isChecked());
+	else
+		return NetworkPreferences(listenIP, ui->port->value(), ui->upnp->isChecked());
 }
 
 void Main::onKeysChanged()
@@ -258,6 +287,7 @@ unsigned Main::installWatch(LogFilter const& _tf, WatchHandler const& _f)
 {
 	auto ret = ethereum()->installWatch(_tf);
 	m_handlers[ret] = _f;
+	_f(LocalisedLogEntries());
 	return ret;
 }
 
@@ -265,6 +295,7 @@ unsigned Main::installWatch(dev::h256 _tf, WatchHandler const& _f)
 {
 	auto ret = ethereum()->installWatch(_tf, Reaping::Manual);
 	m_handlers[ret] = _f;
+	_f(LocalisedLogEntries());
 	return ret;
 }
 
@@ -668,9 +699,7 @@ void Main::writeSettings()
 	}
 
 	s.setValue("upnp", ui->upnp->isChecked());
-	s.setValue("forceAddress", ui->forceAddress->text());
+	s.setValue("forceAddress", ui->forcePublicIP->text());
-	s.setValue("usePast", ui->usePast->isChecked());
-	s.setValue("localNetworking", ui->localNetworking->isChecked());
 	s.setValue("forceMining", ui->forceMining->isChecked());
 	s.setValue("paranoia", ui->paranoia->isChecked());
 	s.setValue("natSpec", ui->natSpec->isChecked());
@@ -678,6 +707,7 @@ void Main::writeSettings()
 	s.setValue("showAllAccounts", ui->showAllAccounts->isChecked());
 	s.setValue("clientName", ui->clientName->text());
 	s.setValue("idealPeers", ui->idealPeers->value());
+	s.setValue("listenIP", ui->listenIP->text());
 	s.setValue("port", ui->port->value());
 	s.setValue("url", ui->urlEdit->text());
 	s.setValue("privateChain", m_privateChain);
@@ -737,9 +767,8 @@ void Main::readSettings(bool _skipGeometry)
 	}
 
 	ui->upnp->setChecked(s.value("upnp", true).toBool());
-	ui->forceAddress->setText(s.value("forceAddress", "").toString());
+	ui->forcePublicIP->setText(s.value("forceAddress", "").toString());
-	ui->usePast->setChecked(s.value("usePast", true).toBool());
+	ui->dropPeers->setChecked(false);
-	ui->localNetworking->setChecked(s.value("localNetworking", true).toBool());
 	ui->forceMining->setChecked(s.value("forceMining", false).toBool());
 	on_forceMining_triggered();
 	ui->paranoia->setChecked(s.value("paranoia", false).toBool());
@@ -750,6 +779,7 @@ void Main::readSettings(bool _skipGeometry)
 	if (ui->clientName->text().isEmpty())
 		ui->clientName->setText(QInputDialog::getText(nullptr, "Enter identity", "Enter a name that will identify you on the peer network"));
 	ui->idealPeers->setValue(s.value("idealPeers", ui->idealPeers->value()).toInt());
+	ui->listenIP->setText(s.value("listenIP", "").toString());
 	ui->port->setValue(s.value("port", ui->port->value()).toInt());
 	ui->nameReg->setText(s.value("nameReg", "").toString());
 	m_privateChain = s.value("privateChain", "").toString();
@@ -907,7 +937,7 @@ void Main::on_nameReg_textChanged()
 
 void Main::on_preview_triggered()
 {
-	ethereum()->setDefault(ui->preview->isChecked() ? 0 : -1);
+	ethereum()->setDefault(ui->preview->isChecked() ? PendingBlock : LatestBlock);
 	refreshAll();
 }
 
@@ -1036,6 +1066,7 @@ void Main::refreshPending()
 
 void Main::refreshAccounts()
 {
+#if ETH_FATDB
 	cwatch << "refreshAccounts()";
 	ui->accounts->clear();
 	ui->contracts->clear();
@@ -1053,6 +1084,7 @@ void Main::refreshAccounts()
 						->setData(Qt::UserRole, QByteArray((char const*)i.data(), Address::size));
 			}
 		}
+#endif
 }
 
 void Main::refreshBlockCount()
@@ -1090,7 +1122,7 @@ void Main::refreshBlockChain()
 				blocks.insert(bc.numberHash(b));
 		}
 		else if (f.toLongLong() <= bc.number())
-			blocks.insert(bc.numberHash(u256(f.toLongLong())));
+			blocks.insert(bc.numberHash((unsigned)f.toLongLong()));
 		else if (f.size() == 40)
 		{
 			Address h(f.toStdString());
@@ -1353,7 +1385,7 @@ void Main::on_transactionQueue_currentItemChanged()
 		if (!!receipt.bloom())
 			s << "<div>Log Bloom: " << receipt.bloom() << "</div>";
 		else
-			s << "<div>Log Bloom: <i>Uneventful</i></div>";
+			s << "<div>Log Bloom: <b><i>Uneventful</i></b></div>";
 		auto r = receipt.rlp();
 		s << "<div>Receipt: " << toString(RLP(r)) << "</div>";
 		s << "<div>Receipt-Hex: " Span(Mono) << toHex(receipt.rlp()) << "</span></div>";
@@ -1452,7 +1484,7 @@ void Main::on_blocks_currentItemChanged()
 			if (!!info.logBloom)
 				s << "<div>Log Bloom: " << info.logBloom << "</div>";
 			else
-				s << "<div>Log Bloom: <i>Uneventful</i></div>";
+				s << "<div>Log Bloom: <b><i>Uneventful</i></b></div>";
 			s << "<div>Transactions: <b>" << block[1].itemCount() << "</b> @<b>" << info.transactionsRoot << "</b>" << "</div>";
 			s << "<div>Uncles: <b>" << block[2].itemCount() << "</b> @<b>" << info.sha3Uncles << "</b>" << "</div>";
 			for (auto u: block[2])
@@ -1625,18 +1657,24 @@ void Main::on_ourAccounts_doubleClicked()
 }*/
 
 void Main::on_accounts_doubleClicked()
+{
+	if (ui->accounts->count())
 	{
 		auto hba = ui->accounts->currentItem()->data(Qt::UserRole).toByteArray();
 		auto h = Address((byte const*)hba.data(), Address::ConstructFromPointer);
 		qApp->clipboard()->setText(QString::fromStdString(toHex(h.asArray())));
 	}
+}
 
 void Main::on_contracts_doubleClicked()
+{
+	if (ui->contracts->count())
 	{
 		auto hba = ui->contracts->currentItem()->data(Qt::UserRole).toByteArray();
 		auto h = Address((byte const*)hba.data(), Address::ConstructFromPointer);
 		qApp->clipboard()->setText(QString::fromStdString(toHex(h.asArray())));
 	}
+}
 
 static shh::FullTopic topicFromText(QString _s)
 {
@@ -1728,17 +1766,15 @@ void Main::on_net_triggered()
 	if (ui->net->isChecked())
 	{
 		web3()->setIdealPeerCount(ui->idealPeers->value());
-		web3()->setNetworkPreferences(netPrefs());
+		web3()->setNetworkPreferences(netPrefs(), ui->dropPeers->isChecked());
 		ethereum()->setNetworkId(m_privateChain.size() ? sha3(m_privateChain.toStdString()) : h256());
-		// TODO: p2p
-//		if (m_networkConfig.size()/* && ui->usePast->isChecked()*/)
-//			web3()->restoreNetwork(bytesConstRef((byte*)m_networkConfig.data(), m_networkConfig.size()));
 		web3()->startNetwork();
 		ui->downloadView->setDownloadMan(ethereum()->downloadMan());
 	}
 	else
 	{
 		ui->downloadView->setDownloadMan(nullptr);
+		writeSettings();
 		web3()->stopNetwork();
 	}
 }
@@ -1750,13 +1786,25 @@ void Main::on_connect_triggered()
 		ui->net->setChecked(true);
 		on_net_triggered();
 	}
-	bool ok = false;
+	
-	QString s = QInputDialog::getItem(this, "Connect to a Network Peer", "Enter a peer to which a connection may be made:", m_servers, m_servers.count() ? rand() % m_servers.count() : 0, true, &ok);
+	m_connect.setEnvironment(m_servers);
-	if (ok && s.contains(":"))
+	if (m_connect.exec() == QDialog::Accepted)
+	{
+		bool required = m_connect.required();
+		string host(m_connect.host().toStdString());
+		NodeId nodeID;
+		try
 		{
-		string host = s.section(":", 0, 0).toStdString();
+			nodeID = NodeId(fromHex(m_connect.nodeId().toStdString()));
-		unsigned short port = s.section(":", 1).toInt();
+		}
-		web3()->connect(host, port);
+		catch (BadHexCharacter&) {}
+		
+		m_connect.reset();
+		
+		if (required)
+			web3()->requirePeer(nodeID, host);
+		else
+			web3()->addNode(nodeID, host);
 	}
 }
 
@@ -1867,7 +1915,7 @@ void Main::on_go_triggered()
 		ui->net->setChecked(true);
 		on_net_triggered();
 	}
-	web3()->connect(Host::pocHost());
+	web3()->addNode(p2p::NodeId(), Host::pocHost());
 }
 
 QString Main::prettyU256(dev::u256 _n) const

alethzero/MainWin.h

@@ -40,6 +40,7 @@
 #include "Context.h"
 #include "Transact.h"
 #include "NatspecHandler.h"
+#include "Connect.h"
 
 namespace Ui {
 class Main;
@@ -256,4 +257,6 @@ private:
 	std::unique_ptr<DappHost> m_dappHost;
 	DappLoader* m_dappLoader;
 	QWebEnginePage* m_webPage;
+	
+	Connect m_connect;
 };

alethzero/Transact.cpp

@@ -316,7 +316,7 @@ void Transact::rejigData()
 		return;
 	}
 	else
-		gasNeeded = min<qint64>((qint64)ethereum()->gasLimitRemaining(), (qint64)((b - value()) / gasPrice()));
+		gasNeeded = (qint64)min<bigint>(ethereum()->gasLimitRemaining(), ((b - value()) / gasPrice()));
 
 	// Dry-run execution to determine gas requirement and any execution errors
 	Address to;
@@ -326,10 +326,10 @@ void Transact::rejigData()
 	else
 	{
 		to = m_context->fromString(ui->destination->currentText());
-		er = ethereum()->call(s, value(), to, m_data, ethereum()->gasLimitRemaining(), gasPrice());
+		er = ethereum()->call(s, value(), to, m_data, gasNeeded, gasPrice());
 	}
-	gasNeeded = (qint64)er.gasUsed;
+	gasNeeded = (qint64)(er.gasUsed + er.gasRefunded);
-	htmlInfo = QString("<div class=\"info\"><span class=\"icon\">INFO</span> Gas required: %1 total = %2 base, %3 exec</div>").arg(gasNeeded).arg(baseGas).arg(gasNeeded - baseGas) + htmlInfo;
+	htmlInfo = QString("<div class=\"info\"><span class=\"icon\">INFO</span> Gas required: %1 total = %2 base, %3 exec [%4 refunded later]</div>").arg(gasNeeded).arg(baseGas).arg(gasNeeded - baseGas).arg((qint64)er.gasRefunded) + htmlInfo;
 
 	if (er.excepted != TransactionException::None)
 	{
@@ -338,7 +338,7 @@ void Transact::rejigData()
 	}
 	if (er.codeDeposit == CodeDeposit::Failed)
 	{
-		bail("<div class=\"error\"><span class=\"icon\">ERROR</span> Code deposit failed due to insufficient gas</div>");
+		bail("<div class=\"error\"><span class=\"icon\">ERROR</span> Code deposit failed due to insufficient gas; " + QString::fromStdString(toString(er.gasForDeposit)) + " GAS &lt; " + QString::fromStdString(toString(er.depositSize)) + " bytes * " + QString::fromStdString(toString(c_createDataGas)) + "GAS/byte</div>");
 		return;
 	}
 

cmake/EthCompilerSettings.cmake

@@ -38,12 +38,14 @@ elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
 	# disable decorated name length exceeded, name was truncated (4503)
 	# disable warning C4535: calling _set_se_translator() requires /EHa (for boost tests)
 	# declare Windows XP requirement
-	add_compile_options(/MP /EHsc /wd4068 /wd4996 /wd4503 -D_WIN32_WINNT=0x0501)
+	# undefine windows.h MAX && MIN macros cause it cause conflicts with std::min && std::max functions
+	add_compile_options(/MP /EHsc /wd4068 /wd4996 /wd4503 -D_WIN32_WINNT=0x0501 /DNOMINMAX)
 	# disable empty object file warning
 	set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} /ignore:4221")
 	# warning LNK4075: ignoring '/EDITANDCONTINUE' due to '/SAFESEH' specification 
 	# warning LNK4099: pdb was not found with lib
-	set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /ignore:4099,4075")
+	# stack size 16MB
+	set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /ignore:4099,4075 /STACK:16777216")
 	# windows likes static
 	set(ETH_STATIC 1)
 

cmake/EthDependencies.cmake

@@ -30,6 +30,9 @@ if (APPLE)
 	set (CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/usr/local/opt/qt5")
 endif()
 
+find_program(CTEST_COMMAND ctest)
+message(STATUS "ctest path: ${CTEST_COMMAND}")
+
 # Dependencies must have a version number, to ensure reproducible build. The version provided here is the one that is in the extdep repository. If you use system libraries, version numbers may be different.
 
 find_package (CryptoPP 5.6.2 EXACT REQUIRED)
@@ -45,16 +48,10 @@ find_package (Jsoncpp 0.60 REQUIRED)
 message(" - Jsoncpp header: ${JSONCPP_INCLUDE_DIRS}")
 message(" - Jsoncpp lib   : ${JSONCPP_LIBRARIES}")
 
-# TODO the JsonRpcCpp package does not yet check for correct version number
-# json-rpc-cpp support is currently not mandatory
-# TODO make headless client optional
 # TODO get rid of -DETH_JSONRPC
+# TODO add EXACT once we commit ourselves to cmake 3.x
 if (JSONRPC)
-
+	find_package (json_rpc_cpp 0.4 REQUIRED)
-	find_package (JsonRpcCpp 0.3.2)
-	if (NOT JSON_RPC_CPP_FOUND)
-		message (FATAL_ERROR "JSONRPC 0.3.2. not found")
-	endif()
 	message (" - json-rpc-cpp header: ${JSON_RPC_CPP_INCLUDE_DIRS}")
 	message (" - json-rpc-cpp lib   : ${JSON_RPC_CPP_LIBRARIES}")
 	add_definitions(-DETH_JSONRPC)
@@ -114,14 +111,17 @@ if (NOT HEADLESS)
 # find all of the Qt packages
 # remember to use 'Qt' instead of 'QT', cause unix is case sensitive
 # TODO make headless client optional
-	find_package (Qt5Core REQUIRED)
+
-	find_package (Qt5Gui REQUIRED)
+	set (ETH_QT_VERSION 5.4)
-	find_package (Qt5Quick REQUIRED)
+
-	find_package (Qt5Qml REQUIRED)
+	find_package (Qt5Core ${ETH_QT_VERSION} REQUIRED)
-	find_package (Qt5Network REQUIRED)
+	find_package (Qt5Gui ${ETH_QT_VERSION} REQUIRED)
-	find_package (Qt5Widgets REQUIRED)
+	find_package (Qt5Quick ${ETH_QT_VERSION} REQUIRED)
-	find_package (Qt5WebEngine REQUIRED)
+	find_package (Qt5Qml ${ETH_QT_VERSION} REQUIRED)
-	find_package (Qt5WebEngineWidgets REQUIRED)
+	find_package (Qt5Network ${ETH_QT_VERSION} REQUIRED)
+	find_package (Qt5Widgets ${ETH_QT_VERSION} REQUIRED)
+	find_package (Qt5WebEngine ${ETH_QT_VERSION} REQUIRED)
+	find_package (Qt5WebEngineWidgets ${ETH_QT_VERSION} REQUIRED)
 
 	# we need to find path to macdeployqt on mac
 	if (APPLE)

cmake/EthUtils.cmake

@@ -22,3 +22,43 @@ macro(replace_if_different SOURCE DST)
 	endif()
 endmacro()
 
+macro(eth_add_test NAME) 
+
+	# parse arguments here
+	set(commands)
+	set(current_command "")
+	foreach (arg ${ARGN})
+		if (arg STREQUAL "ARGS")
+			if (current_command)
+				list(APPEND commands ${current_command})
+			endif()
+			set(current_command "")
+		else ()
+			set(current_command "${current_command} ${arg}")
+		endif()
+	endforeach(arg)
+	list(APPEND commands ${current_command})
+
+	message(STATUS "test: ${NAME} | ${commands}")
+
+	# create tests
+	set(index 0)
+	list(LENGTH commands count)
+	while (index LESS count)
+		list(GET commands ${index} test_arguments)
+
+		set(run_test "--run_test=${NAME}")
+		add_test(NAME "${NAME}.${index}" COMMAND testeth ${run_test} ${test_arguments})
+		
+		math(EXPR index "${index} + 1")
+	endwhile(index LESS count)
+
+	# add target to run them
+	add_custom_target("test.${NAME}"
+		DEPENDS testeth
+		WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+		COMMAND ${CMAKE_COMMAND} -DETH_TEST_NAME="${NAME}" -DCTEST_COMMAND="${CTEST_COMMAND}" -P "${ETH_SCRIPTS_DIR}/runtest.cmake"
+	)
+
+endmacro()
+

cmake/FindJsonRpcCpp.cmake → cmake/Findjson_rpc_cpp.cmake

@@ -10,6 +10,11 @@
 #  JSON_RPC_CPP_SERVER_LIBRARIES, the libraries needed to use json-rpc-cpp-server
 #  JSON_RPC_CPP_CLIENT_LIBRARIES, the libraries needed to use json-rpc-cpp-client
 #  JSON_RCP_CPP_FOUND, If false, do not try to use json-rpc-cpp.
+#  JSON_RPC_CPP_VERSION, version of library
+#  JSON_RPC_CPP_VERSION_MAJOR
+#  JSON_RPC_CPP_VERSION_MINOR
+#  JSON_RPC_CPP_VERSION_PATCH
+
 
 # only look in default directories
 find_path(
@@ -90,10 +95,28 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
 
 endif()
 
+if (JSON_RPC_CPP_INCLUDE_DIR)
+	set (JSON_RPC_CPP_VERSION_HEADER "${JSON_RPC_CPP_INCLUDE_DIR}/jsonrpccpp/version.h")
+	if (EXISTS ${JSON_RPC_CPP_VERSION_HEADER})
+		file (STRINGS ${JSON_RPC_CPP_VERSION_HEADER} JSON_RPC_CPP_VERSION_MAJOR REGEX "^#define JSONRPC_CPP_MAJOR_VERSION[ \t]+[0-9]+$") 
+		file (STRINGS ${JSON_RPC_CPP_VERSION_HEADER} JSON_RPC_CPP_VERSION_MINOR REGEX "^#define JSONRPC_CPP_MINOR_VERSION[ \t]+[0-9]+$") 
+		file (STRINGS ${JSON_RPC_CPP_VERSION_HEADER} JSON_RPC_CPP_VERSION_PATCH REGEX "^#define JSONRPC_CPP_PATCH_VERSION[ \t]+[0-9]+$") 
+		string (REGEX REPLACE "^#define JSONRPC_CPP_MAJOR_VERSION[ \t]+([0-9]+)" "\\1" JSON_RPC_CPP_VERSION_MAJOR ${JSON_RPC_CPP_VERSION_MAJOR})
+		string (REGEX REPLACE "^#define JSONRPC_CPP_MINOR_VERSION[ \t]+([0-9]+)" "\\1" JSON_RPC_CPP_VERSION_MINOR ${JSON_RPC_CPP_VERSION_MINOR})
+		string (REGEX REPLACE "^#define JSONRPC_CPP_PATCH_VERSION[ \t]+([0-9]+)" "\\1" JSON_RPC_CPP_VERSION_PATCH ${JSON_RPC_CPP_VERSION_PATCH})
+		set (JSON_RPC_CPP_VERSION ${JSON_RPC_CPP_VERSION_MAJOR}.${JSON_RPC_CPP_VERSION_MINOR}.${JSON_RPC_CPP_VERSION_PATCH})
+	endif()
+endif()
+
 # handle the QUIETLY and REQUIRED arguments and set JSON_RPC_CPP_FOUND to TRUE
 # if all listed variables are TRUE, hide their existence from configuration view
 include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(json_rpc_cpp DEFAULT_MSG
+
-    JSON_RPC_CPP_COMMON_LIBRARY JSON_RPC_CPP_SERVER_LIBRARY JSON_RPC_CPP_CLIENT_LIBRARY JSON_RPC_CPP_INCLUDE_DIR)                             
+find_package_handle_standard_args(
-mark_as_advanced (JSON_RPC_CPP_COMMON_LIBRARY JSON_RPC_CPP_SERVER_LIBRARY JSON_RPC_CPP_CLIENT_LIBRARY JSON_RPC_CPP_INCLUDE_DIR)
+	json_rpc_cpp
+	REQUIRED_VARS JSON_RPC_CPP_INCLUDE_DIR JSON_RPC_CPP_COMMON_LIBRARY JSON_RPC_CPP_SERVER_LIBRARY JSON_RPC_CPP_CLIENT_LIBRARY
+	VERSION_VAR JSON_RPC_CPP_VERSION
+)
+
+mark_as_advanced (JSON_RPC_CPP_INCLUDE_DIR JSON_RPC_CPP_COMMON_LIBRARY JSON_RPC_CPP_SERVER_LIBRARY JSON_RPC_CPP_CLIENT_LIBRARY)
 

cmake/scripts/runtest.cmake

@@ -0,0 +1,15 @@
+# Should be used to run ctest
+# 
+# example usage:
+# cmake -DETH_TEST_NAME=TestInterfaceStub -DCTEST_COMMAND=/path/to/ctest -P scripts/runtest.cmake
+
+if (NOT CTEST_COMMAND)
+	message(FATAL_ERROR "ctest could not be found!")
+endif()
+
+# verbosity is off, cause BOOST_MESSAGE is not thread safe and output is a trash
+# see https://codecrafter.wordpress.com/2012/11/01/c-unit-test-framework-adapter-part-3/
+#
+# output might not be usefull cause of thread safety issue 
+execute_process(COMMAND ${CTEST_COMMAND} --force-new-ctest-process -C Debug --output-on-failure -j 4 -R "${ETH_TEST_NAME}[.].*")
+

eth/main.cpp

@@ -77,7 +77,6 @@ void interactiveHelp()
 		<< "    setetherprice <p>  Resets the ether price." << endl
 		<< "    setpriority <p>  Resets the transaction priority." << endl
 		<< "    minestart  Starts mining." << endl
-		<< "    minestart  Starts mining." << endl
 		<< "    minestop  Stops mining." << endl
 		<< "    mineforce <enable>  Forces mining, even when there are no transactions." << endl
 		<< "    address  Gives the current address." << endl
@@ -88,8 +87,10 @@ void interactiveHelp()
 		<< "    send  Execute a given transaction with current secret." << endl
 		<< "    contract  Create a new contract with current secret." << endl
 		<< "    peers  List the peers that are connected" << endl
+#if ETH_FATDB
 		<< "    listaccounts  List the accounts on the network." << endl
 		<< "    listcontracts  List the contracts on the network." << endl
+#endif
 		<< "    setsecret <secret>  Set the secret to the hex secret key." << endl
 		<< "    setaddress <addr>  Set the coinbase (mining payout) address." << endl
 		<< "    exportconfig <path>  Export the config (.RLP) to the path provided." << endl
@@ -117,11 +118,12 @@ void help()
 		<< "    -i,--interactive  Enter interactive mode (default: non-interactive)." << endl
 #if ETH_JSONRPC
 		<< "	-j,--json-rpc  Enable JSON-RPC server (default: off)." << endl
-		<< "	--json-rpc-port	 Specify JSON-RPC server port (implies '-j', default: 8080)." << endl
+		<< "	--json-rpc-port	 Specify JSON-RPC server port (implies '-j', default: " << SensibleHttpPort << ")." << endl
 #endif
 		<< "    -K,--kill-blockchain  First kill the blockchain." << endl
-		<< "    -l,--listen <port>  Listen on the given port for incoming connected (default: 30303)." << endl
+		<< "       --listen-ip <port>  Listen on the given port for incoming connections (default: 30303)." << endl
-		<< "    -L,--local-networking Use peers whose addresses are local." << endl
+		<< "    -l,--listen <ip>  Listen on the given IP for incoming connections (default: 0.0.0.0)." << endl
+		<< "    -u,--public-ip <ip>  Force public ip to given (default: auto)." << endl
 		<< "    -m,--mining <on/off/number>  Enable mining, optionally for a specified number of blocks (Default: off)" << endl
 		<< "    -n,--upnp <on/off>  Use upnp for NAT (default: on)." << endl
 		<< "    -o,--mode <full/peer>  Start a full node or a peer node (Default: full)." << endl
@@ -130,7 +132,6 @@ void help()
 		<< "    -r,--remote <host>  Connect to remote host (default: none)." << endl
 		<< "    -s,--secret <secretkeyhex>  Set the secret key for use with send command (default: auto)." << endl
 		<< "    -t,--miners <number>  Number of mining threads to start (Default: " << thread::hardware_concurrency() << ")" << endl
-		<< "    -u,--public-ip <ip>  Force public ip to given (default; auto)." << endl
 		<< "    -v,--verbosity <0 - 9>  Set the log verbosity from 0 to 9 (Default: 8)." << endl
 		<< "    -x,--peers <number>  Attempt to connect to given number of peers (Default: 5)." << endl
 		<< "    -V,--version  Show the version and exit." << endl
@@ -198,7 +199,9 @@ enum class NodeMode
 
 int main(int argc, char** argv)
 {
+	string listenIP;
 	unsigned short listenPort = 30303;
+	string publicIP;
 	string remoteHost;
 	unsigned short remotePort = 30303;
 	string dbPath;
@@ -210,10 +213,8 @@ int main(int argc, char** argv)
 #if ETH_JSONRPC
 	int jsonrpc = -1;
 #endif
-	string publicIP;
 	bool bootstrap = false;
 	bool upnp = true;
-	bool useLocal = false;
 	bool forceMining = false;
 	bool killChain = false;
 	bool jit = false;
@@ -249,7 +250,9 @@ int main(int argc, char** argv)
 	for (int i = 1; i < argc; ++i)
 	{
 		string arg = argv[i];
-		if ((arg == "-l" || arg == "--listen" || arg == "--listen-port") && i + 1 < argc)
+		if (arg == "--listen-ip" && i + 1 < argc)
+			listenIP = argv[++i];
+		else if ((arg == "-l" || arg == "--listen" || arg == "--listen-port") && i + 1 < argc)
 			listenPort = (short)atoi(argv[++i]);
 		else if ((arg == "-u" || arg == "--public-ip" || arg == "--public") && i + 1 < argc)
 			publicIP = argv[++i];
@@ -270,8 +273,6 @@ int main(int argc, char** argv)
 				return -1;
 			}
 		}
-		else if (arg == "-L" || arg == "--local-networking")
-			useLocal = true;
 		else if (arg == "-K" || arg == "--kill-blockchain")
 			killChain = true;
 		else if ((arg == "-c" || arg == "--client-name") && i + 1 < argc)
@@ -370,7 +371,7 @@ int main(int argc, char** argv)
 			interactive = true;
 #if ETH_JSONRPC
 		else if ((arg == "-j" || arg == "--json-rpc"))
-			jsonrpc = jsonrpc == -1 ? 8080 : jsonrpc;
+			jsonrpc = jsonrpc == -1 ? SensibleHttpPort : jsonrpc;
 		else if (arg == "--json-rpc-port" && i + 1 < argc)
 			jsonrpc = atoi(argv[++i]);
 #endif
@@ -420,7 +421,7 @@ int main(int argc, char** argv)
 
 	StructuredLogger::get().initialize(structuredLogging, structuredLoggingFormat);
 	VMFactory::setKind(jit ? VMKind::JIT : VMKind::Interpreter);
-	NetworkPreferences netPrefs(listenPort, publicIP, upnp, useLocal);
+	auto netPrefs = publicIP.empty() ? NetworkPreferences(listenIP ,listenPort, upnp) : NetworkPreferences(publicIP, listenIP ,listenPort, upnp);
 	auto nodesState = contents((dbPath.size() ? dbPath : getDataDir()) + "/network.rlp");
 	std::string clientImplString = "Ethereum(++)/" + clientName + "v" + dev::Version + "/" DEV_QUOTED(ETH_BUILD_TYPE) "/" DEV_QUOTED(ETH_BUILD_PLATFORM) + (jit ? "/JIT" : "");
 	dev::WebThreeDirect web3(
@@ -448,16 +449,16 @@ int main(int argc, char** argv)
 	web3.startNetwork();
 
 	if (bootstrap)
-		web3.connect(Host::pocHost());
+		web3.addNode(p2p::NodeId(), Host::pocHost());
 	if (remoteHost.size())
-		web3.connect(remoteHost, remotePort);
+		web3.addNode(p2p::NodeId(), remoteHost + ":" + toString(remotePort));
 
 #if ETH_JSONRPC
 	shared_ptr<WebThreeStubServer> jsonrpcServer;
 	unique_ptr<jsonrpc::AbstractServerConnector> jsonrpcConnector;
 	if (jsonrpc > -1)
 	{
-		jsonrpcConnector = unique_ptr<jsonrpc::AbstractServerConnector>(new jsonrpc::HttpServer(jsonrpc));
+		jsonrpcConnector = unique_ptr<jsonrpc::AbstractServerConnector>(new jsonrpc::HttpServer(jsonrpc, "", "", SensibleHttpThreads));
 		jsonrpcServer = shared_ptr<WebThreeStubServer>(new WebThreeStubServer(*jsonrpcConnector.get(), web3, vector<KeyPair>({us})));
 		jsonrpcServer->setIdentities({us});
 		jsonrpcServer->StartListening();
@@ -510,7 +511,7 @@ int main(int argc, char** argv)
 				string addr;
 				unsigned port;
 				iss >> addr >> port;
-				web3.connect(addr, (short)port);
+				web3.addNode(p2p::NodeId(), addr + ":" + toString(port ? port : p2p::c_defaultIPPort));
 			}
 			else if (cmd == "netstop")
 			{
@@ -582,8 +583,8 @@ int main(int argc, char** argv)
 			else if (cmd == "jsonstart")
 			{
 				if (jsonrpc < 0)
-					jsonrpc = 8080;
+					jsonrpc = SensibleHttpPort;
-				jsonrpcConnector = unique_ptr<jsonrpc::AbstractServerConnector>(new jsonrpc::HttpServer(jsonrpc));
+				jsonrpcConnector = unique_ptr<jsonrpc::AbstractServerConnector>(new jsonrpc::HttpServer(jsonrpc, "", "", SensibleHttpThreads));
 				jsonrpcServer = shared_ptr<WebThreeStubServer>(new WebThreeStubServer(*jsonrpcConnector.get(), web3, vector<KeyPair>({us})));
 				jsonrpcServer->setIdentities({us});
 				jsonrpcServer->StartListening();
@@ -685,6 +686,7 @@ int main(int argc, char** argv)
 				else
 					cwarn << "Require parameters: submitTransaction ADDRESS AMOUNT GASPRICE GAS SECRET DATA";
 			}
+#if ETH_FATDB
 			else if (c && cmd == "listcontracts")
 			{
 				auto acs =c->addresses();
@@ -707,6 +709,7 @@ int main(int argc, char** argv)
 						cout << ss << endl;
 					}
 			}
+#endif
 			else if (c && cmd == "send")
 			{
 				if (iss.peek() != -1)
@@ -832,11 +835,8 @@ int main(int argc, char** argv)
 					Executive e(state, c->blockChain(), 0);
 					Transaction t = state.pending()[index];
 					state = state.fromPending(index);
-					bytes r = t.rlp();
 					try
 					{
-						e.setup(&r);
-
 						OnOpFunc oof;
 						if (format == "pretty")
 							oof = [&](uint64_t steps, Instruction instr, bigint newMemSize, bigint gasCost, dev::eth::VM* vvm, dev::eth::ExtVMFace const* vextVM)
@@ -869,6 +869,8 @@ int main(int argc, char** argv)
 										f << toHex(dev::toCompactBigEndian(i.first, 1)) << " " << toHex(dev::toCompactBigEndian(i.second, 1)) << endl;
 								f << ext->myAddress << " " << hex << toHex(dev::toCompactBigEndian(vm->curPC(), 1)) << " " << hex << toHex(dev::toCompactBigEndian((int)(byte)instr, 1)) << " " << hex << toHex(dev::toCompactBigEndian((uint64_t)vm->gas(), 1)) << endl;
 							};
+						e.initialize(t);
+						if (!e.execute())
 							e.go(oof);
 						e.finalize();
 					}

ethrpctest/CMakeLists.txt

@@ -0,0 +1,35 @@
+cmake_policy(SET CMP0015 NEW)
+set(CMAKE_AUTOMOC OFF)
+
+aux_source_directory(. SRC_LIST)
+
+include_directories(BEFORE ${JSONCPP_INCLUDE_DIRS})
+include_directories(BEFORE ..)
+include_directories(${Boost_INCLUDE_DIRS})
+include_directories(${JSON_RPC_CPP_INCLUDE_DIRS})
+
+set(EXECUTABLE ethrpctest)
+
+file(GLOB HEADERS "*.h")
+
+add_executable(${EXECUTABLE} ${SRC_LIST} ${HEADERS})
+
+add_dependencies(${EXECUTABLE} BuildInfo.h)
+
+target_link_libraries(${EXECUTABLE} ${Boost_REGEX_LIBRARIES})
+
+if (READLINE_FOUND)
+	target_link_libraries(${EXECUTABLE} ${READLINE_LIBRARIES})
+endif()
+
+target_link_libraries(${EXECUTABLE} ${Boost_FILESYSTEM_LIBRARIES})
+target_link_libraries(${EXECUTABLE} ${Boost_PROGRAM_OPTIONS_LIBRARIES})
+target_link_libraries(${EXECUTABLE} testutils)
+target_link_libraries(${EXECUTABLE} web3jsonrpc)
+
+if (DEFINED WIN32 AND NOT DEFINED CMAKE_COMPILER_IS_MINGW)
+	add_custom_command(TARGET ${EXECUTABLE} POST_BUILD COMMAND ${CMAKE_COMMAND} ARGS -E copy ${MHD_DLL_RELEASE} "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}")
+endif()
+
+install( TARGETS ${EXECUTABLE} DESTINATION bin )
+

ethrpctest/CommandLineInterface.cpp

@@ -0,0 +1,129 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file CommandLineInterface.cpp
+ * @author Marek Kotewicz <marek@ethdev.com>
+ * @date 2015
+ */
+
+#include <string>
+#include <iostream>
+#include <fstream>
+#include <csignal>
+#include <thread>
+#include <boost/filesystem.hpp>
+#include <jsonrpccpp/server/connectors/httpserver.h>
+#include <libtestutils/Common.h>
+#include <libtestutils/BlockChainLoader.h>
+#include <libtestutils/FixedClient.h>
+#include <libtestutils/FixedWebThreeServer.h>
+#include "CommandLineInterface.h"
+#include "BuildInfo.h"
+
+using namespace std;
+using namespace dev;
+using namespace dev::eth;
+using namespace dev::test;
+namespace po = boost::program_options;
+
+bool CommandLineInterface::parseArguments(int argc, char** argv)
+{
+	// Declare the supported options.
+	po::options_description desc("Allowed options");
+	desc.add_options()
+		("help", "Show help message and exit")
+		("json", po::value<vector<string>>()->required(), "input file")
+		("test", po::value<vector<string>>()->required(), "test case name");
+
+	// All positional options should be interpreted as input files
+	po::positional_options_description p;
+
+	// parse the compiler arguments
+	try
+	{
+		po::store(po::command_line_parser(argc, argv).options(desc).positional(p).allow_unregistered().run(), m_args);
+
+		if (m_args.count("help"))
+		{
+			cout << desc;
+			return false;
+		}
+
+		po::notify(m_args);
+	}
+	catch (po::error const& _exception)
+	{
+		cout << _exception.what() << endl;
+		return false;
+	}
+
+	return true;
+}
+
+bool CommandLineInterface::processInput()
+{
+	string infile = m_args["json"].as<vector<string>>()[0];
+
+	auto path = boost::filesystem::path(infile);
+	if (!boost::filesystem::exists(path))
+	{
+		cout << "Non existant input file \"" << infile << "\"" << endl;
+		return false;
+	}
+
+	string test = m_args["test"].as<vector<string>>()[0];
+	Json::Value j = dev::test::loadJsonFromFile(path.string());
+
+	if (j[test].empty())
+	{
+		cout << "Non existant test case \"" << infile << "\"" << endl;
+		return false;
+	}
+	
+	if (!j[test].isObject())
+	{
+		cout << "Incorrect JSON file \"" << infile << "\"" << endl;
+		return false;
+	}
+	
+	m_json = j[test];
+
+	return true;
+}
+
+bool g_exit = false;
+
+void sighandler(int)
+{
+	g_exit = true;
+}
+
+void CommandLineInterface::actOnInput()
+{
+	BlockChainLoader bcl(m_json);
+	FixedClient client(bcl.bc(), bcl.state());
+	unique_ptr<FixedWebThreeServer> jsonrpcServer;
+	auto server = new jsonrpc::HttpServer(8080, "", "", 2);
+	jsonrpcServer.reset(new FixedWebThreeServer(*server, {}, &client));
+	jsonrpcServer->StartListening();
+
+	signal(SIGABRT, &sighandler);
+	signal(SIGTERM, &sighandler);
+	signal(SIGINT, &sighandler);
+
+	while (!g_exit)
+		this_thread::sleep_for(chrono::milliseconds(1000));
+}

ethrpctest/CommandLineInterface.h

@@ -0,0 +1,46 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** CommandLineInterface.h
+ * @author Marek Kotewicz <marek@ethdev.com>
+ * @date 2015
+ */
+#pragma once
+
+#include <json/json.h>
+#include <boost/program_options.hpp>
+
+class CommandLineInterface
+{
+public:
+	CommandLineInterface() {}
+
+	/// Parse command line arguments and return false if we should not continue
+	bool parseArguments(int argc, char** argv);
+	/// Parse input file and check if test exists
+	bool processInput();
+	/// Start FixedJsonRpcServer
+	void actOnInput();
+
+private:
+
+	/// Compiler arguments variable map
+	boost::program_options::variables_map m_args;
+
+	/// loaded json test case
+	Json::Value m_json;
+};
+

ethrpctest/main.cpp

@@ -0,0 +1,34 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** main.cpp
+ * @author Marek Kotewicz <c@ethdev.com>
+ * @date 2015
+ */
+
+#include "CommandLineInterface.h"
+
+int main(int argc, char** argv)
+{
+	CommandLineInterface cli;
+	if (!cli.parseArguments(argc, argv))
+		return 1;
+	if (!cli.processInput())
+		return 1;
+	cli.actOnInput();
+
+	return 0;
+}

evmjit/libevmjit-cpp/Env.cpp

@@ -52,7 +52,6 @@ extern "C"
 		auto endowment = llvm2eth(*_endowment);
 		if (_env->balance(_env->myAddress) >= endowment && _env->depth < 1024)
 		{
-			_env->subBalance(endowment);
 			u256 gas = *io_gas;
 			h256 address(_env->create(endowment, gas, {_initBeg, _initSize}, {}), h256::AlignRight);
 			*io_gas = static_cast<int64_t>(gas);
@@ -89,10 +88,7 @@ extern "C"
 		auto ret = false;
 		auto callGas = u256{_callGas};
 		if (_env->balance(_env->myAddress) >= value && _env->depth < 1024)
-		{
-			_env->subBalance(value);
 			ret = _env->call(receiveAddress, value, {_inBeg, _inSize}, callGas, {_outBeg, _outSize}, {}, {}, codeAddress);
-		}
 
 		*io_gas += static_cast<int64_t>(callGas); // it is never more than initial _callGas
 		return ret;

evmjit/libevmjit/ExecutionEngine.cpp

@@ -4,6 +4,8 @@
 #include <mutex>
 #include <iostream>
 #include <unordered_map>
+#include <cstdlib>
+#include <cstring>
 
 #include "preprocessor/llvm_includes_start.h"
 #include <llvm/IR/Module.h>
@@ -82,7 +84,20 @@ void parseOptions()
 {
 	static llvm::llvm_shutdown_obj shutdownObj{};
 	cl::AddExtraVersionPrinter(printVersion);
-	cl::ParseEnvironmentOptions("evmjit", "EVMJIT", "Ethereum EVM JIT Compiler");
+	//cl::ParseEnvironmentOptions("evmjit", "EVMJIT", "Ethereum EVM JIT Compiler");
+
+	// FIXME: LLVM workaround:
+	// Manually select instruction scheduler other than "source".
+	// "source" scheduler has a bug: http://llvm.org/bugs/show_bug.cgi?id=22304
+	auto envLine = std::getenv("EVMJIT");
+	auto commandLine = std::string{"evmjit "} + (envLine ? envLine : "") + " -pre-RA-sched=list-burr\0";
+	static const auto c_maxArgs = 20;
+	char const* argv[c_maxArgs] = {nullptr, };
+	auto arg = std::strtok(&*commandLine.begin(), " ");
+	auto i = 0;
+	for (; i < c_maxArgs && arg; ++i, arg = std::strtok(nullptr, " "))
+		argv[i] = arg;
+	cl::ParseCommandLineOptions(i, argv, "Ethereum EVM JIT Compiler");
 }
 
 }

libdevcore/CommonData.h

@@ -42,14 +42,15 @@ enum class WhenError
 };
 
 /// Convert a series of bytes to the corresponding string of hex duplets.
-/// @param _w specifies the width of each of the elements. Defaults to two - enough to represent a byte.
+/// @param _w specifies the width of the first of the elements. Defaults to two - enough to represent a byte.
 /// @example toHex("A\x69") == "4169"
 template <class _T>
 std::string toHex(_T const& _data, int _w = 2)
 {
 	std::ostringstream ret;
+	unsigned ii = 0;
 	for (auto i: _data)
-		ret << std::hex << std::setfill('0') << std::setw(_w) << (int)(typename std::make_unsigned<decltype(i)>::type)i;
+		ret << std::hex << std::setfill('0') << std::setw(ii++ ? 2 : _w) << (int)(typename std::make_unsigned<decltype(i)>::type)i;
 	return ret.str();
 }
 
@@ -74,6 +75,13 @@ inline std::string asString(bytes const& _b)
 	return std::string((char const*)_b.data(), (char const*)(_b.data() + _b.size()));
 }
 
+/// Converts byte array ref to a string containing the same (binary) data. Unless
+/// the byte array happens to contain ASCII data, this won't be printable.
+inline std::string asString(bytesConstRef _b)
+{
+	return std::string((char const*)_b.data(), (char const*)(_b.data() + _b.size()));
+}
+
 /// Converts a string to a byte array containing the string's (byte) data.
 inline bytes asBytes(std::string const& _b)
 {

libdevcore/CommonIO.cpp

@@ -112,6 +112,6 @@ string dev::contentsString(std::string const& _file)
 
 void dev::writeFile(std::string const& _file, bytesConstRef _data)
 {
-	ofstream(_file, ios::trunc).write((char const*)_data.data(), _data.size());
+	ofstream(_file, ios::trunc|ios::binary).write((char const*)_data.data(), _data.size());
 }
 

libdevcore/CommonJS.h

@@ -38,7 +38,10 @@ template <unsigned S> std::string toJS(FixedHash<S> const& _h)
 
 template <unsigned N> std::string toJS(boost::multiprecision::number<boost::multiprecision::cpp_int_backend<N, N, boost::multiprecision::unsigned_magnitude, boost::multiprecision::unchecked, void>> const& _n)
 {
-	return "0x" + toHex(toCompactBigEndian(_n, 1));
+	std::string h = toHex(toCompactBigEndian(_n, 1));
+	// remove first 0, if it is necessary;
+	std::string res = h[0] != '0' ? h : h.substr(1);
+	return "0x" + res;
 }
 
 inline std::string toJS(bytes const& _n, std::size_t _padding = 0)

libdevcore/FixedHash.h

@@ -141,6 +141,7 @@ public:
 		return ret;
 	}
 
+	/// @returns a random valued object.
 	static FixedHash random() { return random(s_fixedHashEngine); }
 
 	/// A generic std::hash compatible function object.

libdevcore/vector_ref.h

@@ -39,7 +39,8 @@ public:
 	size_t size() const { return m_count; }
 	bool empty() const { return !m_count; }
 	vector_ref<_T> next() const { return vector_ref<_T>(m_data + m_count, m_count); }
-	vector_ref<_T> cropped(size_t _begin, size_t _count = ~size_t(0)) const { if (m_data && _begin + std::max(size_t(0), _count) <= m_count) return vector_ref<_T>(m_data + _begin, _count == ~size_t(0) ? m_count - _begin : _count); else return vector_ref<_T>(); }
+	vector_ref<_T> cropped(size_t _begin, size_t _count) const { if (m_data && _begin + _count <= m_count) return vector_ref<_T>(m_data + _begin, _count == ~size_t(0) ? m_count - _begin : _count); else return vector_ref<_T>(); }
+	vector_ref<_T> cropped(size_t _begin) const { if (m_data && _begin <= m_count) return vector_ref<_T>(m_data + _begin, m_count - _begin); else return vector_ref<_T>(); }
 	void retarget(_T* _d, size_t _s) { m_data = _d; m_count = _s; }
 	void retarget(std::vector<_T> const& _t) { m_data = _t.data(); m_count = _t.size(); }
 	void copyTo(vector_ref<typename std::remove_const<_T>::type> _t) const { memcpy(_t.data(), m_data, std::min(_t.size(), m_count) * sizeof(_T)); }

libdevcrypto/MemoryDB.h

@@ -32,8 +32,10 @@ namespace dev
 {
 
 struct DBChannel: public LogChannel  { static const char* name() { return "TDB"; } static const int verbosity = 18; };
+struct DBWarn: public LogChannel  { static const char* name() { return "TDB"; } static const int verbosity = 1; };
 
 #define dbdebug clog(DBChannel)
+#define dbwarn clog(DBWarn)
 
 class MemoryDB
 {

libethash/CMakeLists.txt

@@ -12,6 +12,7 @@ endif()
 
 set(FILES 	util.c
           	util.h
+          	io.c
           	internal.c
           	ethash.h
           	endian.h
@@ -19,6 +20,12 @@ set(FILES 	util.c
           	fnv.h
           	data_sizes.h)
 
+if (MSVC)
+	list(APPEND FILES io_win32.c)
+else()
+	list(APPEND FILES io_posix.c)
+endif()
+
 if (NOT CRYPTOPP_FOUND)
 	find_package(CryptoPP 5.6.2)
 endif()

libethash/data_sizes.h

@@ -48,7 +48,7 @@ extern "C" {
 //             Sow[i*HashBytes]; j++]]]][[2]][[1]]
 
 
-static const size_t dag_sizes[2048] = {
+static const uint64_t dag_sizes[2048] = {
         1073739904U, 1082130304U, 1090514816U, 1098906752U, 1107293056U,
         1115684224U, 1124070016U, 1132461952U, 1140849536U, 1149232768U,
         1157627776U, 1166013824U, 1174404736U, 1182786944U, 1191180416U,
@@ -477,7 +477,7 @@ static const size_t dag_sizes[2048] = {
 //         While[! PrimeQ[i], i--];
 //         Sow[i*HashBytes]; j++]]]][[2]][[1]]
 
-const size_t cache_sizes[2048] = {
+const uint64_t cache_sizes[2048] = {
         16776896U, 16907456U, 17039296U, 17170112U, 17301056U, 17432512U, 17563072U,
         17693888U, 17824192U, 17955904U, 18087488U, 18218176U, 18349504U, 18481088U,
         18611392U, 18742336U, 18874304U, 19004224U, 19135936U, 19267264U, 19398208U,

libethash/ethash.h

@@ -24,27 +24,28 @@
 #include <stdbool.h>
 #include <string.h>
 #include <stddef.h>
+#include <stdlib.h>
 #include "compiler.h"
 
-#define REVISION 23
+#define ETHASH_REVISION 23
-#define DATASET_BYTES_INIT 1073741824U // 2**30
+#define ETHASH_DATASET_BYTES_INIT 1073741824U // 2**30
-#define DATASET_BYTES_GROWTH 8388608U  // 2**23
+#define ETHASH_DATASET_BYTES_GROWTH 8388608U  // 2**23
-#define CACHE_BYTES_INIT 1073741824U // 2**24
+#define ETHASH_CACHE_BYTES_INIT 1073741824U // 2**24
-#define CACHE_BYTES_GROWTH 131072U  // 2**17
+#define ETHASH_CACHE_BYTES_GROWTH 131072U  // 2**17
-#define EPOCH_LENGTH 30000U
+#define ETHASH_EPOCH_LENGTH 30000U
-#define MIX_BYTES 128
+#define ETHASH_MIX_BYTES 128
-#define HASH_BYTES 64
+#define ETHASH_HASH_BYTES 64
-#define DATASET_PARENTS 256
+#define ETHASH_DATASET_PARENTS 256
-#define CACHE_ROUNDS 3
+#define ETHASH_CACHE_ROUNDS 3
-#define ACCESSES 64
+#define ETHASH_ACCESSES 64
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 typedef struct ethash_params {
-    size_t full_size;               // Size of full data set (in bytes, multiple of mix size (128)).
+	uint64_t full_size;               // Size of full data set (in bytes, multiple of mix size (128)).
-    size_t cache_size;              // Size of compute cache (in bytes, multiple of node size (64)).
+	uint64_t cache_size;              // Size of compute cache (in bytes, multiple of node size (64)).
 } ethash_params;
 
 typedef struct ethash_return_value {
@@ -52,8 +53,8 @@ typedef struct ethash_return_value {
 	uint8_t mix_hash[32];
 } ethash_return_value;
 
-size_t ethash_get_datasize(const uint32_t block_number);
+uint64_t ethash_get_datasize(const uint32_t block_number);
-size_t ethash_get_cachesize(const uint32_t block_number);
+uint64_t ethash_get_cachesize(const uint32_t block_number);
 
 // initialize the parameters
 static inline void ethash_params_init(ethash_params *params, const uint32_t block_number) {
@@ -61,55 +62,77 @@ static inline void ethash_params_init(ethash_params *params, const uint32_t bloc
 	params->cache_size = ethash_get_cachesize(block_number);
 }
 
-typedef struct ethash_cache {
+/***********************************
-    void *mem;
+ * OLD API *************************
-} ethash_cache;
+ ***********************************
+ ******************** (deprecated) *
+ ***********************************/
 
-void ethash_mkcache(ethash_cache *cache, ethash_params const *params, const uint8_t seed[32]);
-void ethash_compute_full_data(void *mem, ethash_params const *params, ethash_cache const *cache);
-void ethash_full(ethash_return_value *ret, void const *full_mem, ethash_params const *params, const uint8_t header_hash[32], const uint64_t nonce);
-void ethash_light(ethash_return_value *ret, ethash_cache const *cache, ethash_params const *params, const uint8_t header_hash[32], const uint64_t nonce);
 void ethash_get_seedhash(uint8_t seedhash[32], const uint32_t block_number);
+void ethash_mkcache(void *cache, ethash_params const *params, const uint8_t seed[32]);
+void ethash_light(ethash_return_value *ret, void const *cache, ethash_params const *params, const uint8_t header_hash[32], const uint64_t nonce);
+void ethash_compute_full_data(void *mem, ethash_params const *params, void const *cache);
+void ethash_full(ethash_return_value *ret, void const *full_mem, ethash_params const *params, const uint8_t header_hash[32], const uint64_t nonce);
 
-static inline void ethash_prep_light(void *cache, ethash_params const *params, const uint8_t seed[32]) {
+/***********************************
-    ethash_cache c;
+ * NEW API *************************
-    c.mem = cache;
+ ***********************************/
-    ethash_mkcache(&c, params, seed);
+
-}
+// TODO: compute params and seed in ethash_new_light; it should take only block_number
+// TODO: store params in ethash_light_t/ethash_full_t to avoid having to repass into compute/new_full
+
+typedef uint8_t const ethash_seedhash_t[32];
 
-static inline void ethash_compute_light(ethash_return_value *ret, void const *cache, ethash_params const *params, const uint8_t header_hash[32], const uint64_t nonce) {
+typedef void const* ethash_light_t;
-    ethash_cache c;
+static inline ethash_light_t ethash_new_light(ethash_params const* params, ethash_seedhash_t seed) {
-    c.mem = (void *) cache;
+	void* ret = malloc(params->cache_size);
-    ethash_light(ret, &c, params, header_hash, nonce);
+	ethash_mkcache(ret, params, seed);
+	return ret;
+}
+static inline void ethash_compute_light(ethash_return_value *ret, ethash_light_t light, ethash_params const *params, const uint8_t header_hash[32], const uint64_t nonce) {
+	ethash_light(ret, light, params, header_hash, nonce);
+}
+static inline void ethash_delete_light(ethash_light_t light) {
+	free((void*)light);
 }
 
+typedef void const* ethash_full_t;
+static inline ethash_full_t ethash_new_full(ethash_params const* params, ethash_light_t light) {
+	void* ret = malloc(params->full_size);
+	ethash_compute_full_data(ret, params, light);
+	return ret;
+}
 static inline void ethash_prep_full(void *full, ethash_params const *params, void const *cache) {
-    ethash_cache c;
+	ethash_compute_full_data(full, params, cache);
-    c.mem = (void *) cache;
-    ethash_compute_full_data(full, params, &c);
 }
-
 static inline void ethash_compute_full(ethash_return_value *ret, void const *full, ethash_params const *params, const uint8_t header_hash[32], const uint64_t nonce) {
 	ethash_full(ret, full, params, header_hash, nonce);
 }
 
-// Returns if hash is less than or equal to difficulty
+/// @brief Compare two s256-bit big-endian values.
-static inline int ethash_check_difficulty(
+/// @returns 1 if @a a is less than or equal to @a b, 0 otherwise.
-        const uint8_t hash[32],
+/// Both parameters are 256-bit big-endian values.
-        const uint8_t difficulty[32]) {
+static inline int ethash_leq_be256(const uint8_t a[32], const uint8_t b[32]) {
-    // Difficulty is big endian
+	// Boundary is big endian
 	for (int i = 0; i < 32; i++) {
-        if (hash[i] == difficulty[i]) continue;
+		if (a[i] == b[i])
-        return hash[i] < difficulty[i];
+			continue;
+		return a[i] < b[i];
 	}
 	return 1;
 }
 
-int ethash_quick_check_difficulty(
+/// Perofrms a cursory check on the validity of the nonce.
+/// @returns 1 if the nonce may possibly be valid for the given header_hash & boundary.
+/// @p boundary equivalent to 2 ^ 256 / block_difficulty, represented as a 256-bit big-endian.
+int ethash_preliminary_check_boundary(
 	const uint8_t header_hash[32],
 	const uint64_t nonce,
 	const uint8_t mix_hash[32],
-        const uint8_t difficulty[32]);
+	const uint8_t boundary[32]);
+
+#define ethash_quick_check_difficulty ethash_preliminary_check_boundary
+#define ethash_check_difficulty ethash_leq_be256
 
 #ifdef __cplusplus
 }

libethash/internal.c

@@ -37,14 +37,14 @@
 #include "sha3.h"
 #endif // WITH_CRYPTOPP
 
-size_t ethash_get_datasize(const uint32_t block_number) {
+uint64_t ethash_get_datasize(const uint32_t block_number) {
-    assert(block_number / EPOCH_LENGTH < 2048);
+    assert(block_number / ETHASH_EPOCH_LENGTH < 2048);
-    return dag_sizes[block_number / EPOCH_LENGTH];
+    return dag_sizes[block_number / ETHASH_EPOCH_LENGTH];
 }
 
-size_t ethash_get_cachesize(const uint32_t block_number) {
+uint64_t ethash_get_cachesize(const uint32_t block_number) {
-    assert(block_number / EPOCH_LENGTH < 2048);
+    assert(block_number / ETHASH_EPOCH_LENGTH < 2048);
-    return cache_sizes[block_number / EPOCH_LENGTH];
+    return cache_sizes[block_number / ETHASH_EPOCH_LENGTH];
 }
 
 // Follows Sergio's "STRICT MEMORY HARD HASHING FUNCTIONS" (2014)
@@ -63,7 +63,7 @@ void static ethash_compute_cache_nodes(
         SHA3_512(nodes[i].bytes, nodes[i - 1].bytes, 64);
     }
 
-    for (unsigned j = 0; j != CACHE_ROUNDS; j++) {
+    for (unsigned j = 0; j != ETHASH_CACHE_ROUNDS; j++) {
         for (unsigned i = 0; i != num_nodes; i++) {
             uint32_t const idx = nodes[i].words[0] % num_nodes;
             node data;
@@ -85,10 +85,10 @@ void static ethash_compute_cache_nodes(
 }
 
 void ethash_mkcache(
-        ethash_cache *cache,
+		void *cache,
         ethash_params const *params,
         const uint8_t seed[32]) {
-    node *nodes = (node *) cache->mem;
+	node *nodes = (node *) cache;
     ethash_compute_cache_nodes(nodes, params, seed);
 }
 
@@ -96,10 +96,10 @@ void ethash_calculate_dag_item(
         node *const ret,
         const unsigned node_index,
         const struct ethash_params *params,
-        const struct ethash_cache *cache) {
+		const void *cache) {
 
     uint32_t num_parent_nodes = (uint32_t) (params->cache_size / sizeof(node));
-    node const *cache_nodes = (node const *) cache->mem;
+	node const *cache_nodes = (node const *) cache;
     node const *init = &cache_nodes[node_index % num_parent_nodes];
 
     memcpy(ret, init, sizeof(node));
@@ -114,7 +114,7 @@ void ethash_calculate_dag_item(
     __m128i xmm3 = ret->xmm[3];
 #endif
 
-    for (unsigned i = 0; i != DATASET_PARENTS; ++i) {
+    for (unsigned i = 0; i != ETHASH_DATASET_PARENTS; ++i) {
         uint32_t parent_index = ((node_index ^ i) * FNV_PRIME ^ ret->words[i % NODE_WORDS]) % num_parent_nodes;
         node const *parent = &cache_nodes[parent_index];
 
@@ -150,7 +150,7 @@ void ethash_calculate_dag_item(
 void ethash_compute_full_data(
         void *mem,
         ethash_params const *params,
-        ethash_cache const *cache) {
+		void const *cache) {
     assert((params->full_size % (sizeof(uint32_t) * MIX_WORDS)) == 0);
     assert((params->full_size % sizeof(node)) == 0);
     node *full_nodes = mem;
@@ -164,7 +164,7 @@ void ethash_compute_full_data(
 static void ethash_hash(
         ethash_return_value *ret,
         node const *full_nodes,
-        ethash_cache const *cache,
+		void const *cache,
         ethash_params const *params,
         const uint8_t header_hash[32],
         const uint64_t nonce) {
@@ -201,7 +201,7 @@ static void ethash_hash(
             num_full_pages = (unsigned) (params->full_size / page_size);
 
 
-    for (unsigned i = 0; i != ACCESSES; ++i) {
+    for (unsigned i = 0; i != ETHASH_ACCESSES; ++i) {
         uint32_t const index = ((s_mix->words[0] ^ i) * FNV_PRIME ^ mix->words[i % MIX_WORDS]) % num_full_pages;
 
         for (unsigned n = 0; n != MIX_NODES; ++n) {
@@ -275,12 +275,12 @@ void ethash_quick_hash(
 
 void ethash_get_seedhash(uint8_t seedhash[32], const uint32_t block_number) {
     memset(seedhash, 0, 32);
-    const uint32_t epochs = block_number / EPOCH_LENGTH;
+    const uint32_t epochs = block_number / ETHASH_EPOCH_LENGTH;
     for (uint32_t i = 0; i < epochs; ++i)
         SHA3_256(seedhash, seedhash, 32);
 }
 
-int ethash_quick_check_difficulty(
+int ethash_preliminary_check_boundary(
         const uint8_t header_hash[32],
         const uint64_t nonce,
         const uint8_t mix_hash[32],
@@ -288,13 +288,13 @@ int ethash_quick_check_difficulty(
 
 	uint8_t return_hash[32];
     ethash_quick_hash(return_hash, header_hash, nonce, mix_hash);
-    return ethash_check_difficulty(return_hash, difficulty);
+	return ethash_leq_be256(return_hash, difficulty);
 }
 
 void ethash_full(ethash_return_value *ret, void const *full_mem, ethash_params const *params, const uint8_t previous_hash[32], const uint64_t nonce) {
     ethash_hash(ret, (node const *) full_mem, NULL, params, previous_hash, nonce);
 }
 
-void ethash_light(ethash_return_value *ret, ethash_cache const *cache, ethash_params const *params, const uint8_t previous_hash[32], const uint64_t nonce) {
+void ethash_light(ethash_return_value *ret, void const *cache, ethash_params const *params, const uint8_t previous_hash[32], const uint64_t nonce) {
     ethash_hash(ret, NULL, cache, params, previous_hash, nonce);
 }

libethash/internal.h

@@ -3,7 +3,7 @@
 #include "endian.h"
 #include "ethash.h"
 
-#define ENABLE_SSE 1
+#define ENABLE_SSE 0
 
 #if defined(_M_X64) && ENABLE_SSE
 #include <smmintrin.h>
@@ -15,7 +15,7 @@ extern "C" {
 
 // compile time settings
 #define NODE_WORDS (64/4)
-#define MIX_WORDS (MIX_BYTES/4)
+#define MIX_WORDS (ETHASH_MIX_BYTES/4)
 #define MIX_NODES (MIX_WORDS / NODE_WORDS)
 #include <stdint.h>
 
@@ -34,7 +34,7 @@ void ethash_calculate_dag_item(
         node *const ret,
         const unsigned node_index,
         ethash_params const *params,
-        ethash_cache const *cache
+		void const *cache
 );
 
 void ethash_quick_hash(

libethash/io.c

@@ -0,0 +1,89 @@
+/*
+  This file is part of ethash.
+
+  ethash is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  ethash is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with ethash.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file io.c
+ * @author Lefteris Karapetsas <lefteris@ethdev.com>
+ * @date 2015
+ */
+#include "io.h"
+#include <string.h>
+#include <stdio.h>
+
+// silly macro to save some typing
+#define PASS_ARR(c_) (c_), sizeof(c_)
+
+static bool ethash_io_write_file(char const *dirname,
+                                 char const* filename,
+                                 size_t filename_length,
+                                 void const* data,
+                                 size_t data_size)
+{
+    bool ret = false;
+    char *fullname = ethash_io_create_filename(dirname, filename, filename_length);
+    if (!fullname) {
+        return false;
+    }
+    FILE *f = fopen(fullname, "wb");
+    if (!f) {
+        goto free_name;
+    }
+    if (data_size != fwrite(data, 1, data_size, f)) {
+        goto close;
+    }
+
+    ret = true;
+close:
+    fclose(f);
+free_name:
+    free(fullname);
+    return ret;
+}
+
+bool ethash_io_write(char const *dirname,
+                     ethash_params const* params,
+                     ethash_blockhash_t seedhash,
+                     void const* cache,
+                     uint8_t **data,
+                     size_t *data_size)
+{
+    char info_buffer[DAG_MEMO_BYTESIZE];
+    // allocate the bytes
+    uint8_t *temp_data_ptr = malloc(params->full_size);
+    if (!temp_data_ptr) {
+        goto end;
+    }
+    ethash_compute_full_data(temp_data_ptr, params, cache);
+
+    if (!ethash_io_write_file(dirname, PASS_ARR(DAG_FILE_NAME), temp_data_ptr, params->full_size)) {
+        goto fail_free;
+    }
+
+    ethash_io_serialize_info(ETHASH_REVISION, seedhash, info_buffer);
+    if (!ethash_io_write_file(dirname, PASS_ARR(DAG_MEMO_NAME), info_buffer, DAG_MEMO_BYTESIZE)) {
+        goto fail_free;
+    }
+
+    *data = temp_data_ptr;
+    *data_size = params->full_size;
+    return true;
+
+fail_free:
+    free(temp_data_ptr);
+end:
+    return false;
+}
+
+#undef PASS_ARR

libethash/io.h

@@ -0,0 +1,116 @@
+/*
+  This file is part of ethash.
+
+  ethash is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  ethash is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with ethash.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file io.h
+ * @author Lefteris Karapetsas <lefteris@ethdev.com>
+ * @date 2015
+ */
+#pragma once
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include "ethash.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct ethash_blockhash { uint8_t b[32]; } ethash_blockhash_t;
+
+static const char DAG_FILE_NAME[] = "full";
+static const char DAG_MEMO_NAME[] = "full.info";
+// MSVC thinks that "static const unsigned int" is not a compile time variable. Sorry for the #define :(
+#define DAG_MEMO_BYTESIZE 36
+
+/// Possible return values of @see ethash_io_prepare
+enum ethash_io_rc {
+    ETHASH_IO_FAIL = 0,      ///< There has been an IO failure
+    ETHASH_IO_MEMO_MISMATCH, ///< Memo file either did not exist or there was content mismatch
+    ETHASH_IO_MEMO_MATCH,    ///< Memo file existed and contents matched. No need to do anything
+};
+
+/**
+ * Prepares io for ethash
+ *
+ * Create the DAG directory if it does not exist, and check if the memo file matches.
+ * If it does not match then it's deleted to pave the way for @ref ethash_io_write()
+ *
+ * @param dirname        A null terminated c-string of the path of the ethash
+ *                       data directory. If it does not exist it's created.
+ * @param seedhash       The seedhash of the current block number
+ * @return               For possible return values @see enum ethash_io_rc
+ */
+enum ethash_io_rc ethash_io_prepare(char const *dirname, ethash_blockhash_t seedhash);
+/**
+ * Fully computes data and writes it to the file on disk.
+ *
+ * This function should be called after @see ethash_io_prepare() and only if
+ * its return value is @c ETHASH_IO_MEMO_MISMATCH. Will write both the full data
+ * and the memo file.
+ *
+ * @param[in] dirname        A null terminated c-string of the path of the ethash
+ *                           data directory. Has to exist.
+ * @param[in] params         An ethash_params object containing the full size
+ *                           and the cache size
+ * @param[in] seedhash       The seedhash of the current block number
+ * @param[in] cache          The cache data. Would have usually been calulated by
+ *                           @see ethash_prep_light().
+ * @param[out] data          Pass a pointer to uint8_t by reference here. If the
+ *                           function is succesfull then this point to the allocated
+ *                           data calculated by @see ethash_prep_full(). Memory
+ *                           ownership is transfered to the callee. Remember that
+ *                           you eventually need to free this with a call to free().
+ * @param[out] data_size     Pass a size_t by value. If the function is succesfull
+ *                           then this will contain the number of bytes allocated
+ *                           for @a data.
+ * @return                   True for success and false in case of failure.
+ */
+bool ethash_io_write(char const *dirname,
+                     ethash_params const* params,
+                     ethash_blockhash_t seedhash,
+                     void const* cache,
+                     uint8_t **data,
+                     size_t *data_size);
+
+static inline void ethash_io_serialize_info(uint32_t revision,
+                                            ethash_blockhash_t seed_hash,
+                                            char *output)
+{
+    // if .info is only consumed locally we don't really care about endianess
+    memcpy(output, &revision, 4);
+    memcpy(output + 4, &seed_hash, 32);
+}
+
+static inline char *ethash_io_create_filename(char const *dirname,
+                                              char const* filename,
+                                              size_t filename_length)
+{
+    // in C the cast is not needed, but a C++ compiler will complain for invalid conversion
+    char *name = (char*)malloc(strlen(dirname) + filename_length);
+    if (!name) {
+        return NULL;
+    }
+
+    name[0] = '\0';
+    strcat(name, dirname);
+    strcat(name, filename);
+    return name;
+}
+
+
+#ifdef __cplusplus
+}
+#endif

libethash/io_posix.c

@@ -0,0 +1,76 @@
+/*
+  This file is part of ethash.
+
+  ethash is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  ethash is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with ethash.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file io_posix.c
+ * @author Lefteris Karapetsas <lefteris@ethdev.com>
+ * @date 2015
+ */
+
+#include "io.h"
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <libgen.h>
+#include <stdio.h>
+#include <unistd.h>
+
+enum ethash_io_rc ethash_io_prepare(char const *dirname, ethash_blockhash_t seedhash)
+{
+    char read_buffer[DAG_MEMO_BYTESIZE];
+    char expect_buffer[DAG_MEMO_BYTESIZE];
+    enum ethash_io_rc ret = ETHASH_IO_FAIL;
+
+    // assert directory exists, full owner permissions and read/search for others
+    int rc = mkdir(dirname, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+    if (rc == -1 && errno != EEXIST) {
+        goto end;
+    }
+
+    char *memofile = ethash_io_create_filename(dirname, DAG_MEMO_NAME, sizeof(DAG_MEMO_NAME));
+    if (!memofile) {
+        goto end;
+    }
+
+    // try to open memo file
+    FILE *f = fopen(memofile, "rb");
+    if (!f) {
+        // file does not exist, so no checking happens. All is fine.
+        ret = ETHASH_IO_MEMO_MISMATCH;
+        goto free_memo;
+    }
+
+    if (fread(read_buffer, 1, DAG_MEMO_BYTESIZE, f) != DAG_MEMO_BYTESIZE) {
+        goto close;
+    }
+
+    ethash_io_serialize_info(ETHASH_REVISION, seedhash, expect_buffer);
+    if (memcmp(read_buffer, expect_buffer, DAG_MEMO_BYTESIZE) != 0) {
+        // we have different memo contents so delete the memo file
+        if (unlink(memofile) != 0) {
+            goto close;
+        }
+        ret = ETHASH_IO_MEMO_MISMATCH;
+    }
+
+    ret = ETHASH_IO_MEMO_MATCH;
+
+close:
+    fclose(f);
+free_memo:
+    free(memofile);
+end:
+    return ret;
+}

libethash/io_win32.c

@@ -0,0 +1,73 @@
+/*
+  This file is part of ethash.
+
+  ethash is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  ethash is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with ethash.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file io_win32.c
+ * @author Lefteris Karapetsas <lefteris@ethdev.com>
+ * @date 2015
+ */
+
+#include "io.h"
+#include <direct.h>
+#include <errno.h>
+#include <stdio.h>
+
+enum ethash_io_rc ethash_io_prepare(char const *dirname, ethash_blockhash_t seedhash)
+{
+    char read_buffer[DAG_MEMO_BYTESIZE];
+    char expect_buffer[DAG_MEMO_BYTESIZE];
+    enum ethash_io_rc ret = ETHASH_IO_FAIL;
+
+    // assert directory exists
+    int rc = _mkdir(dirname);
+    if (rc == -1 && errno != EEXIST) {
+        goto end;
+    }
+
+    char *memofile = ethash_io_create_filename(dirname, DAG_MEMO_NAME, sizeof(DAG_MEMO_NAME));
+    if (!memofile) {
+        goto end;
+    }
+
+    // try to open memo file
+    FILE *f = fopen(memofile, "rb");
+    if (!f) {
+        // file does not exist, so no checking happens. All is fine.
+        ret = ETHASH_IO_MEMO_MISMATCH;
+        goto free_memo;
+    }
+
+    if (fread(read_buffer, 1, DAG_MEMO_BYTESIZE, f) != DAG_MEMO_BYTESIZE) {
+        goto close;
+    }
+
+    ethash_io_serialize_info(ETHASH_REVISION, seedhash, expect_buffer);
+    if (memcmp(read_buffer, expect_buffer, DAG_MEMO_BYTESIZE) != 0) {
+        // we have different memo contents so delete the memo file
+        if (_unlink(memofile) != 0) {
+            goto close;
+        }
+        ret = ETHASH_IO_MEMO_MISMATCH;
+    }
+
+    ret = ETHASH_IO_MEMO_MATCH;
+
+close:
+    fclose(f);
+free_memo:
+    free(memofile);
+end:
+    return ret;
+}

libethcore/Common.cpp

@@ -34,7 +34,7 @@ namespace eth
 {
 
 const unsigned c_ethashVersion = c_ethashRevision;
-const unsigned c_protocolVersion = 58;
+const unsigned c_protocolVersion = 59;
 const unsigned c_databaseBaseVersion = 8;
 #if ETH_FATDB
 const unsigned c_databaseVersionModifier = 1;

libethcore/Ethasher.cpp

@@ -41,7 +41,23 @@ using namespace eth;
 
 Ethasher* dev::eth::Ethasher::s_this = nullptr;
 
-bytes const& Ethasher::cache(BlockInfo const& _header)
+Ethasher::~Ethasher()
+{
+	while (!m_caches.empty())
+		killCache(m_caches.begin()->first);
+}
+
+void Ethasher::killCache(h256 const& _s)
+{
+	RecursiveGuard l(x_this);
+	if (m_caches.count(_s))
+	{
+		ethash_delete_light(m_caches.at(_s));
+		m_caches.erase(_s);
+	}
+}
+
+void const* Ethasher::cache(BlockInfo const& _header)
 {
 	RecursiveGuard l(x_this);
 	if (_header.number > c_ethashEpochLength * 2048)
@@ -54,8 +70,7 @@ bytes const& Ethasher::cache(BlockInfo const& _header)
 	if (!m_caches.count(_header.seedHash()))
 	{
 		ethash_params p = params((unsigned)_header.number);
-		m_caches[_header.seedHash()].resize(p.cache_size);
+		m_caches[_header.seedHash()] = ethash_new_light(&p, _header.seedHash().data());
-		ethash_prep_light(m_caches[_header.seedHash()].data(), &p, _header.seedHash().data());
 	}
 	return m_caches[_header.seedHash()];
 }
@@ -84,7 +99,7 @@ bytesConstRef Ethasher::full(BlockInfo const& _header)
 			ethash_params p = params((unsigned)_header.number);
 			m_fulls[_header.seedHash()] = bytesRef(new byte[p.full_size], p.full_size);
 			auto c = cache(_header);
-			ethash_prep_full(m_fulls[_header.seedHash()].data(), &p, c.data());
+			ethash_prep_full(m_fulls[_header.seedHash()].data(), &p, c);
 			writeFile(memoFile, m_fulls[_header.seedHash()]);
 			writeFile(memoFile + ".info", info);
 		}
@@ -112,22 +127,42 @@ bool Ethasher::verify(BlockInfo const& _header)
 
 	h256 boundary = u256((bigint(1) << 256) / _header.difficulty);
 
-	// should be equivalent to:
+	bool quick = ethash_quick_check_difficulty(
-	auto r = eval(_header);
-	return r.mixHash == _header.mixHash && r.value <= boundary;
-
-	return ethash_quick_check_difficulty(
 		_header.headerHash(WithoutNonce).data(),
 		(uint64_t)(u64)_header.nonce,
 		_header.mixHash.data(),
 		boundary.data());
+
+#if !ETH_DEBUG
+	if (!quick)
+		return false;
+#endif
+
+	auto result = eval(_header);
+	bool slow = result.value <= boundary && result.mixHash == _header.mixHash;
+
+#if ETH_DEBUG
+	if (!quick && slow)
+	{
+		cwarn << "WARNING: evaluated result gives true whereas ethash_quick_check_difficulty gives false.";
+		cwarn << "headerHash:" << _header.headerHash(WithoutNonce);
+		cwarn << "nonce:" << _header.nonce;
+		cwarn << "mixHash:" << _header.mixHash;
+		cwarn << "difficulty:" << _header.difficulty;
+		cwarn << "boundary:" << boundary;
+		cwarn << "result.value:" << result.value;
+		cwarn << "result.mixHash:" << result.mixHash;
+	}
+#endif
+
+	return slow;
 }
 
 Ethasher::Result Ethasher::eval(BlockInfo const& _header, Nonce const& _nonce)
 {
 	auto p = Ethasher::params(_header);
 	ethash_return_value r;
-	ethash_compute_light(&r, Ethasher::get()->cache(_header).data(), &p, _header.headerHash(WithoutNonce).data(), (uint64_t)(u64)_nonce);
+	ethash_compute_light(&r, Ethasher::get()->cache(_header), &p, _header.headerHash(WithoutNonce).data(), (uint64_t)(u64)_nonce);
 //	cdebug << "Ethasher::eval sha3(cache):" << sha3(Ethasher::get()->cache(_header)) << "hh:" << _header.headerHash(WithoutNonce) << "nonce:" << _nonce << " => " << h256(r.result, h256::ConstructFromPointer);
 	return Result{h256(r.result, h256::ConstructFromPointer), h256(r.mix_hash, h256::ConstructFromPointer)};
 }

libethcore/Ethasher.h

@@ -30,21 +30,8 @@
 #include <libdevcore/Log.h>
 #include <libdevcrypto/SHA3.h>
 #include <libethash/ethash.h>		// TODO: REMOVE once everything merged into this class and an opaque API can be provided.
-static const unsigned c_ethashRevision = REVISION;
+static const unsigned c_ethashRevision = ETHASH_REVISION;
-static const unsigned c_ethashEpochLength = EPOCH_LENGTH;
+static const unsigned c_ethashEpochLength = ETHASH_EPOCH_LENGTH;
-#undef REVISION
-#undef DATASET_BYTES_INIT
-#undef DATASET_BYTES_GROWTH
-#undef CACHE_BYTES_INIT
-#undef CACHE_BYTES_GROWTH
-#undef DAGSIZE_BYTES_INIT
-#undef DAG_GROWTH
-#undef EPOCH_LENGTH
-#undef MIX_BYTES
-#undef HASH_BYTES
-#undef DATASET_PARENTS
-#undef CACHE_ROUNDS
-#undef ACCESSES
 #include "Common.h"
 #include "BlockInfo.h"
 
@@ -57,10 +44,14 @@ class Ethasher
 {
 public:
 	Ethasher() {}
+	~Ethasher();
 
 	static Ethasher* get() { if (!s_this) s_this = new Ethasher(); return s_this; }
 
-	bytes const& cache(BlockInfo const& _header);
+	using LightType = void const*;
+	using FullType = void const*;
+
+	LightType cache(BlockInfo const& _header);
 	bytesConstRef full(BlockInfo const& _header);
 	static ethash_params params(BlockInfo const& _header);
 	static ethash_params params(unsigned _n);
@@ -104,9 +95,11 @@ public:
 	};
 
 private:
+	void killCache(h256 const& _s);
+
 	static Ethasher* s_this;
 	RecursiveMutex x_this;
-	std::map<h256, bytes> m_caches;
+	std::map<h256, LightType> m_caches;
 	std::map<h256, bytesRef> m_fulls;
 };
 

libethcore/Exceptions.h

@@ -38,6 +38,7 @@ using errinfo_difficulty = boost::error_info<struct tag_difficulty, u256>;
 using BadFieldError = boost::tuple<errinfo_field, errinfo_data>;
 
 struct DatabaseAlreadyOpen: virtual dev::Exception {};
+struct NotEnoughAvailableSpace: virtual dev::Exception {};
 struct NotEnoughCash: virtual dev::Exception {};
 struct GasPriceTooLow: virtual dev::Exception {};
 struct BlockGasLimitReached: virtual dev::Exception {};

libethcore/Params.h

@@ -29,15 +29,15 @@ namespace eth
 {
 
 //--- BEGIN: AUTOGENERATED FROM /feeStructure.json
-extern u256 const c_genesisDifficulty;
-extern u256 const c_maximumExtraDataSize;
-extern u256 const c_epochDuration;
 extern u256 const c_genesisGasLimit;
 extern u256 const c_minGasLimit;
 extern u256 const c_gasLimitBoundDivisor;
+extern u256 const c_genesisDifficulty;
 extern u256 const c_minimumDifficulty;
 extern u256 const c_difficultyBoundDivisor;
 extern u256 const c_durationLimit;
+extern u256 const c_maximumExtraDataSize;
+extern u256 const c_epochDuration;
 extern u256 const c_stackLimit;
 
 extern u256 const c_tierStepGas[8];			///< Once per operation, for a selection of them.

libethereum/ABI.cpp

@@ -0,0 +1,27 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file ABI.cpp
+ * @author Gav Wood <i@gavwood.com>
+ * @date 2014
+ */
+
+#include "ABI.h"
+
+using namespace std;
+using namespace dev;
+using namespace dev::eth;
+

libethereum/ABI.h

@@ -0,0 +1,64 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file ABI.h
+ * @author Gav Wood <i@gavwood.com>
+ * @date 2014
+ */
+
+#pragma once
+
+#include <libdevcore/Common.h>
+#include <libdevcore/FixedHash.h>
+#include <libdevcore/CommonData.h>
+#include <libdevcrypto/SHA3.h>
+
+namespace dev
+{
+namespace eth
+{
+
+template <class T> struct ABISerialiser {};
+template <unsigned N> struct ABISerialiser<FixedHash<N>> { static bytes serialise(FixedHash<N> const& _t) { static_assert(N <= 32, "Cannot serialise hash > 32 bytes."); static_assert(N > 0, "Cannot serialise zero-length hash."); return bytes(32 - N, 0) + _t.asBytes(); } };
+template <> struct ABISerialiser<u256> { static bytes serialise(u256 const& _t) { return h256(_t).asBytes(); } };
+template <> struct ABISerialiser<u160> { static bytes serialise(u160 const& _t) { return bytes(12, 0) + h160(_t).asBytes(); } };
+template <> struct ABISerialiser<string32> { static bytes serialise(string32 const& _t) { return bytesConstRef((byte const*)_t.data(), 32).toBytes(); } };
+
+inline bytes abiInAux() { return {}; }
+template <class T, class ... U> bytes abiInAux(T const& _t, U const& ... _u)
+{
+	return ABISerialiser<T>::serialise(_t) + abiInAux(_u ...);
+}
+
+template <class ... T> bytes abiIn(std::string _id, T const& ... _t)
+{
+	return sha3(_id).ref().cropped(0, 4).toBytes() + abiInAux(_t ...);
+}
+
+template <class T> struct ABIDeserialiser {};
+template <unsigned N> struct ABIDeserialiser<FixedHash<N>> { static FixedHash<N> deserialise(bytesConstRef& io_t) { static_assert(N <= 32, "Parameter sizes must be at most 32 bytes."); FixedHash<N> ret; io_t.cropped(32 - N, N).populate(ret.ref()); io_t = io_t.cropped(32); return ret; } };
+template <> struct ABIDeserialiser<u256> { static u256 deserialise(bytesConstRef& io_t) { u256 ret = fromBigEndian<u256>(io_t.cropped(0, 32)); io_t = io_t.cropped(32); return ret; } };
+template <> struct ABIDeserialiser<u160> { static u160 deserialise(bytesConstRef& io_t) { u160 ret = fromBigEndian<u160>(io_t.cropped(12, 20)); io_t = io_t.cropped(32); return ret; } };
+template <> struct ABIDeserialiser<string32> { static string32 deserialise(bytesConstRef& io_t) { string32 ret; io_t.cropped(0, 32).populate(bytesRef((byte*)ret.data(), 32)); io_t = io_t.cropped(32); return ret; } };
+
+template <class T> T abiOut(bytes const& _data)
+{
+	bytesConstRef o(&_data);
+	return ABIDeserialiser<T>::deserialise(o);
+}
+
+}
+}

libethereum/BlockChain.cpp

@@ -63,7 +63,7 @@ std::ostream& dev::eth::operator<<(std::ostream& _out, BlockChain const& _bc)
 	return _out;
 }
 
-ldb::Slice dev::eth::toSlice(h256 _h, unsigned _sub)
+ldb::Slice dev::eth::toSlice(h256 const& _h, unsigned _sub)
 {
 #if ALL_COMPILERS_ARE_CPP11_COMPLIANT
 	static thread_local h256 h = _h ^ sha3(h256(u256(_sub)));
@@ -131,10 +131,19 @@ void BlockChain::open(std::string _path, bool _killExisting)
 	o.create_if_missing = true;
 	ldb::DB::Open(o, _path + "/blocks", &m_blocksDB);
 	ldb::DB::Open(o, _path + "/details", &m_extrasDB);
-	if (!m_blocksDB)
+	if (!m_blocksDB || !m_extrasDB)
-		BOOST_THROW_EXCEPTION(DatabaseAlreadyOpen());
+	{
-	if (!m_extrasDB)
+		if (boost::filesystem::space(_path + "/blocks").available < 1024)
+		{
+			cwarn << "Not enough available space found on hard drive. Please free some up and then re-run. Bailing.";
+			BOOST_THROW_EXCEPTION(NotEnoughAvailableSpace());
+		}
+		else
+		{
+			cwarn << "Database already open. You appear to have another instance of ethereum running. Bailing.";
 			BOOST_THROW_EXCEPTION(DatabaseAlreadyOpen());
+		}
+	}
 
 	if (!details(m_genesisHash))
 	{
@@ -184,6 +193,19 @@ inline string toString(h256s const& _bs)
 	return out.str();
 }
 
+LastHashes BlockChain::lastHashes(unsigned _n) const
+{
+	Guard l(x_lastLastHashes);
+	if (m_lastLastHashesNumber != _n || m_lastLastHashes.empty())
+	{
+		m_lastLastHashes.resize(256);
+		for (unsigned i = 0; i < 256; ++i)
+			m_lastLastHashes[i] = _n >= i ? numberHash(_n - i) : h256();
+		m_lastLastHashesNumber = _n;
+	}
+	return m_lastLastHashes;
+}
+
 h256s BlockChain::sync(BlockQueue& _bq, OverlayDB const& _stateDB, unsigned _max)
 {
 	_bq.tick(*this);
@@ -317,6 +339,13 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
 #endif
 		// All ok - insert into DB
 		{
+			// ensure parent is cached for later addition.
+			// TODO: this is a bit horrible would be better refactored into an enveloping UpgradableGuard
+			// together with an "ensureCachedWithUpdatableLock(l)" method.
+			// This is safe in practice since the caches don't get flushed nearly often enough to be
+			// done here.
+			details(bi.parentHash);
+
 			WriteGuard l(x_details);
 			m_details[newHash] = BlockDetails((unsigned)pd.number + 1, td, bi.parentHash, {});
 			m_details[bi.parentHash].children.push_back(newHash);
@@ -412,6 +441,9 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
 			WriteGuard l(x_lastBlockHash);
 			m_lastBlockHash = newHash;
 		}
+
+		noteCanonChanged();
+
 		m_extrasDB->Put(m_writeOptions, ldb::Slice("best"), ldb::Slice((char const*)&newHash, 32));
 		clog(BlockChainNote) << "   Imported and best" << td << ". Has" << (details(bi.parentHash).children.size() - 1) << "siblings. Route:" << toString(ret);
 		StructuredLogger::chainNewHead(
@@ -428,48 +460,52 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
 	return ret;
 }
 
-h256s BlockChain::treeRoute(h256 _from, h256 _to, h256* o_common, bool _pre, bool _post) const
+h256s BlockChain::treeRoute(h256 const& _from, h256 const& _to, h256* o_common, bool _pre, bool _post) const
 {
-	//	cdebug << "treeRoute" << _from.abridged() << "..." << _to.abridged();
+	cdebug << "treeRoute" << _from.abridged() << "..." << _to.abridged();
 	if (!_from || !_to)
 		return h256s();
 	h256s ret;
 	h256s back;
 	unsigned fn = details(_from).number;
 	unsigned tn = details(_to).number;
-	//	cdebug << "treeRoute" << fn << "..." << tn;
+	cdebug << "treeRoute" << fn << "..." << tn;
+	h256 from = _from;
 	while (fn > tn)
 	{
 		if (_pre)
-			ret.push_back(_from);
+			ret.push_back(from);
-		_from = details(_from).parent;
+		from = details(from).parent;
 		fn--;
-		//		cdebug << "from:" << fn << _from.abridged();
+		cdebug << "from:" << fn << _from.abridged();
 	}
+	h256 to = _to;
 	while (fn < tn)
 	{
 		if (_post)
-			back.push_back(_to);
+			back.push_back(to);
-		_to = details(_to).parent;
+		to = details(to).parent;
 		tn--;
-		//		cdebug << "to:" << tn << _to.abridged();
+		cdebug << "to:" << tn << _to.abridged();
 	}
-	while (_from != _to)
+	while (from != to)
 	{
-		assert(_from);
+		if (!from)
-		assert(_to);
+			assert(from);
-		_from = details(_from).parent;
+		if (!to)
-		_to = details(_to).parent;
+			assert(to);
+		from = details(from).parent;
+		to = details(to).parent;
 		if (_pre)
-			ret.push_back(_from);
+			ret.push_back(from);
 		if (_post)
-			back.push_back(_to);
+			back.push_back(to);
 		fn--;
 		tn--;
 		//		cdebug << "from:" << fn << _from.abridged() << "; to:" << tn << _to.abridged();
 	}
 	if (o_common)
-		*o_common = _from;
+		*o_common = from;
 	ret.reserve(ret.size() + back.size());
 	for (auto it = back.cbegin(); it != back.cend(); ++it)
 		ret.push_back(*it);
@@ -677,7 +713,7 @@ vector<unsigned> BlockChain::withBlockBloom(LogBloom const& _b, unsigned _earlie
 	return ret;
 }
 
-h256Set BlockChain::allUnclesFrom(h256 _parent) const
+h256Set BlockChain::allUnclesFrom(h256 const& _parent) const
 {
 	// Get all uncles cited given a parent (i.e. featured as uncles/main in parent, parent + 1, ... parent + 5).
 	h256Set ret;
@@ -692,7 +728,7 @@ h256Set BlockChain::allUnclesFrom(h256 _parent) const
 	return ret;
 }
 
-bool BlockChain::isKnown(h256 _hash) const
+bool BlockChain::isKnown(h256 const& _hash) const
 {
 	if (_hash == m_genesisHash)
 		return true;
@@ -706,7 +742,7 @@ bool BlockChain::isKnown(h256 _hash) const
 	return !!d.size();
 }
 
-bytes BlockChain::block(h256 _hash) const
+bytes BlockChain::block(h256 const& _hash) const
 {
 	if (_hash == m_genesisHash)
 		return m_genesisBlock;

libethereum/BlockChain.h

@@ -30,9 +30,10 @@
 #include <chrono>
 #include <libdevcore/Log.h>
 #include <libdevcore/Exceptions.h>
+#include <libdevcore/Guards.h>
 #include <libethcore/Common.h>
 #include <libethcore/BlockInfo.h>
-#include <libdevcore/Guards.h>
+#include <libevm/ExtVMFace.h>
 #include "BlockDetails.h"
 #include "Account.h"
 #include "Transaction.h"
@@ -61,10 +62,11 @@ struct BlockChainNote: public LogChannel { static const char* name() { return "=
 // TODO: Move all this Genesis stuff into Genesis.h/.cpp
 std::map<Address, Account> const& genesisState();
 
-ldb::Slice toSlice(h256 _h, unsigned _sub = 0);
+ldb::Slice toSlice(h256 const& _h, unsigned _sub = 0);
 
 using BlocksHash = std::map<h256, bytes>;
 using TransactionHashes = h256s;
+using UncleHashes = h256s;
 
 enum {
 	ExtraDetails = 0,
@@ -104,34 +106,42 @@ public:
 	h256s import(bytes const& _block, OverlayDB const& _stateDB);
 
 	/// Returns true if the given block is known (though not necessarily a part of the canon chain).
-	bool isKnown(h256 _hash) const;
+	bool isKnown(h256 const& _hash) const;
 
 	/// Get the familial details concerning a block (or the most recent mined if none given). Thread-safe.
-	BlockInfo info(h256 _hash) const { return BlockInfo(block(_hash)); }
+	BlockInfo info(h256 const& _hash) const { return BlockInfo(block(_hash)); }
 	BlockInfo info() const { return BlockInfo(block()); }
 
 	/// Get a block (RLP format) for the given hash (or the most recent mined if none given). Thread-safe.
-	bytes block(h256 _hash) const;
+	bytes block(h256 const& _hash) const;
 	bytes block() const { return block(currentHash()); }
 
 	/// Get the familial details concerning a block (or the most recent mined if none given). Thread-safe.
-	BlockDetails details(h256 _hash) const { return queryExtras<BlockDetails, ExtraDetails>(_hash, m_details, x_details, NullBlockDetails); }
+	BlockDetails details(h256 const& _hash) const { return queryExtras<BlockDetails, ExtraDetails>(_hash, m_details, x_details, NullBlockDetails); }
 	BlockDetails details() const { return details(currentHash()); }
 
 	/// Get the transactions' log blooms of a block (or the most recent mined if none given). Thread-safe.
-	BlockLogBlooms logBlooms(h256 _hash) const { return queryExtras<BlockLogBlooms, ExtraLogBlooms>(_hash, m_logBlooms, x_logBlooms, NullBlockLogBlooms); }
+	BlockLogBlooms logBlooms(h256 const& _hash) const { return queryExtras<BlockLogBlooms, ExtraLogBlooms>(_hash, m_logBlooms, x_logBlooms, NullBlockLogBlooms); }
 	BlockLogBlooms logBlooms() const { return logBlooms(currentHash()); }
 
 	/// Get the transactions' receipts of a block (or the most recent mined if none given). Thread-safe.
-	BlockReceipts receipts(h256 _hash) const { return queryExtras<BlockReceipts, ExtraReceipts>(_hash, m_receipts, x_receipts, NullBlockReceipts); }
+	BlockReceipts receipts(h256 const& _hash) const { return queryExtras<BlockReceipts, ExtraReceipts>(_hash, m_receipts, x_receipts, NullBlockReceipts); }
 	BlockReceipts receipts() const { return receipts(currentHash()); }
 
 	/// Get a list of transaction hashes for a given block. Thread-safe.
-	TransactionHashes transactionHashes(h256 _hash) const { auto b = block(_hash); RLP rlp(b); h256s ret; for (auto t: rlp[1]) ret.push_back(sha3(t.data())); return ret; }
+	TransactionHashes transactionHashes(h256 const& _hash) const { auto b = block(_hash); RLP rlp(b); h256s ret; for (auto t: rlp[1]) ret.push_back(sha3(t.data())); return ret; }
 	TransactionHashes transactionHashes() const { return transactionHashes(currentHash()); }
 
-	/// Get a list of transaction hashes for a given block. Thread-safe.
+	/// Get a list of uncle hashes for a given block. Thread-safe.
-	h256 numberHash(u256 _index) const { if (!_index) return genesisHash(); return queryExtras<BlockHash, ExtraBlockHash>(h256(_index), m_blockHashes, x_blockHashes, NullBlockHash).value; }
+	UncleHashes uncleHashes(h256 const& _hash) const { auto b = block(_hash); RLP rlp(b); h256s ret; for (auto t: rlp[2]) ret.push_back(sha3(t.data())); return ret; }
+	UncleHashes uncleHashes() const { return uncleHashes(currentHash()); }
+	
+	/// Get the hash for a given block's number.
+	h256 numberHash(unsigned _i) const { if (!_i) return genesisHash(); return queryExtras<BlockHash, ExtraBlockHash>(h256(u256(_i)), m_blockHashes, x_blockHashes, NullBlockHash).value; }
+
+	/// Get the last N hashes for a given block. (N is determined by the LastHashes type.)
+	LastHashes lastHashes() const { return lastHashes(number()); }
+	LastHashes lastHashes(unsigned _i) const;
 
 	/** Get the block blooms for a number of blocks. Thread-safe.
 	 * @returns the object pertaining to the blocks:
@@ -154,15 +164,15 @@ public:
 	std::vector<unsigned> withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest, unsigned _topLevel, unsigned _index) const;
 
 	/// Get a transaction from its hash. Thread-safe.
-	bytes transaction(h256 _transactionHash) const { TransactionAddress ta = queryExtras<TransactionAddress, ExtraTransactionAddress>(_transactionHash, m_transactionAddresses, x_transactionAddresses, NullTransactionAddress); if (!ta) return bytes(); return transaction(ta.blockHash, ta.index); }
+	bytes transaction(h256 const& _transactionHash) const { TransactionAddress ta = queryExtras<TransactionAddress, ExtraTransactionAddress>(_transactionHash, m_transactionAddresses, x_transactionAddresses, NullTransactionAddress); if (!ta) return bytes(); return transaction(ta.blockHash, ta.index); }
-	std::pair<h256, unsigned> transactionLocation(h256 _transactionHash) const { TransactionAddress ta = queryExtras<TransactionAddress, ExtraTransactionAddress>(_transactionHash, m_transactionAddresses, x_transactionAddresses, NullTransactionAddress); if (!ta) return std::pair<h256, unsigned>(h256(), 0); return std::make_pair(ta.blockHash, ta.index); }
+	std::pair<h256, unsigned> transactionLocation(h256 const& _transactionHash) const { TransactionAddress ta = queryExtras<TransactionAddress, ExtraTransactionAddress>(_transactionHash, m_transactionAddresses, x_transactionAddresses, NullTransactionAddress); if (!ta) return std::pair<h256, unsigned>(h256(), 0); return std::make_pair(ta.blockHash, ta.index); }
 
 	/// Get a block's transaction (RLP format) for the given block hash (or the most recent mined if none given) & index. Thread-safe.
-	bytes transaction(h256 _blockHash, unsigned _i) const { bytes b = block(_blockHash); return RLP(b)[1][_i].data().toBytes(); }
+	bytes transaction(h256 const& _blockHash, unsigned _i) const { bytes b = block(_blockHash); return RLP(b)[1][_i].data().toBytes(); }
 	bytes transaction(unsigned _i) const { return transaction(currentHash(), _i); }
 
 	/// Get a number for the given hash (or the most recent mined if none given). Thread-safe.
-	unsigned number(h256 _hash) const { return details(_hash).number; }
+	unsigned number(h256 const& _hash) const { return details(_hash).number; }
 	unsigned number() const { return number(currentHash()); }
 
 	/// Get a given block (RLP format). Thread-safe.
@@ -174,7 +184,7 @@ public:
 	/// Get all blocks not allowed as uncles given a parent (i.e. featured as uncles/main in parent, parent + 1, ... parent + 5).
 	/// @returns set including the header-hash of every parent (including @a _parent) up to and including generation +5
 	/// togther with all their quoted uncles.
-	h256Set allUnclesFrom(h256 _parent) const;
+	h256Set allUnclesFrom(h256 const& _parent) const;
 
 	/** @returns the hash of all blocks between @a _from and @a _to, all blocks are ordered first by a number of
 	 * blocks that are parent-to-child, then two sibling blocks, then a number of blocks that are child-to-parent.
@@ -190,7 +200,7 @@ public:
 	 * treeRoute(1b, 2a) == { 1b, 1a, 2a }; // *o_common == g
 	 * @endcode
 	 */
-	h256s treeRoute(h256 _from, h256 _to, h256* o_common = nullptr, bool _pre = true, bool _post = true) const;
+	h256s treeRoute(h256 const& _from, h256 const& _to, h256* o_common = nullptr, bool _pre = true, bool _post = true) const;
 
 	struct Statistics
 	{
@@ -215,7 +225,7 @@ private:
 	void open(std::string _path, bool _killExisting = false);
 	void close();
 
-	template<class T, unsigned N> T queryExtras(h256 _h, std::map<h256, T>& _m, boost::shared_mutex& _x, T const& _n) const
+	template<class T, unsigned N> T queryExtras(h256 const& _h, std::map<h256, T>& _m, boost::shared_mutex& _x, T const& _n) const
 	{
 		{
 			ReadGuard l(_x);
@@ -264,6 +274,11 @@ private:
 	void noteUsed(h256 const& _h, unsigned _extra = (unsigned)-1) const;
 	std::chrono::system_clock::time_point m_lastCollection;
 
+	void noteCanonChanged() const { Guard l(x_lastLastHashes); m_lastLastHashes.clear(); }
+	mutable Mutex x_lastLastHashes;
+	mutable LastHashes m_lastLastHashes;
+	mutable unsigned m_lastLastHashesNumber = (unsigned)-1;
+
 	void updateStats() const;
 	mutable Statistics m_lastStats;
 

libethereum/Client.cpp

@@ -186,11 +186,6 @@ void Client::doneWorking()
 	m_postMine = m_preMine;
 }
 
-void Client::flushTransactions()
-{
-	doWork();
-}
-
 void Client::killChain()
 {
 	bool wasMining = isMining();
@@ -249,117 +244,29 @@ void Client::clearPending()
 	noteChanged(changeds);
 }
 
-unsigned Client::installWatch(h256 _h, Reaping _r)
-{
-	unsigned ret;
-	{
-		Guard l(m_filterLock);
-		ret = m_watches.size() ? m_watches.rbegin()->first + 1 : 0;
-		m_watches[ret] = ClientWatch(_h, _r);
-		cwatch << "+++" << ret << _h.abridged();
-	}
-	auto ch = logs(ret);
-	if (ch.empty())
-		ch.push_back(InitialChange);
-	{
-		Guard l(m_filterLock);
-		swap(m_watches[ret].changes, ch);
-	}
-	return ret;
-}
-
-unsigned Client::installWatch(LogFilter const& _f, Reaping _r)
-{
-	h256 h = _f.sha3();
-	{
-		Guard l(m_filterLock);
-		if (!m_filters.count(h))
-		{
-			cwatch << "FFF" << _f << h.abridged();
-			m_filters.insert(make_pair(h, _f));
-		}
-	}
-	return installWatch(h, _r);
-}
-
-bool Client::uninstallWatch(unsigned _i)
-{
-	cwatch << "XXX" << _i;
-
-	Guard l(m_filterLock);
-
-	auto it = m_watches.find(_i);
-	if (it == m_watches.end())
-		return false;
-	auto id = it->second.id;
-	m_watches.erase(it);
-
-	auto fit = m_filters.find(id);
-	if (fit != m_filters.end())
-		if (!--fit->second.refCount)
-		{
-			cwatch << "*X*" << fit->first << ":" << fit->second.filter;
-			m_filters.erase(fit);
-		}
-	return true;
-}
-
 void Client::noteChanged(h256Set const& _filters)
 {
-	Guard l(m_filterLock);
+	Guard l(x_filtersWatches);
 	if (_filters.size())
 		cnote << "noteChanged(" << _filters << ")";
 	// accrue all changes left in each filter into the watches.
-	for (auto& i: m_watches)
+	for (auto& w: m_watches)
-		if (_filters.count(i.second.id))
+		if (_filters.count(w.second.id))
 		{
-			cwatch << "!!!" << i.first << i.second.id;
+			cwatch << "!!!" << w.first << w.second.id;
-			if (m_filters.count(i.second.id))
+			if (m_filters.count(w.second.id))	// Normal filtering watch
-				i.second.changes += m_filters.at(i.second.id).changes;
+				w.second.changes += m_filters.at(w.second.id).changes;
-			else
+			else								// Special ('pending'/'latest') watch
-				i.second.changes.push_back(LocalisedLogEntry(SpecialLogEntry, 0));
+				w.second.changes.push_back(LocalisedLogEntry(SpecialLogEntry, 0));
 		}
 	// clear the filters now.
 	for (auto& i: m_filters)
 		i.second.changes.clear();
 }
 
-LocalisedLogEntries Client::peekWatch(unsigned _watchId) const
-{
-	Guard l(m_filterLock);
-
-#if ETH_DEBUG
-	cdebug << "peekWatch" << _watchId;
-#endif
-	auto& w = m_watches.at(_watchId);
-#if ETH_DEBUG
-	cdebug << "lastPoll updated to " << chrono::duration_cast<chrono::seconds>(chrono::system_clock::now().time_since_epoch()).count();
-#endif
-	w.lastPoll = chrono::system_clock::now();
-	return w.changes;
-}
-
-LocalisedLogEntries Client::checkWatch(unsigned _watchId)
-{
-	Guard l(m_filterLock);
-	LocalisedLogEntries ret;
-
-#if ETH_DEBUG && 0
-	cdebug << "checkWatch" << _watchId;
-#endif
-	auto& w = m_watches.at(_watchId);
-#if ETH_DEBUG && 0
-	cdebug << "lastPoll updated to " << chrono::duration_cast<chrono::seconds>(chrono::system_clock::now().time_since_epoch()).count();
-#endif
-	std::swap(ret, w.changes);
-	w.lastPoll = chrono::system_clock::now();
-
-	return ret;
-}
-
 void Client::appendFromNewPending(TransactionReceipt const& _receipt, h256Set& io_changed, h256 _transactionHash)
 {
-	Guard l(m_filterLock);
+	Guard l(x_filtersWatches);
 	for (pair<h256 const, InstalledFilter>& i: m_filters)
 		if (i.second.filter.envelops(RelativeBlock::Pending, m_bc.number() + 1))
 		{
@@ -381,7 +288,7 @@ void Client::appendFromNewBlock(h256 const& _block, h256Set& io_changed)
 	auto d = m_bc.info(_block);
 	auto br = m_bc.receipts(_block);
 
-	Guard l(m_filterLock);
+	Guard l(x_filtersWatches);
 	for (pair<h256 const, InstalledFilter>& i: m_filters)
 		if (i.second.filter.envelops(RelativeBlock::Latest, d.number) && i.second.filter.matches(d.logBloom))
 			// acceptable number & looks like block may contain a matching log entry.
@@ -475,68 +382,6 @@ void Client::setupState(State& _s)
 		_s.commitToMine(m_bc);
 }
 
-void Client::submitTransaction(Secret _secret, u256 _value, Address _dest, bytes const& _data, u256 _gas, u256 _gasPrice)
-{
-	startWorking();
-
-	u256 n;
-	{
-		ReadGuard l(x_stateDB);
-		n = m_postMine.transactionsFrom(toAddress(_secret));
-	}
-	Transaction t(_value, _gasPrice, _gas, _dest, _data, n, _secret);
-//	cdebug << "Nonce at " << toAddress(_secret) << " pre:" << m_preMine.transactionsFrom(toAddress(_secret)) << " post:" << m_postMine.transactionsFrom(toAddress(_secret));
-	StructuredLogger::transactionReceived(t.sha3().abridged(), t.sender().abridged());
-	cnote << "New transaction " << t;
-	m_tq.attemptImport(t.rlp());
-}
-
-ExecutionResult Client::call(Secret _secret, u256 _value, Address _dest, bytes const& _data, u256 _gas, u256 _gasPrice, BlockNumber _blockNumber)
-{
-	ExecutionResult ret;
-	try
-	{
-		u256 n;
-		State temp;
-	//	cdebug << "Nonce at " << toAddress(_secret) << " pre:" << m_preMine.transactionsFrom(toAddress(_secret)) << " post:" << m_postMine.transactionsFrom(toAddress(_secret));
-		{
-			ReadGuard l(x_stateDB);
-			temp = asOf(_blockNumber);
-			n = temp.transactionsFrom(toAddress(_secret));
-		}
-		Transaction t(_value, _gasPrice, _gas, _dest, _data, n, _secret);
-		ret = temp.execute(m_bc, t.rlp(), Permanence::Reverted);
-	}
-	catch (...)
-	{
-		// TODO: Some sort of notification of failure.
-	}
-	return ret;
-}
-
-ExecutionResult Client::create(Secret _secret, u256 _value, bytes const& _data, u256 _gas, u256 _gasPrice, BlockNumber _blockNumber)
-{
-	ExecutionResult ret;
-	try
-	{
-		u256 n;
-		State temp;
-	//	cdebug << "Nonce at " << toAddress(_secret) << " pre:" << m_preMine.transactionsFrom(toAddress(_secret)) << " post:" << m_postMine.transactionsFrom(toAddress(_secret));
-		{
-			ReadGuard l(x_stateDB);
-			temp = asOf(_blockNumber);
-			n = temp.transactionsFrom(toAddress(_secret));
-		}
-		Transaction t(_value, _gasPrice, _gas, _data, n, _secret);
-		ret = temp.execute(m_bc, t.rlp(), Permanence::Reverted);
-	}
-	catch (...)
-	{
-		// TODO: Some sort of notification of failure.
-	}
-	return ret;
-}
-
 ExecutionResult Client::call(Address _dest, bytes const& _data, u256 _gas, u256 _value, u256 _gasPrice)
 {
 	ExecutionResult ret;
@@ -547,6 +392,7 @@ ExecutionResult Client::call(Address _dest, bytes const& _data, u256 _gas, u256
 		{
 			ReadGuard l(x_stateDB);
 			temp = m_postMine;
+			temp.addBalance(Address(), _value + _gasPrice * _gas);
 		}
 		Executive e(temp, LastHashes(), 0);
 		if (!e.call(_dest, _dest, Address(), _value, _gasPrice, &_data, _gas, Address()))
@@ -560,28 +406,6 @@ ExecutionResult Client::call(Address _dest, bytes const& _data, u256 _gas, u256
 	return ret;
 }
 
-Address Client::submitTransaction(Secret _secret, u256 _endowment, bytes const& _init, u256 _gas, u256 _gasPrice)
-{
-	startWorking();
-
-	u256 n;
-	{
-		ReadGuard l(x_stateDB);
-		n = m_postMine.transactionsFrom(toAddress(_secret));
-	}
-	Transaction t(_endowment, _gasPrice, _gas, _init, n, _secret);
-	cnote << "New transaction " << t;
-	m_tq.attemptImport(t.rlp());
-	return right160(sha3(rlpList(t.sender(), t.nonce())));
-}
-
-void Client::inject(bytesConstRef _rlp)
-{
-	startWorking();
-
-	m_tq.attemptImport(_rlp);
-}
-
 pair<h256, u256> Client::getWork()
 {
 	Guard l(x_remoteMiner);
@@ -711,7 +535,7 @@ void Client::doWork()
 		// watches garbage collection
 		vector<unsigned> toUninstall;
 		{
-			Guard l(m_filterLock);
+			Guard l(x_filtersWatches);
 			for (auto key: keysOf(m_watches))
 				if (m_watches[key].lastPoll != chrono::system_clock::time_point::max() && chrono::system_clock::now() - m_watches[key].lastPoll > chrono::seconds(20))
 				{
@@ -729,25 +553,15 @@ void Client::doWork()
 	}
 }
 
-unsigned Client::numberOf(int _n) const
+State Client::asOf(h256 const& _block) const
 {
-	if (_n > 0)
+	ReadGuard l(x_stateDB);
-		return _n;
+	return State(m_stateDB, bc(), _block);
-	else if (_n == GenesisBlock)
-		return 0;
-	else
-		return m_bc.details().number + max(-(int)m_bc.details().number, 1 + _n);
 }
 
-State Client::asOf(unsigned _h) const
+void Client::prepareForTransaction()
 {
-	ReadGuard l(x_stateDB);
+	startWorking();
-	if (_h == PendingBlock)
-		return m_postMine;
-	else if (_h == LatestBlock)
-		return m_preMine;
-	else
-		return State(m_stateDB, m_bc, m_bc.numberHash(numberOf(_h)));
 }
 
 State Client::state(unsigned _txi, h256 _block) const
@@ -768,183 +582,14 @@ eth::State Client::state(unsigned _txi) const
 	return m_postMine.fromPending(_txi);
 }
 
-StateDiff Client::diff(unsigned _txi, BlockNumber _block) const
+void Client::inject(bytesConstRef _rlp)
-{
-	State st = asOf(_block);
-	return st.fromPending(_txi).diff(st.fromPending(_txi + 1));
-}
-
-StateDiff Client::diff(unsigned _txi, h256 _block) const
-{
-	State st = state(_block);
-	return st.fromPending(_txi).diff(st.fromPending(_txi + 1));
-}
-
-std::vector<Address> Client::addresses(BlockNumber _block) const
-{
-	vector<Address> ret;
-	for (auto const& i: asOf(_block).addresses())
-		ret.push_back(i.first);
-	return ret;
-}
-
-u256 Client::balanceAt(Address _a, BlockNumber _block) const
-{
-	return asOf(_block).balance(_a);
-}
-
-std::map<u256, u256> Client::storageAt(Address _a, BlockNumber _block) const
-{
-	return asOf(_block).storage(_a);
-}
-
-u256 Client::countAt(Address _a, BlockNumber _block) const
-{
-	return asOf(_block).transactionsFrom(_a);
-}
-
-u256 Client::stateAt(Address _a, u256 _l, BlockNumber _block) const
-{
-	return asOf(_block).storage(_a, _l);
-}
-
-bytes Client::codeAt(Address _a, BlockNumber _block) const
-{
-	return asOf(_block).code(_a);
-}
-
-Transaction Client::transaction(h256 _transactionHash) const
-{
-	return Transaction(m_bc.transaction(_transactionHash), CheckSignature::Range);
-}
-
-Transaction Client::transaction(h256 _blockHash, unsigned _i) const
-{
-	auto bl = m_bc.block(_blockHash);
-	RLP b(bl);
-	if (_i < b[1].itemCount())
-		return Transaction(b[1][_i].data(), CheckSignature::Range);
-	else
-		return Transaction();
-}
-
-BlockInfo Client::uncle(h256 _blockHash, unsigned _i) const
-{
-	auto bl = m_bc.block(_blockHash);
-	RLP b(bl);
-	if (_i < b[2].itemCount())
-		return BlockInfo::fromHeader(b[2][_i].data());
-	else
-		return BlockInfo();
-}
-
-unsigned Client::transactionCount(h256 _blockHash) const
-{
-	auto bl = m_bc.block(_blockHash);
-	RLP b(bl);
-	return b[1].itemCount();
-}
-
-unsigned Client::uncleCount(h256 _blockHash) const
-{
-	auto bl = m_bc.block(_blockHash);
-	RLP b(bl);
-	return b[2].itemCount();
-}
-
-Transactions Client::transactions(h256 _blockHash) const
-{
-	auto bl = m_bc.block(_blockHash);
-	RLP b(bl);
-	Transactions res;
-	for (unsigned i = 0; i < b[1].itemCount(); i++)
-		res.emplace_back(b[1][i].data(), CheckSignature::Range);
-	return res;
-}
-
-TransactionHashes Client::transactionHashes(h256 _blockHash) const
-{
-	return m_bc.transactionHashes(_blockHash);
-}
-
-LocalisedLogEntries Client::logs(unsigned _watchId) const
-{
-	LogFilter f;
-	try
-	{
-		Guard l(m_filterLock);
-		f = m_filters.at(m_watches.at(_watchId).id).filter;
-	}
-	catch (...)
-	{
-		return LocalisedLogEntries();
-	}
-	return logs(f);
-}
-
-LocalisedLogEntries Client::logs(LogFilter const& _f) const
 {
-	LocalisedLogEntries ret;
+	startWorking();
-	unsigned begin = min<unsigned>(m_bc.number() + 1, (unsigned)_f.latest());
-	unsigned end = min(m_bc.number(), min(begin, (unsigned)_f.earliest()));
 	
-	// Handle pending transactions differently as they're not on the block chain.
+	m_tq.attemptImport(_rlp);
-	if (begin > m_bc.number())
-	{
-		ReadGuard l(x_stateDB);
-		for (unsigned i = 0; i < m_postMine.pending().size(); ++i)
-		{
-			// Might have a transaction that contains a matching log.
-			TransactionReceipt const& tr = m_postMine.receipt(i);
-			auto sha3 = m_postMine.pending()[i].sha3();
-			LogEntries le = _f.matches(tr);
-			if (le.size())
-				for (unsigned j = 0; j < le.size(); ++j)
-					ret.insert(ret.begin(), LocalisedLogEntry(le[j], begin, sha3));
-		}
-		begin = m_bc.number();
 }
 
-	set<unsigned> matchingBlocks;
+void Client::flushTransactions()
-	for (auto const& i: _f.bloomPossibilities())
-		for (auto u: m_bc.withBlockBloom(i, end, begin))
-			matchingBlocks.insert(u);
-
-#if ETH_DEBUG
-	unsigned falsePos = 0;
-#endif
-	for (auto n: matchingBlocks)
-	{
-#if ETH_DEBUG
-		int total = 0;
-#endif
-		auto h = m_bc.numberHash(n);
-		auto receipts = m_bc.receipts(h).receipts;
-		for (size_t i = 0; i < receipts.size(); i++)
-		{
-			TransactionReceipt receipt = receipts[i];
-			if (_f.matches(receipt.bloom()))
-			{
-				auto info = m_bc.info(h);
-				auto h = transaction(info.hash, i).sha3();
-				LogEntries le = _f.matches(receipt);
-				if (le.size())
 {
-#if ETH_DEBUG
+	doWork();
-					total += le.size();
-#endif
-					for (unsigned j = 0; j < le.size(); ++j)
-						ret.insert(ret.begin(), LocalisedLogEntry(le[j], n, h));
-				}
-			}
-#if ETH_DEBUG
-			if (!total)
-				falsePos++;
-#endif
-		}
-	}
-#if ETH_DEBUG
-	cdebug << matchingBlocks.size() << "searched from" << (end - begin) << "skipped; " << falsePos << "false +ves";
-#endif
-	return ret;
 }

libethereum/Client.h

@@ -40,9 +40,9 @@
 #include "TransactionQueue.h"
 #include "State.h"
 #include "CommonNet.h"
-#include "LogFilter.h"
 #include "Miner.h"
-#include "Interface.h"
+#include "ABI.h"
+#include "ClientBase.h"
 
 namespace dev
 {
@@ -72,71 +72,6 @@ private:
 	std::string m_path;
 };
 
-static const int GenesisBlock = INT_MIN;
-
-struct InstalledFilter
-{
-	InstalledFilter(LogFilter const& _f): filter(_f) {}
-
-	LogFilter filter;
-	unsigned refCount = 1;
-	LocalisedLogEntries changes;
-};
-
-static const h256 PendingChangedFilter = u256(0);
-static const h256 ChainChangedFilter = u256(1);
-
-static const LogEntry SpecialLogEntry = LogEntry(Address(), h256s(), bytes());
-static const LocalisedLogEntry InitialChange(SpecialLogEntry, 0);
-
-struct ClientWatch
-{
-	ClientWatch(): lastPoll(std::chrono::system_clock::now()) {}
-	explicit ClientWatch(h256 _id, Reaping _r): id(_id), lastPoll(_r == Reaping::Automatic ? std::chrono::system_clock::now() : std::chrono::system_clock::time_point::max()) {}
-
-	h256 id;
-	LocalisedLogEntries changes = LocalisedLogEntries{ InitialChange };
-	mutable std::chrono::system_clock::time_point lastPoll = std::chrono::system_clock::now();
-};
-
-struct WatchChannel: public LogChannel { static const char* name() { return "(o)"; } static const int verbosity = 7; };
-#define cwatch dev::LogOutputStream<dev::eth::WatchChannel, true>()
-struct WorkInChannel: public LogChannel { static const char* name() { return ">W>"; } static const int verbosity = 16; };
-struct WorkOutChannel: public LogChannel { static const char* name() { return "<W<"; } static const int verbosity = 16; };
-struct WorkChannel: public LogChannel { static const char* name() { return "-W-"; } static const int verbosity = 16; };
-#define cwork dev::LogOutputStream<dev::eth::WorkChannel, true>()
-#define cworkin dev::LogOutputStream<dev::eth::WorkInChannel, true>()
-#define cworkout dev::LogOutputStream<dev::eth::WorkOutChannel, true>()
-
-template <class T> struct ABISerialiser {};
-template <unsigned N> struct ABISerialiser<FixedHash<N>> { static bytes serialise(FixedHash<N> const& _t) { static_assert(N <= 32, "Cannot serialise hash > 32 bytes."); static_assert(N > 0, "Cannot serialise zero-length hash."); return bytes(32 - N, 0) + _t.asBytes(); } };
-template <> struct ABISerialiser<u256> { static bytes serialise(u256 const& _t) { return h256(_t).asBytes(); } };
-template <> struct ABISerialiser<u160> { static bytes serialise(u160 const& _t) { return bytes(12, 0) + h160(_t).asBytes(); } };
-template <> struct ABISerialiser<string32> { static bytes serialise(string32 const& _t) { return bytesConstRef((byte const*)_t.data(), 32).toBytes(); } };
-
-inline bytes abiInAux() { return {}; }
-template <class T, class ... U> bytes abiInAux(T const& _t, U const& ... _u)
-{
-	return ABISerialiser<T>::serialise(_t) + abiInAux(_u ...);
-}
-
-template <class ... T> bytes abiIn(std::string _id, T const& ... _t)
-{
-	return sha3(_id).ref().cropped(0, 4).toBytes() + abiInAux(_t ...);
-}
-
-template <class T> struct ABIDeserialiser {};
-template <unsigned N> struct ABIDeserialiser<FixedHash<N>> { static FixedHash<N> deserialise(bytesConstRef& io_t) { static_assert(N <= 32, "Parameter sizes must be at most 32 bytes."); FixedHash<N> ret; io_t.cropped(32 - N, N).populate(ret.ref()); io_t = io_t.cropped(32); return ret; } };
-template <> struct ABIDeserialiser<u256> { static u256 deserialise(bytesConstRef& io_t) { u256 ret = fromBigEndian<u256>(io_t.cropped(0, 32)); io_t = io_t.cropped(32); return ret; } };
-template <> struct ABIDeserialiser<u160> { static u160 deserialise(bytesConstRef& io_t) { u160 ret = fromBigEndian<u160>(io_t.cropped(12, 20)); io_t = io_t.cropped(32); return ret; } };
-template <> struct ABIDeserialiser<string32> { static string32 deserialise(bytesConstRef& io_t) { string32 ret; io_t.cropped(0, 32).populate(bytesRef((byte*)ret.data(), 32)); io_t = io_t.cropped(32); return ret; } };
-
-template <class T> T abiOut(bytes const& _data)
-{
-	bytesConstRef o(&_data);
-	return ABIDeserialiser<T>::deserialise(o);
-}
-
 class RemoteMiner: public Miner
 {
 public:
@@ -175,14 +110,14 @@ public:
 private:
 	u256 m_weiPerRef;
 	u256 m_refsPerBlock;
-	u256 m_gasPerBlock = 1000000;
+	u256 m_gasPerBlock = 3141592;
 	std::array<u256, 9> m_octiles;
 };
 
 /**
  * @brief Main API hub for interfacing with Ethereum.
  */
-class Client: public MinerHost, public Interface, Worker
+class Client: public MinerHost, public ClientBase, Worker
 {
 	friend class Miner;
 
@@ -211,88 +146,20 @@ public:
 	/// Resets the gas pricer to some other object.
 	void setGasPricer(std::shared_ptr<GasPricer> _gp) { m_gp = _gp; }
 
-	/// Submits the given message-call transaction.
-	virtual void submitTransaction(Secret _secret, u256 _value, Address _dest, bytes const& _data = bytes(), u256 _gas = 10000, u256 _gasPrice = 10 * szabo) override;
-
-	/// Submits a new contract-creation transaction.
-	/// @returns the new contract's address (assuming it all goes through).
-	virtual Address submitTransaction(Secret _secret, u256 _endowment, bytes const& _init, u256 _gas = 10000, u256 _gasPrice = 10 * szabo) override;
-
 	/// Injects the RLP-encoded transaction given by the _rlp into the transaction queue directly.
-	virtual void inject(bytesConstRef _rlp) override;
+	virtual void inject(bytesConstRef _rlp);
 
 	/// Blocks until all pending transactions have been processed.
 	virtual void flushTransactions() override;
 
-	/// Makes the given call. Nothing is recorded into the state.
+	using Interface::call; // to remove warning about hiding virtual function
-	virtual ExecutionResult call(Secret _secret, u256 _value, Address _dest, bytes const& _data = bytes(), u256 _gas = 10000, u256 _gasPrice = 10 * szabo, BlockNumber _blockNumber = PendingBlock) override;
-
-	/// Does the given creation. Nothing is recorded into the state.
-	/// @returns the pair of the Address of the created contract together with its code.
-	virtual ExecutionResult create(Secret _secret, u256 _value, bytes const& _data = bytes(), u256 _gas = 10000, u256 _gasPrice = 10 * szabo, BlockNumber _blockNumber = PendingBlock) override;
-
 	/// Makes the given call. Nothing is recorded into the state. This cheats by creating a null address and endowing it with a lot of ETH.
 	ExecutionResult call(Address _dest, bytes const& _data = bytes(), u256 _gas = 125000, u256 _value = 0, u256 _gasPrice = 1 * ether);
 
-	// Informational stuff
-
-	// [NEW API]
-
-	using Interface::balanceAt;
-	using Interface::countAt;
-	using Interface::stateAt;
-	using Interface::codeAt;
-	using Interface::storageAt;
-
-	virtual u256 balanceAt(Address _a, BlockNumber _block) const;
-	virtual u256 countAt(Address _a, BlockNumber _block) const;
-	virtual u256 stateAt(Address _a, u256 _l, BlockNumber _block) const;
-	virtual bytes codeAt(Address _a, BlockNumber _block) const;
-	virtual std::map<u256, u256> storageAt(Address _a, BlockNumber _block) const;
-
-	virtual unsigned installWatch(LogFilter const& _filter, Reaping _r = Reaping::Automatic) override;
-	virtual unsigned installWatch(h256 _filterId, Reaping _r = Reaping::Automatic) override;
-	virtual bool uninstallWatch(unsigned _watchId) override;
-	virtual LocalisedLogEntries peekWatch(unsigned _watchId) const;
-	virtual LocalisedLogEntries checkWatch(unsigned _watchId);
-
-	virtual LocalisedLogEntries logs(unsigned _watchId) const;
-	virtual LocalisedLogEntries logs(LogFilter const& _filter) const;
-
-	// [EXTRA API]:
-
-	/// @returns the length of the chain.
-	virtual unsigned number() const { return m_bc.number(); }
-
-	/// Get the list of pending transactions.
-	/// @TODO: Remove in favour of transactions().
-	virtual Transactions pending() const { return m_postMine.pending(); }
-
-	virtual h256 hashFromNumber(unsigned _number) const { return m_bc.numberHash(_number); }
-	virtual BlockInfo blockInfo(h256 _hash) const { return BlockInfo(m_bc.block(_hash)); }
-	virtual BlockDetails blockDetails(h256 _hash) const { return m_bc.details(_hash); }
-	virtual Transaction transaction(h256 _transactionHash) const;
-	virtual Transaction transaction(h256 _blockHash, unsigned _i) const;
-	virtual BlockInfo uncle(h256 _blockHash, unsigned _i) const;
-	virtual unsigned transactionCount(h256 _blockHash) const;
-	virtual unsigned uncleCount(h256 _blockHash) const;
-	virtual Transactions transactions(h256 _blockHash) const;
-	virtual TransactionHashes transactionHashes(h256 _blockHash) const;
-
-	/// Differences between transactions.
-	using Interface::diff;
-	virtual StateDiff diff(unsigned _txi, h256 _block) const;
-	virtual StateDiff diff(unsigned _txi, BlockNumber _block) const;
-
-	/// Get a list of all active addresses.
-	using Interface::addresses;
-	virtual std::vector<Address> addresses(BlockNumber _block) const;
-
 	/// Get the remaining gas limit in this block.
 	virtual u256 gasLimitRemaining() const { return m_postMine.gasLimitRemaining(); }
 
 	// [PRIVATE API - only relevant for base clients, not available in general]
-
 	dev::eth::State state(unsigned _txi, h256 _block) const;
 	dev::eth::State state(h256 _block) const;
 	dev::eth::State state(unsigned _txi) const;
@@ -304,6 +171,8 @@ public:
 
 	// Mining stuff:
 
+	void setAddress(Address _us) { WriteGuard l(x_stateDB); m_preMine.setAddress(_us); }
+
 	/// Check block validity prior to mining.
 	bool miningParanoia() const { return m_paranoia; }
 	/// Change whether we check block validity prior to mining.
@@ -317,10 +186,6 @@ public:
 	/// Enable/disable fast mining.
 	void setTurboMining(bool _enable = true) { m_turboMining = _enable; }
 
-	/// Set the coinbase address.
-	virtual void setAddress(Address _us) { m_preMine.setAddress(_us); }
-	/// Get the coinbase address.
-	virtual Address address() const { return m_preMine.address(); }
 	/// Stops mining and sets the number of mining threads (0 for automatic).
 	virtual void setMiningThreads(unsigned _threads = 0);
 	/// Get the effective number of mining threads.
@@ -356,6 +221,17 @@ public:
 	void killChain();
 
 protected:
+	/// InterfaceStub methods
+	virtual BlockChain const& bc() const override { return m_bc; }
+	
+	/// Returns the state object for the full block (i.e. the terminal state) for index _h.
+	/// Works properly with LatestBlock and PendingBlock.
+	using ClientBase::asOf;
+	virtual State asOf(h256 const& _block) const override;
+	virtual State preMine() const override { ReadGuard l(x_stateDB); return m_preMine; }
+	virtual State postMine() const override { ReadGuard l(x_stateDB); return m_postMine; }
+	virtual void prepareForTransaction() override;
+
 	/// Collate the changed filters for the bloom filter of the given pending transaction.
 	/// Insert any filters that are activated into @a o_changed.
 	void appendFromNewPending(TransactionReceipt const& _receipt, h256Set& io_changed, h256 _sha3);
@@ -380,16 +256,8 @@ private:
 	virtual bool turbo() const { return m_turboMining; }
 	virtual bool force() const { return m_forceMining; }
 
-	/// Return the actual block number of the block with the given int-number (positive is the same, INT_MIN is genesis block, < 0 is negative age, thus -1 is most recently mined, 0 is pending.
-	unsigned numberOf(int _b) const;
-
-	/// Returns the state object for the full block (i.e. the terminal state) for index _h.
-	/// Works properly with LatestBlock and PendingBlock.
-	State asOf(unsigned _h) const;
-
 	VersionChecker m_vc;					///< Dummy object to check & update the protocol version.
 	CanonBlockChain m_bc;					///< Maintains block database.
-	TransactionQueue m_tq;					///< Maintains a list of incoming transactions not yet in a block on the blockchain.
 	BlockQueue m_bq;						///< Maintains a list of incoming blocks not yet on the blockchain (to be imported).
 	std::shared_ptr<GasPricer> m_gp;		///< The gas pricer.
 
@@ -410,10 +278,6 @@ private:
 	bool m_forceMining = false;				///< Mine even when there are no transactions pending?
 	bool m_verifyOwnBlocks = true;			///< Should be verify blocks that we mined?
 
-	mutable Mutex m_filterLock;
-	std::map<h256, InstalledFilter> m_filters;
-	std::map<unsigned, ClientWatch> m_watches;
-
 	mutable std::chrono::system_clock::time_point m_lastGarbageCollection;
 };
 

libethereum/ClientBase.cpp

@@ -0,0 +1,399 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+ */
+/** @file ClientBase.cpp
+ * @author Gav Wood <i@gavwood.com>
+ * @author Marek Kotewicz <marek@ethdev.com>
+ * @date 2015
+ */
+
+#include <libdevcore/StructuredLogger.h>
+#include "ClientBase.h"
+#include "BlockChain.h"
+#include "Executive.h"
+
+using namespace std;
+using namespace dev;
+using namespace dev::eth;
+
+State ClientBase::asOf(BlockNumber _h) const
+{
+	if (_h == PendingBlock)
+		return postMine();
+	else if (_h == LatestBlock)
+		return preMine();
+	return asOf(bc().numberHash(_h));
+}
+
+void ClientBase::submitTransaction(Secret _secret, u256 _value, Address _dest, bytes const& _data, u256 _gas, u256 _gasPrice)
+{
+	prepareForTransaction();
+	
+	u256 n = postMine().transactionsFrom(toAddress(_secret));
+	Transaction t(_value, _gasPrice, _gas, _dest, _data, n, _secret);
+	m_tq.attemptImport(t.rlp());
+	
+	StructuredLogger::transactionReceived(t.sha3().abridged(), t.sender().abridged());
+	cnote << "New transaction " << t;
+}
+
+Address ClientBase::submitTransaction(Secret _secret, u256 _endowment, bytes const& _init, u256 _gas, u256 _gasPrice)
+{
+	prepareForTransaction();
+	
+	u256 n = postMine().transactionsFrom(toAddress(_secret));
+	Transaction t(_endowment, _gasPrice, _gas, _init, n, _secret);
+	m_tq.attemptImport(t.rlp());
+
+	StructuredLogger::transactionReceived(t.sha3().abridged(), t.sender().abridged());
+	cnote << "New transaction " << t;
+	
+	return right160(sha3(rlpList(t.sender(), t.nonce())));
+}
+
+// TODO: remove try/catch, allow exceptions
+ExecutionResult ClientBase::call(Secret _secret, u256 _value, Address _dest, bytes const& _data, u256 _gas, u256 _gasPrice, BlockNumber _blockNumber)
+{
+	ExecutionResult ret;
+	try
+	{
+		State temp = asOf(_blockNumber);
+		u256 n = temp.transactionsFrom(toAddress(_secret));
+		Transaction t(_value, _gasPrice, _gas, _dest, _data, n, _secret);
+		ret = temp.execute(bc().lastHashes(), t, Permanence::Reverted);
+	}
+	catch (...)
+	{
+		// TODO: Some sort of notification of failure.
+	}
+	return ret;
+}
+
+ExecutionResult ClientBase::create(Secret _secret, u256 _value, bytes const& _data, u256 _gas, u256 _gasPrice, BlockNumber _blockNumber)
+{
+	ExecutionResult ret;
+	try
+	{
+		State temp = asOf(_blockNumber);
+		u256 n = temp.transactionsFrom(toAddress(_secret));
+		//	cdebug << "Nonce at " << toAddress(_secret) << " pre:" << m_preMine.transactionsFrom(toAddress(_secret)) << " post:" << m_postMine.transactionsFrom(toAddress(_secret));
+		
+		Transaction t(_value, _gasPrice, _gas, _data, n, _secret);
+		ret = temp.execute(bc().lastHashes(), t, Permanence::Reverted);
+	}
+	catch (...)
+	{
+		// TODO: Some sort of notification of failure.
+	}
+	return ret;
+}
+
+u256 ClientBase::balanceAt(Address _a, BlockNumber _block) const
+{
+	return asOf(_block).balance(_a);
+}
+
+u256 ClientBase::countAt(Address _a, BlockNumber _block) const
+{
+	return asOf(_block).transactionsFrom(_a);
+}
+
+u256 ClientBase::stateAt(Address _a, u256 _l, BlockNumber _block) const
+{
+	return asOf(_block).storage(_a, _l);
+}
+
+bytes ClientBase::codeAt(Address _a, BlockNumber _block) const
+{
+	return asOf(_block).code(_a);
+}
+
+map<u256, u256> ClientBase::storageAt(Address _a, BlockNumber _block) const
+{
+	return asOf(_block).storage(_a);
+}
+
+// TODO: remove try/catch, allow exceptions
+LocalisedLogEntries ClientBase::logs(unsigned _watchId) const
+{
+	LogFilter f;
+	try
+	{
+		Guard l(x_filtersWatches);
+		f = m_filters.at(m_watches.at(_watchId).id).filter;
+	}
+	catch (...)
+	{
+		return LocalisedLogEntries();
+	}
+	return logs(f);
+}
+
+LocalisedLogEntries ClientBase::logs(LogFilter const& _f) const
+{
+	LocalisedLogEntries ret;
+	unsigned begin = min<unsigned>(bc().number() + 1, (unsigned)_f.latest());
+	unsigned end = min(bc().number(), min(begin, (unsigned)_f.earliest()));
+	
+	// Handle pending transactions differently as they're not on the block chain.
+	if (begin > bc().number())
+	{
+		State temp = postMine();
+		for (unsigned i = 0; i < temp.pending().size(); ++i)
+		{
+			// Might have a transaction that contains a matching log.
+			TransactionReceipt const& tr = temp.receipt(i);
+			auto th = temp.pending()[i].sha3();
+			LogEntries le = _f.matches(tr);
+			if (le.size())
+				for (unsigned j = 0; j < le.size(); ++j)
+					ret.insert(ret.begin(), LocalisedLogEntry(le[j], begin, th));
+		}
+		begin = bc().number();
+	}
+	
+	set<unsigned> matchingBlocks;
+	for (auto const& i: _f.bloomPossibilities())
+		for (auto u: bc().withBlockBloom(i, end, begin))
+			matchingBlocks.insert(u);
+
+	unsigned falsePos = 0;
+	for (auto n: matchingBlocks)
+	{
+		int total = 0;
+		auto h = bc().numberHash(n);
+		auto receipts = bc().receipts(h).receipts;
+		for (size_t i = 0; i < receipts.size(); i++)
+		{
+			TransactionReceipt receipt = receipts[i];
+			if (_f.matches(receipt.bloom()))
+			{
+				auto info = bc().info(h);
+				auto th = transaction(info.hash, i).sha3();
+				LogEntries le = _f.matches(receipt);
+				if (le.size())
+				{
+					total += le.size();
+					for (unsigned j = 0; j < le.size(); ++j)
+						ret.insert(ret.begin(), LocalisedLogEntry(le[j], n, th));
+				}
+			}
+			
+			if (!total)
+				falsePos++;
+		}
+	}
+
+	cdebug << matchingBlocks.size() << "searched from" << (end - begin) << "skipped; " << falsePos << "false +ves";
+	return ret;
+}
+
+unsigned ClientBase::installWatch(LogFilter const& _f, Reaping _r)
+{
+	h256 h = _f.sha3();
+	{
+		Guard l(x_filtersWatches);
+		if (!m_filters.count(h))
+		{
+			cwatch << "FFF" << _f << h.abridged();
+			m_filters.insert(make_pair(h, _f));
+		}
+	}
+	return installWatch(h, _r);
+}
+
+unsigned ClientBase::installWatch(h256 _h, Reaping _r)
+{
+	unsigned ret;
+	{
+		Guard l(x_filtersWatches);
+		ret = m_watches.size() ? m_watches.rbegin()->first + 1 : 0;
+		m_watches[ret] = ClientWatch(_h, _r);
+		cwatch << "+++" << ret << _h.abridged();
+	}
+#if INITIAL_STATE_AS_CHANGES
+	auto ch = logs(ret);
+	if (ch.empty())
+		ch.push_back(InitialChange);
+	{
+		Guard l(x_filtersWatches);
+		swap(m_watches[ret].changes, ch);
+	}
+#endif
+	return ret;
+}
+
+bool ClientBase::uninstallWatch(unsigned _i)
+{
+	cwatch << "XXX" << _i;
+	
+	Guard l(x_filtersWatches);
+	
+	auto it = m_watches.find(_i);
+	if (it == m_watches.end())
+		return false;
+	auto id = it->second.id;
+	m_watches.erase(it);
+	
+	auto fit = m_filters.find(id);
+	if (fit != m_filters.end())
+		if (!--fit->second.refCount)
+		{
+			cwatch << "*X*" << fit->first << ":" << fit->second.filter;
+			m_filters.erase(fit);
+		}
+	return true;
+}
+
+LocalisedLogEntries ClientBase::peekWatch(unsigned _watchId) const
+{
+	Guard l(x_filtersWatches);
+	
+//	cwatch << "peekWatch" << _watchId;
+	auto& w = m_watches.at(_watchId);
+//	cwatch << "lastPoll updated to " << chrono::duration_cast<chrono::seconds>(chrono::system_clock::now().time_since_epoch()).count();
+	w.lastPoll = chrono::system_clock::now();
+	return w.changes;
+}
+
+LocalisedLogEntries ClientBase::checkWatch(unsigned _watchId)
+{
+	Guard l(x_filtersWatches);
+	LocalisedLogEntries ret;
+	
+//	cwatch << "checkWatch" << _watchId;
+	auto& w = m_watches.at(_watchId);
+//	cwatch << "lastPoll updated to " << chrono::duration_cast<chrono::seconds>(chrono::system_clock::now().time_since_epoch()).count();
+	std::swap(ret, w.changes);
+	w.lastPoll = chrono::system_clock::now();
+	
+	return ret;
+}
+
+h256 ClientBase::hashFromNumber(unsigned _number) const
+{
+	return bc().numberHash(_number);
+}
+
+BlockInfo ClientBase::blockInfo(h256 _hash) const
+{
+	return BlockInfo(bc().block(_hash));
+}
+
+BlockDetails ClientBase::blockDetails(h256 _hash) const
+{
+	return bc().details(_hash);
+}
+
+Transaction ClientBase::transaction(h256 _transactionHash) const
+{
+	return Transaction(bc().transaction(_transactionHash), CheckSignature::Range);
+}
+
+Transaction ClientBase::transaction(h256 _blockHash, unsigned _i) const
+{
+	auto bl = bc().block(_blockHash);
+	RLP b(bl);
+	if (_i < b[1].itemCount())
+		return Transaction(b[1][_i].data(), CheckSignature::Range);
+	else
+		return Transaction();
+}
+
+Transactions ClientBase::transactions(h256 _blockHash) const
+{
+	auto bl = bc().block(_blockHash);
+	RLP b(bl);
+	Transactions res;
+	for (unsigned i = 0; i < b[1].itemCount(); i++)
+		res.emplace_back(b[1][i].data(), CheckSignature::Range);
+	return res;
+}
+
+TransactionHashes ClientBase::transactionHashes(h256 _blockHash) const
+{
+	return bc().transactionHashes(_blockHash);
+}
+
+BlockInfo ClientBase::uncle(h256 _blockHash, unsigned _i) const
+{
+	auto bl = bc().block(_blockHash);
+	RLP b(bl);
+	if (_i < b[2].itemCount())
+		return BlockInfo::fromHeader(b[2][_i].data());
+	else
+		return BlockInfo();
+}
+
+UncleHashes ClientBase::uncleHashes(h256 _blockHash) const
+{
+	return bc().uncleHashes(_blockHash);
+}
+
+unsigned ClientBase::transactionCount(h256 _blockHash) const
+{
+	auto bl = bc().block(_blockHash);
+	RLP b(bl);
+	return b[1].itemCount();
+}
+
+unsigned ClientBase::uncleCount(h256 _blockHash) const
+{
+	auto bl = bc().block(_blockHash);
+	RLP b(bl);
+	return b[2].itemCount();
+}
+
+unsigned ClientBase::number() const
+{
+	return bc().number();
+}
+
+Transactions ClientBase::pending() const
+{
+	return postMine().pending();
+}
+
+
+StateDiff ClientBase::diff(unsigned _txi, h256 _block) const
+{
+	State st = asOf(_block);
+	return st.fromPending(_txi).diff(st.fromPending(_txi + 1));
+}
+
+StateDiff ClientBase::diff(unsigned _txi, BlockNumber _block) const
+{
+	State st = asOf(_block);
+	return st.fromPending(_txi).diff(st.fromPending(_txi + 1));
+}
+
+Addresses ClientBase::addresses(BlockNumber _block) const
+{
+	Addresses ret;
+	for (auto const& i: asOf(_block).addresses())
+		ret.push_back(i.first);
+	return ret;
+}
+
+u256 ClientBase::gasLimitRemaining() const
+{
+	return postMine().gasLimitRemaining();
+}
+
+Address ClientBase::address() const
+{
+	return preMine().address();
+}

libethereum/ClientBase.h

@@ -0,0 +1,170 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+ */
+/** @file ClientBase.h
+ * @author Gav Wood <i@gavwood.com>
+ * @author Marek Kotewicz <marek@ethdev.com>
+ * @date 2015
+ */
+
+#pragma once
+
+#include <chrono>
+#include "Interface.h"
+#include "LogFilter.h"
+
+namespace dev {
+
+namespace eth {
+
+struct InstalledFilter
+{
+	InstalledFilter(LogFilter const& _f): filter(_f) {}
+
+	LogFilter filter;
+	unsigned refCount = 1;
+	LocalisedLogEntries changes;
+};
+
+static const h256 PendingChangedFilter = u256(0);
+static const h256 ChainChangedFilter = u256(1);
+
+static const LogEntry SpecialLogEntry = LogEntry(Address(), h256s(), bytes());
+static const LocalisedLogEntry InitialChange(SpecialLogEntry, 0);
+
+struct ClientWatch
+{
+	ClientWatch(): lastPoll(std::chrono::system_clock::now()) {}
+	explicit ClientWatch(h256 _id, Reaping _r): id(_id), lastPoll(_r == Reaping::Automatic ? std::chrono::system_clock::now() : std::chrono::system_clock::time_point::max()) {}
+
+	h256 id;
+#if INITIAL_STATE_AS_CHANGES
+	LocalisedLogEntries changes = LocalisedLogEntries{ InitialChange };
+#else
+	LocalisedLogEntries changes;
+#endif
+	mutable std::chrono::system_clock::time_point lastPoll = std::chrono::system_clock::now();
+};
+
+struct WatchChannel: public LogChannel { static const char* name() { return "(o)"; } static const int verbosity = 7; };
+#define cwatch dev::LogOutputStream<dev::eth::WatchChannel, true>()
+struct WorkInChannel: public LogChannel { static const char* name() { return ">W>"; } static const int verbosity = 16; };
+struct WorkOutChannel: public LogChannel { static const char* name() { return "<W<"; } static const int verbosity = 16; };
+struct WorkChannel: public LogChannel { static const char* name() { return "-W-"; } static const int verbosity = 16; };
+#define cwork dev::LogOutputStream<dev::eth::WorkChannel, true>()
+#define cworkin dev::LogOutputStream<dev::eth::WorkInChannel, true>()
+#define cworkout dev::LogOutputStream<dev::eth::WorkOutChannel, true>()
+
+class ClientBase: public dev::eth::Interface
+{
+public:
+	ClientBase() {}
+	virtual ~ClientBase() {}
+
+	/// Submits the given message-call transaction.
+	virtual void submitTransaction(Secret _secret, u256 _value, Address _dest, bytes const& _data = bytes(), u256 _gas = 10000, u256 _gasPrice = 10 * szabo) override;
+
+	/// Submits a new contract-creation transaction.
+	/// @returns the new contract's address (assuming it all goes through).
+	virtual Address submitTransaction(Secret _secret, u256 _endowment, bytes const& _init, u256 _gas = 10000, u256 _gasPrice = 10 * szabo) override;
+
+	/// Makes the given call. Nothing is recorded into the state.
+	virtual ExecutionResult call(Secret _secret, u256 _value, Address _dest, bytes const& _data = bytes(), u256 _gas = 10000, u256 _gasPrice = 10 * szabo, BlockNumber _blockNumber = PendingBlock) override;
+
+	virtual ExecutionResult create(Secret _secret, u256 _value, bytes const& _data = bytes(), u256 _gas = 10000, u256 _gasPrice = 10 * szabo, BlockNumber _blockNumber = PendingBlock) override;
+	
+	using Interface::balanceAt;
+	using Interface::countAt;
+	using Interface::stateAt;
+	using Interface::codeAt;
+	using Interface::storageAt;
+
+	virtual u256 balanceAt(Address _a, BlockNumber _block) const override;
+	virtual u256 countAt(Address _a, BlockNumber _block) const override;
+	virtual u256 stateAt(Address _a, u256 _l, BlockNumber _block) const override;
+	virtual bytes codeAt(Address _a, BlockNumber _block) const override;
+	virtual std::map<u256, u256> storageAt(Address _a, BlockNumber _block) const override;
+
+	virtual LocalisedLogEntries logs(unsigned _watchId) const override;
+	virtual LocalisedLogEntries logs(LogFilter const& _filter) const override;
+
+	/// Install, uninstall and query watches.
+	virtual unsigned installWatch(LogFilter const& _filter, Reaping _r = Reaping::Automatic) override;
+	virtual unsigned installWatch(h256 _filterId, Reaping _r = Reaping::Automatic) override;
+	virtual bool uninstallWatch(unsigned _watchId) override;
+	virtual LocalisedLogEntries peekWatch(unsigned _watchId) const override;
+	virtual LocalisedLogEntries checkWatch(unsigned _watchId) override;
+
+	virtual h256 hashFromNumber(unsigned _number) const override;
+	virtual eth::BlockInfo blockInfo(h256 _hash) const override;
+	virtual eth::BlockDetails blockDetails(h256 _hash) const override;
+	virtual eth::Transaction transaction(h256 _transactionHash) const override;
+	virtual eth::Transaction transaction(h256 _blockHash, unsigned _i) const override;
+	virtual eth::Transactions transactions(h256 _blockHash) const override;
+	virtual eth::TransactionHashes transactionHashes(h256 _blockHash) const override;
+	virtual eth::BlockInfo uncle(h256 _blockHash, unsigned _i) const override;
+	virtual eth::UncleHashes uncleHashes(h256 _blockHash) const override;
+	virtual unsigned transactionCount(h256 _blockHash) const override;
+	virtual unsigned uncleCount(h256 _blockHash) const override;
+	virtual unsigned number() const override;
+	virtual eth::Transactions pending() const override;
+
+	using Interface::diff;
+	virtual StateDiff diff(unsigned _txi, h256 _block) const override;
+	virtual StateDiff diff(unsigned _txi, BlockNumber _block) const override;
+
+	using Interface::addresses;
+	virtual Addresses addresses(BlockNumber _block) const override;
+	virtual u256 gasLimitRemaining() const override;
+
+	/// Set the coinbase address
+	virtual void setAddress(Address _us) = 0;
+
+	/// Get the coinbase address
+	virtual Address address() const override;
+
+	/// TODO: consider moving it to a separate interface
+
+	virtual void setMiningThreads(unsigned _threads) override { (void)_threads; BOOST_THROW_EXCEPTION(InterfaceNotSupported("dev::eth::ClientBase::setMiningThreads")); }
+	virtual unsigned miningThreads() const override { BOOST_THROW_EXCEPTION(InterfaceNotSupported("dev::eth::ClientBase::miningThreads")); }
+	virtual void startMining() override { BOOST_THROW_EXCEPTION(InterfaceNotSupported("dev::eth::ClientBase::startMining")); }
+	virtual void stopMining() override { BOOST_THROW_EXCEPTION(InterfaceNotSupported("dev::eth::ClientBase::stopMining")); }
+	virtual bool isMining() override { BOOST_THROW_EXCEPTION(InterfaceNotSupported("dev::eth::ClientBase::isMining")); }
+	virtual eth::MineProgress miningProgress() const override { BOOST_THROW_EXCEPTION(InterfaceNotSupported("dev::eth::ClientBase::miningProgress")); }
+	virtual std::pair<h256, u256> getWork() override { BOOST_THROW_EXCEPTION(InterfaceNotSupported("dev::eth::ClientBase::getWork")); }
+	virtual bool submitWork(eth::ProofOfWork::Proof const&) override { BOOST_THROW_EXCEPTION(InterfaceNotSupported("dev::eth::ClientBase::submitWork")); }
+
+	State asOf(BlockNumber _h) const;
+
+protected:
+	/// The interface that must be implemented in any class deriving this.
+	/// {
+	virtual BlockChain const& bc() const = 0;
+	virtual State asOf(h256 const& _h) const = 0;
+	virtual State preMine() const = 0;
+	virtual State postMine() const = 0;
+	virtual void prepareForTransaction() = 0;
+	/// }
+
+	TransactionQueue m_tq;							///< Maintains a list of incoming transactions not yet in a block on the blockchain.
+
+	// filters
+	mutable Mutex x_filtersWatches;					///< Our lock.
+	std::map<h256, InstalledFilter> m_filters;		///< The dictionary of filters that are active.
+	std::map<unsigned, ClientWatch> m_watches;		///< Each and every watch - these reference a filter.
+};
+
+}}

libethereum/EthereumHost.cpp

@@ -178,7 +178,7 @@ void EthereumHost::maintainTransactions()
 			for (auto const& i: m_tq.transactions())
 				if (ep->m_requireTransactions || (!m_transactionsSent.count(i.first) && !ep->m_knownTransactions.count(i.first)))
 				{
-					b += i.second;
+					b += i.second.rlp();
 					++n;
 					m_transactionsSent.insert(i.first);
 				}

libethereum/Executive.cpp

@@ -35,7 +35,7 @@ using namespace dev::eth;
 
 Executive::Executive(State& _s, BlockChain const& _bc, unsigned _level):
 	m_s(_s),
-	m_lastHashes(_s.getLastHashes(_bc, (unsigned)_s.info().number - 1)),
+	m_lastHashes(_bc.lastHashes((unsigned)_s.info().number - 1)),
 	m_depth(_level)
 {}
 
@@ -44,18 +44,44 @@ u256 Executive::gasUsed() const
 	return m_t.gas() - m_endGas;
 }
 
+ExecutionResult Executive::executionResult() const
+{
+	return ExecutionResult(gasUsed(), m_excepted, m_newAddress, m_out, m_codeDeposit, m_ext ? m_ext->sub.refunds : 0, m_depositSize, m_gasForDeposit);
+}
+
 void Executive::accrueSubState(SubState& _parentContext)
 {
 	if (m_ext)
 		_parentContext += m_ext->sub;
 }
 
-bool Executive::setup(bytesConstRef _rlp)
+void Executive::initialize(Transaction const& _transaction)
 {
-	// Entry point for a user-executed transaction.
+	m_t = _transaction;
+
+	// Avoid transactions that would take us beyond the block gas limit.
+	u256 startGasUsed = m_s.gasUsed();
+	if (startGasUsed + (bigint)m_t.gas() > m_s.m_currentBlock.gasLimit)
+	{
+		clog(StateDetail) << "Too much gas used in this block: Require <" << (m_s.m_currentBlock.gasLimit - startGasUsed) << " Got" << m_t.gas();
+		m_excepted = TransactionException::BlockGasLimitReached;
+		BOOST_THROW_EXCEPTION(BlockGasLimitReached() << RequirementError((bigint)(m_s.m_currentBlock.gasLimit - startGasUsed), (bigint)m_t.gas()));
+	}
+
+	// Check gas cost is enough.
+	m_gasRequired = Interface::txGas(m_t.data());
+	if (m_t.gas() < m_gasRequired)
+	{
+		clog(StateDetail) << "Not enough gas to pay for the transaction: Require >" << m_gasRequired << " Got" << m_t.gas();
+		m_excepted = TransactionException::OutOfGas;
+		BOOST_THROW_EXCEPTION(OutOfGas() << RequirementError((bigint)m_gasRequired, (bigint)m_t.gas()));
+	}
+
+	// Avoid invalid transactions.
+	u256 nonceReq;
 	try
 	{
-		m_t = Transaction(_rlp, CheckSignature::Sender);
+		nonceReq = m_s.transactionsFrom(m_t.sender());
 	}
 	catch (...)
 	{
@@ -63,15 +89,6 @@ bool Executive::setup(bytesConstRef _rlp)
 		m_excepted = TransactionException::InvalidSignature;
 		throw;
 	}
-	return setup();
-}
-
-bool Executive::setup()
-{
-	// Entry point for a user-executed transaction.
-
-	// Avoid invalid transactions.
-	auto nonceReq = m_s.transactionsFrom(m_t.sender());
 	if (m_t.nonce() != nonceReq)
 	{
 		clog(StateDetail) << "Invalid Nonce: Require" << nonceReq << " Got" << m_t.nonce();
@@ -79,50 +96,38 @@ bool Executive::setup()
 		BOOST_THROW_EXCEPTION(InvalidNonce() << RequirementError((bigint)nonceReq, (bigint)m_t.nonce()));
 	}
 
-	// Check gas cost is enough.
-	auto gasCost = Interface::txGas(m_t.data());
-
-	if (m_t.gas() < gasCost)
-	{
-		clog(StateDetail) << "Not enough gas to pay for the transaction: Require >" << gasCost << " Got" << m_t.gas();
-		BOOST_THROW_EXCEPTION(OutOfGas() << RequirementError((bigint)gasCost, (bigint)m_t.gas()));
-	}
-
-	bigint cost = m_t.value() + (bigint)m_t.gas() * m_t.gasPrice();
-
 	// Avoid unaffordable transactions.
-	if (m_s.balance(m_t.sender()) < cost)
+	m_gasCost = (bigint)m_t.gas() * m_t.gasPrice();
+	m_totalCost = m_t.value() + m_gasCost;
+	if (m_s.balance(m_t.sender()) < m_totalCost)
 	{
-		clog(StateDetail) << "Not enough cash: Require >" << cost << " Got" << m_s.balance(m_t.sender());
+		clog(StateDetail) << "Not enough cash: Require >" << m_totalCost << " Got" << m_s.balance(m_t.sender());
-		BOOST_THROW_EXCEPTION(NotEnoughCash() << RequirementError(cost, (bigint)m_s.balance(m_t.sender())));
+		m_excepted = TransactionException::NotEnoughCash;
+		BOOST_THROW_EXCEPTION(NotEnoughCash() << RequirementError(m_totalCost, (bigint)m_s.balance(m_t.sender())));
+	}
 }
 
-	u256 startGasUsed = m_s.gasUsed();
+bool Executive::execute()
-	if (startGasUsed + (bigint)m_t.gas() > m_s.m_currentBlock.gasLimit)
 {
-		clog(StateDetail) << "Too much gas used in this block: Require <" << (m_s.m_currentBlock.gasLimit - startGasUsed) << " Got" << m_t.gas();
+	// Entry point for a user-executed transaction.
-		BOOST_THROW_EXCEPTION(BlockGasLimitReached() << RequirementError((bigint)(m_s.m_currentBlock.gasLimit - startGasUsed), (bigint)m_t.gas()));
-	}
 
 	// Increment associated nonce for sender.
 	m_s.noteSending(m_t.sender());
 
 	// Pay...
-	clog(StateDetail) << "Paying" << formatBalance(u256(cost)) << "from sender (includes" << m_t.gas() << "gas at" << formatBalance(m_t.gasPrice()) << ")";
+	clog(StateDetail) << "Paying" << formatBalance(u256(m_gasCost)) << "from sender for gas (" << m_t.gas() << "gas at" << formatBalance(m_t.gasPrice()) << ")";
-	m_s.subBalance(m_t.sender(), cost);
+	m_s.subBalance(m_t.sender(), m_gasCost);
 
 	if (m_t.isCreation())
-		return create(m_t.sender(), m_t.value(), m_t.gasPrice(), m_t.gas() - (u256)gasCost, &m_t.data(), m_t.sender());
+		return create(m_t.sender(), m_t.value(), m_t.gasPrice(), m_t.gas() - (u256)m_gasRequired, &m_t.data(), m_t.sender());
 	else
-		return call(m_t.receiveAddress(), m_t.receiveAddress(), m_t.sender(), m_t.value(), m_t.gasPrice(), bytesConstRef(&m_t.data()), m_t.gas() - (u256)gasCost, m_t.sender());
+		return call(m_t.receiveAddress(), m_t.receiveAddress(), m_t.sender(), m_t.value(), m_t.gasPrice(), bytesConstRef(&m_t.data()), m_t.gas() - (u256)m_gasRequired, m_t.sender());
 }
 
 bool Executive::call(Address _receiveAddress, Address _codeAddress, Address _senderAddress, u256 _value, u256 _gasPrice, bytesConstRef _data, u256 _gas, Address _originAddress)
 {
 	m_isCreation = false;
 //	cnote << "Transferring" << formatBalance(_value) << "to receiver.";
-	m_s.addBalance(_receiveAddress, _value);
-
 	auto it = !(_codeAddress & ~h160(0xffffffff)) ? precompiled().find((unsigned)(u160)_codeAddress) : precompiled().end();
 	if (it != precompiled().end())
 	{
@@ -131,6 +136,8 @@ bool Executive::call(Address _receiveAddress, Address _codeAddress, Address _sen
 		{
 			m_endGas = 0;
 			m_excepted = TransactionException::OutOfGasBase;
+			// Bail from exception.
+			return true;	// true actually means "all finished - nothing more to be done regarding go().
 		}
 		else
 		{
@@ -147,6 +154,9 @@ bool Executive::call(Address _receiveAddress, Address _codeAddress, Address _sen
 	}
 	else
 		m_endGas = _gas;
+
+	m_s.transferBalance(_senderAddress, _receiveAddress, _value);
+
 	return !m_ext;
 }
 
@@ -158,20 +168,22 @@ bool Executive::create(Address _sender, u256 _endowment, u256 _gasPrice, u256 _g
 	// we delete it explicitly if we decide we need to revert.
 	m_newAddress = right160(sha3(rlpList(_sender, m_s.transactionsFrom(_sender) - 1)));
 
-	// Set up new account...
-	m_s.m_cache[m_newAddress] = Account(m_s.balance(m_newAddress) + _endowment, Account::ContractConception);
-
 	// Execute _init.
+	if (!_init.empty())
+	{
+		m_vm = VMFactory::create(_gas);
+		m_ext = make_shared<ExtVM>(m_s, m_lastHashes, m_newAddress, _sender, _origin, _endowment, _gasPrice, bytesConstRef(), _init, m_depth);
+	}
+
+	m_s.m_cache[m_newAddress] = Account(m_s.balance(m_newAddress), Account::ContractConception);
+	m_s.transferBalance(_sender, m_newAddress, _endowment);
+
 	if (_init.empty())
 	{
 		m_s.m_cache[m_newAddress].setCode({});
 		m_endGas = _gas;
 	}
-	else
+
-	{
-		m_vm = VMFactory::create(_gas);
-		m_ext = make_shared<ExtVM>(m_s, m_lastHashes, m_newAddress, _sender, _origin, _endowment, _gasPrice, bytesConstRef(), _init, m_depth);
-	}
 	return !m_ext;
 }
 
@@ -209,6 +221,8 @@ bool Executive::go(OnOpFunc const& _onOp)
 
 			if (m_isCreation)
 			{
+				m_gasForDeposit = m_endGas;
+				m_depositSize = m_out.size();
 				if (m_out.size() * c_createDataGas <= m_endGas)
 				{
 					m_codeDeposit = CodeDeposit::Success;
@@ -216,6 +230,7 @@ bool Executive::go(OnOpFunc const& _onOp)
 				}
 				else
 				{
+
 					m_codeDeposit = CodeDeposit::Failed;
 					m_out.reset();
 				}

libethereum/Executive.h

@@ -41,10 +41,21 @@ struct VMTraceChannel: public LogChannel { static const char* name() { return "E
  * @brief Message-call/contract-creation executor; useful for executing transactions.
  *
  * Two ways of using this class - either as a transaction executive or a CALL/CREATE executive.
- * In the first use, after construction, begin with setup() and end with finalize(). Call go()
+ *
- * after setup() only if it returns false.
+ * In the first use, after construction, begin with initialize(), then execute() and end with finalize(). Call go()
+ * after execute() only if it returns false.
+ *
  * In the second use, after construction, begin with call() or create() and end with
  * accrueSubState(). Call go() after call()/create() only if it returns false.
+ *
+ * Example:
+ * @code
+ * Executive e(state, blockchain, 0);
+ * e.initialize(transaction);
+ * if (!e.execute())
+ *    e.go();
+ * e.finalize();
+ * @endcode
  */
 class Executive
 {
@@ -59,17 +70,17 @@ public:
 	Executive(Executive const&) = delete;
 	void operator=(Executive) = delete;
 
-	/// Set up the executive for evaluating a transaction. You must call finalize() following this.
+	/// Initializes the executive for evaluating a transaction. You must call finalize() at some point following this.
-	/// @returns true iff go() must be called (and thus a VM execution in required).
+	void initialize(bytesConstRef _transaction) { initialize(Transaction(_transaction, CheckSignature::None)); }
-	bool setup(bytesConstRef _transaction);
+	void initialize(Transaction const& _transaction);
-	/// Set up the executive for evaluating a transaction. You must call finalize() following this.
+	/// Finalise a transaction previously set up with initialize().
-	/// @returns true iff go() must be called (and thus a VM execution in required).
+	/// @warning Only valid after initialize() and execute(), and possibly go().
-	bool setup(Transaction const& _transaction) { m_t = _transaction; return setup(); }
-	/// Finalise a transaction previously set up with setup().
-	/// @warning Only valid after setup(), and possibly go().
 	void finalize();
-	/// @returns the transaction from setup().
+	/// Begins execution of a transaction. You must call finalize() following this.
-	/// @warning Only valid after setup().
+	/// @returns true if the transaction is done, false if go() must be called.
+	bool execute();
+	/// @returns the transaction from initialize().
+	/// @warning Only valid after initialize().
 	Transaction const& t() const { return m_t; }
 	/// @returns the log entries created by this operation.
 	/// @warning Only valid after finalise().
@@ -104,11 +115,9 @@ public:
 	bool excepted() const { return m_excepted != TransactionException::None; }
 
 	/// Get the above in an amalgamated fashion.
-	ExecutionResult executionResult() const { return ExecutionResult(gasUsed(), m_excepted, m_newAddress, m_out, m_codeDeposit); }
+	ExecutionResult executionResult() const;
 
 private:
-	bool setup();
-
 	State& m_s;							///< The state to which this operation/transaction is applied.
 	LastHashes m_lastHashes;
 	std::shared_ptr<ExtVM> m_ext;		///< The VM externality object for the VM execution or null if no VM is required.
@@ -119,12 +128,18 @@ private:
 
 	unsigned m_depth = 0;				///< The context's call-depth.
 	bool m_isCreation = false;			///< True if the transaction creates a contract, or if create() is called.
+	unsigned m_depositSize = 0;			///< Amount of code of the creation's attempted deposit.
+	u256 m_gasForDeposit;				///< Amount of gas remaining for the code deposit phase.
 	CodeDeposit m_codeDeposit = CodeDeposit::None;	///< True if an attempted deposit failed due to lack of gas.
 	TransactionException m_excepted = TransactionException::None;	///< Details if the VM's execution resulted in an exception.
 	u256 m_endGas;						///< The final amount of gas for the transaction.
 
 	Transaction m_t;					///< The original transaction. Set by setup().
 	LogEntries m_logs;					///< The log entries created by this transaction. Set by finalize().
+
+	bigint m_gasRequired;
+	bigint m_gasCost;
+	bigint m_totalCost;
 };
 
 }

libethereum/ExtVM.h

@@ -82,7 +82,11 @@ public:
 
 	/// Revert any changes made (by any of the other calls).
 	/// @TODO check call site for the parent manifest being discarded.
-	virtual void revert() override final { m_s.m_cache = m_origCache; sub.clear(); }
+	virtual void revert() override final
+	{
+		m_s.m_cache = m_origCache;
+		sub.clear();
+	}
 
 	State& state() const { return m_s; }
 

libethereum/Interface.h

@@ -38,6 +38,7 @@ namespace eth
 {
 
 using TransactionHashes = h256s;
+using UncleHashes = h256s;
 
 enum class Reaping
 {
@@ -66,18 +67,17 @@ public:
 	/// @returns the new contract's address (assuming it all goes through).
 	virtual Address submitTransaction(Secret _secret, u256 _endowment, bytes const& _init, u256 _gas = 10000, u256 _gasPrice = 10 * szabo) = 0;
 
-	/// Injects the RLP-encoded transaction given by the _rlp into the transaction queue directly.
-	virtual void inject(bytesConstRef _rlp) = 0;
-
 	/// Blocks until all pending transactions have been processed.
 	virtual void flushTransactions() = 0;
 
 	/// Makes the given call. Nothing is recorded into the state.
-	virtual ExecutionResult call(Secret _secret, u256 _value, Address _dest, bytes const& _data = bytes(), u256 _gas = 10000, u256 _gasPrice = 10 * szabo, BlockNumber _blockNumber = 0) = 0;
+	virtual ExecutionResult call(Secret _secret, u256 _value, Address _dest, bytes const& _data, u256 _gas, u256 _gasPrice, BlockNumber _blockNumber) = 0;
+	ExecutionResult call(Secret _secret, u256 _value, Address _dest, bytes const& _data = bytes(), u256 _gas = 10000, u256 _gasPrice = 10 * szabo) { return call(_secret, _value, _dest, _data, _gas, _gasPrice, m_default); }
 
 	/// Does the given creation. Nothing is recorded into the state.
 	/// @returns the pair of the Address of the created contract together with its code.
-	virtual ExecutionResult create(Secret _secret, u256 _value, bytes const& _data = bytes(), u256 _gas = 10000, u256 _gasPrice = 10 * szabo, BlockNumber _blockNumber = 0) = 0;
+	virtual ExecutionResult create(Secret _secret, u256 _value, bytes const& _data, u256 _gas, u256 _gasPrice, BlockNumber _blockNumber) = 0;
+	ExecutionResult create(Secret _secret, u256 _value, bytes const& _data = bytes(), u256 _gas = 10000, u256 _gasPrice = 10 * szabo) { return create(_secret, _value, _data, _gas, _gasPrice, m_default); }
 
 	// [STATE-QUERY API]
 
@@ -118,6 +118,7 @@ public:
 	virtual Transaction transaction(h256 _transactionHash) const = 0;
 	virtual Transaction transaction(h256 _blockHash, unsigned _i) const = 0;
 	virtual BlockInfo uncle(h256 _blockHash, unsigned _i) const = 0;
+	virtual UncleHashes uncleHashes(h256 _blockHash) const = 0;
 	virtual unsigned transactionCount(h256 _blockHash) const = 0;
 	virtual unsigned uncleCount(h256 _blockHash) const = 0;
 	virtual Transactions transactions(h256 _blockHash) const = 0;
@@ -138,6 +139,7 @@ public:
 	virtual StateDiff diff(unsigned _txi, BlockNumber _block) const = 0;
 
 	/// Get a list of all active addresses.
+	/// NOTE: This only works when compiled with ETH_FATDB; otherwise will throw InterfaceNotSupported.
 	virtual Addresses addresses() const { return addresses(m_default); }
 	virtual Addresses addresses(BlockNumber _block) const = 0;
 

libethereum/State.cpp

@@ -60,7 +60,18 @@ OverlayDB State::openDB(std::string _path, bool _killExisting)
 	ldb::DB* db = nullptr;
 	ldb::DB::Open(o, _path + "/state", &db);
 	if (!db)
+	{
+		if (boost::filesystem::space(_path + "/state").available < 1024)
+		{
+			cwarn << "Not enough available space found on hard drive. Please free some up and then re-run. Bailing.";
+			BOOST_THROW_EXCEPTION(NotEnoughAvailableSpace());
+		}
+		else
+		{
+			cwarn << "Database already open. You appear to have another instance of ethereum running. Bailing.";
 			BOOST_THROW_EXCEPTION(DatabaseAlreadyOpen());
+		}
+	}
 
 	cnote << "Opened state DB.";
 	return OverlayDB(db);
@@ -100,23 +111,37 @@ State::State(OverlayDB const& _db, BlockChain const& _bc, h256 _h):
 	m_state(&m_db),
 	m_blockReward(c_blockReward)
 {
-	// TODO THINK: is this necessary?
-	m_state.init();
-
 	auto b = _bc.block(_h);
-	BlockInfo bi;
+	BlockInfo bi(b);
-	BlockInfo bip;
+
-	if (_h)
+	if (!bi)
-		bi.populate(b);
+	{
-	if (bi && bi.number)
+		// Might be worth throwing here.
-		bip.populate(_bc.block(bi.parentHash));
+		cwarn << "Invalid block given for state population: " << _h;
-	if (!_h || !bip)
 		return;
-	m_ourAddress = bi.coinbaseAddress;
+	}
 
+	if (bi.number)
+	{
+		// Non-genesis:
+
+		// 1. Start at parent's end state (state root).
+		BlockInfo bip;
+		bip.populate(_bc.block(bi.parentHash));
 		sync(_bc, bi.parentHash, bip);
+
+		// 2. Enact the block's transactions onto this state.
+		m_ourAddress = bi.coinbaseAddress;
 		enact(&b, _bc);
 	}
+	else
+	{
+		// Genesis required:
+		// We know there are no transactions, so just populate directly.
+		m_state.init();
+		sync(_bc, _h, bi);
+	}
+}
 
 State::State(State const& _s):
 	m_db(_s.m_db),
@@ -342,6 +367,7 @@ u256 State::enactOn(bytesConstRef _block, BlockInfo const& _bi, BlockChain const
 
 map<Address, u256> State::addresses() const
 {
+#if ETH_FATDB
 	map<Address, u256> ret;
 	for (auto i: m_cache)
 		if (i.second.isAlive())
@@ -350,6 +376,9 @@ map<Address, u256> State::addresses() const
 		if (m_cache.find(i.first) == m_cache.end())
 			ret[i.first] = RLP(i.second)[1].toInt<u256>();
 	return ret;
+#else
+	throw InterfaceNotSupported("State::addresses()");
+#endif
 }
 
 void State::resetCurrent()
@@ -384,8 +413,7 @@ bool State::cull(TransactionQueue& _tq) const
 		{
 			try
 			{
-				Transaction t(i.second, CheckSignature::Sender);
+				if (i.second.nonce() <= transactionsFrom(i.second.sender()))
-				if (t.nonce() <= transactionsFrom(t.sender()))
 				{
 					_tq.drop(i.first);
 					ret = true;
@@ -407,7 +435,7 @@ TransactionReceipts State::sync(BlockChain const& _bc, TransactionQueue& _tq, Ga
 	TransactionReceipts ret;
 	auto ts = _tq.transactions();
 
-	auto lh = getLastHashes(_bc, _bc.number());
+	LastHashes lh;
 
 	for (int goodTxs = 1; goodTxs;)
 	{
@@ -417,12 +445,11 @@ TransactionReceipts State::sync(BlockChain const& _bc, TransactionQueue& _tq, Ga
 			{
 				try
 				{
-					Transaction t(i.second, CheckSignature::Sender);
+					if (i.second.gasPrice() >= _gp.ask(*this))
-					if (t.gasPrice() >= _gp.ask(*this))
 					{
-						// don't have it yet! Execute it now.
-						uncommitToMine();
 	//					boost::timer t;
+						if (lh.empty())
+							lh = _bc.lastHashes();
 						execute(lh, i.second);
 						ret.push_back(m_receipts.back());
 						_tq.noteGood(i);
@@ -430,6 +457,7 @@ TransactionReceipts State::sync(BlockChain const& _bc, TransactionQueue& _tq, Ga
 	//					cnote << "TX took:" << t.elapsed() * 1000;
 					}
 				}
+#if ETH_DEBUG
 				catch (InvalidNonce const& in)
 				{
 					bigint const* req = boost::get_error_info<errinfo_required>(in);
@@ -445,13 +473,19 @@ TransactionReceipts State::sync(BlockChain const& _bc, TransactionQueue& _tq, Ga
 					else
 						_tq.setFuture(i);
 				}
+				catch (BlockGasLimitReached const& e)
+				{
+					_tq.setFuture(i);
+				}
+#endif
 				catch (Exception const& _e)
 				{
 					// Something else went wrong - drop it.
 					_tq.drop(i.first);
 					if (o_transactionQueueChanged)
 						*o_transactionQueueChanged = true;
-					cwarn << "Sync went wrong\n" << diagnostic_information(_e);
+					cnote << "Dropping invalid transaction:";
+					cnote << diagnostic_information(_e);
 				}
 				catch (std::exception const&)
 				{
@@ -459,6 +493,7 @@ TransactionReceipts State::sync(BlockChain const& _bc, TransactionQueue& _tq, Ga
 					_tq.drop(i.first);
 					if (o_transactionQueueChanged)
 						*o_transactionQueueChanged = true;
+					cnote << "Transaction caused low-level exception :(";
 				}
 			}
 	}
@@ -494,7 +529,7 @@ u256 State::enact(bytesConstRef _block, BlockChain const& _bc, bool _checkNonce)
 	GenericTrieDB<MemoryDB> receiptsTrie(&rm);
 	receiptsTrie.init();
 
-	LastHashes lh = getLastHashes(_bc, (unsigned)m_previousBlock.number);
+	LastHashes lh = _bc.lastHashes((unsigned)m_previousBlock.number);
 	RLP rlp(_block);
 
 	// All ok with the block generally. Play back the transactions now...
@@ -505,7 +540,7 @@ u256 State::enact(bytesConstRef _block, BlockChain const& _bc, bool _checkNonce)
 		k << i;
 
 		transactionsTrie.insert(&k.out(), tr.data());
-		execute(lh, tr.data());
+		execute(lh, Transaction(tr.data(), CheckSignature::Sender));
 
 		RLPStream receiptrlp;
 		m_receipts.back().streamRLP(receiptrlp);
@@ -690,7 +725,7 @@ void State::commitToMine(BlockChain const& _bc)
 	uncommitToMine();
 
 //	cnote << "Committing to mine on block" << m_previousBlock.hash.abridged();
-#ifdef ETH_PARANOIA
+#if  ETH_PARANOIA && 0
 	commit();
 	cnote << "Pre-reward stateRoot:" << m_state.root();
 #endif
@@ -1036,52 +1071,30 @@ bool State::isTrieGood(bool _enforceRefs, bool _requireNoLeftOvers) const
 	return true;
 }
 
-LastHashes State::getLastHashes(BlockChain const& _bc, unsigned _n) const
+ExecutionResult State::execute(LastHashes const& _lh, Transaction const& _t, Permanence _p)
 {
-	LastHashes ret;
+#if ETH_PARANOIA
-	ret.resize(256);
-	if (eth::c_protocolVersion > 49)
-	{
-		ret[0] = _bc.numberHash(_n);
-		for (unsigned i = 1; i < 256; ++i)
-			ret[i] = ret[i - 1] ? _bc.details(ret[i - 1]).parent : h256();
-	}
-	return ret;
-}
-
-ExecutionResult State::execute(BlockChain const& _bc, bytes const& _rlp, Permanence _p)
-{
-	return execute(getLastHashes(_bc, _bc.number()), &_rlp, _p);
-}
-
-ExecutionResult State::execute(BlockChain const& _bc, bytesConstRef _rlp, Permanence _p)
-{
-	return execute(getLastHashes(_bc, _bc.number()), _rlp, _p);
-}
-
-// TODO: maintain node overlay revisions for stateroots -> each commit gives a stateroot + OverlayDB; allow overlay copying for rewind operations.
-ExecutionResult State::execute(LastHashes const& _lh, bytesConstRef _rlp, Permanence _p)
-{
-#ifndef ETH_RELEASE
-	commit();	// get an updated hash
-#endif
-
 	paranoia("start of execution.", true);
-
 	State old(*this);
-#if ETH_PARANOIA
 	auto h = rootHash();
 #endif
 
+	// Create and initialize the executive. This will throw fairly cheaply and quickly if the
+	// transaction is bad in any way.
 	Executive e(*this, _lh, 0);
-	e.setup(_rlp);
+	e.initialize(_t);
 
-	u256 startGasUsed = gasUsed();
+	// Uncommitting is a non-trivial operation - only do it once we've verified as much of the
+	// transaction as possible.
+	uncommitToMine();
 
+	// OK - transaction looks valid - execute.
+	u256 startGasUsed = gasUsed();
 #if ETH_PARANOIA
 	ctrace << "Executing" << e.t() << "on" << h;
 	ctrace << toHex(e.t().rlp());
 #endif
+	if (!e.execute())
 #if ETH_VMTRACE
 		e.go(e.simpleTrace());
 #else

libethereum/State.h

@@ -41,7 +41,7 @@
 namespace dev
 {
 
-namespace test { class ImportTest; }
+namespace test { class ImportTest; class StateLoader; }
 
 namespace eth
 {
@@ -99,6 +99,7 @@ class State
 {
 	friend class ExtVM;
 	friend class dev::test::ImportTest;
+	friend class dev::test::StateLoader;
 	friend class Executive;
 
 public:
@@ -127,6 +128,7 @@ public:
 	OverlayDB const& db() const { return m_db; }
 
 	/// @returns the set containing all addresses currently in use in Ethereum.
+	/// @throws InterfaceNotSupported if compiled without ETH_FATDB.
 	std::map<Address, u256> addresses() const;
 
 	/// Get the header information on the present block.
@@ -186,15 +188,9 @@ public:
 	/// Like sync but only operate on _tq, killing the invalid/old ones.
 	bool cull(TransactionQueue& _tq) const;
 
-	/// Returns the last few block hashes of the current chain.
-	LastHashes getLastHashes(BlockChain const& _bc, unsigned _n) const;
-
 	/// Execute a given transaction.
 	/// This will append @a _t to the transaction list and change the state accordingly.
-	ExecutionResult execute(BlockChain const& _bc, bytes const& _rlp, Permanence _p = Permanence::Committed);
+	ExecutionResult execute(LastHashes const& _lh, Transaction const& _t, Permanence _p = Permanence::Committed);
-	ExecutionResult execute(BlockChain const& _bc, bytesConstRef _rlp, Permanence _p = Permanence::Committed);
-	ExecutionResult execute(LastHashes const& _lh, bytes const& _rlp, Permanence _p = Permanence::Committed) { return execute(_lh, &_rlp, _p); }
-	ExecutionResult execute(LastHashes const& _lh, bytesConstRef _rlp, Permanence _p = Permanence::Committed);
 
 	/// Get the remaining gas limit in this block.
 	u256 gasLimitRemaining() const { return m_currentBlock.gasLimit - gasUsed(); }
@@ -219,6 +215,14 @@ public:
 	 */
 	void subBalance(Address _id, bigint _value);
 
+	/**
+	 * @brief Transfers "the balance @a _value between two accounts.
+	 * @param _from Account from which @a _value will be deducted.
+	 * @param _to Account to which @a _value will be added.
+	 * @param _value Amount to be transferred.
+	 */
+	void transferBalance(Address _from, Address _to, u256 _value) { subBalance(_from, _value); addBalance(_to, _value); }
+
 	/// Get the root of the storage of an account.
 	h256 storageRoot(Address _contract) const;
 

libethereum/Transaction.cpp

@@ -68,6 +68,10 @@ Transaction::Transaction(bytesConstRef _rlpData, CheckSignature _checkSig)
 		m_type = rlp[field = 3].isEmpty() ? ContractCreation : MessageCall;
 		m_receiveAddress = rlp[field = 3].isEmpty() ? Address() : rlp[field = 3].toHash<Address>(RLP::VeryStrict);
 		m_value = rlp[field = 4].toInt<u256>();
+
+		if (!rlp[field = 5].isData())
+			BOOST_THROW_EXCEPTION(BadRLP() << errinfo_comment("transaction data RLP must be an array"));
+
 		m_data = rlp[field = 5].toBytes();
 		byte v = rlp[field = 6].toInt<byte>() - 27;
 		h256 r = rlp[field = 7].toInt<u256>();
@@ -89,7 +93,7 @@ Transaction::Transaction(bytesConstRef _rlpData, CheckSignature _checkSig)
 	}
 }
 
-Address Transaction::safeSender() const noexcept
+Address const& Transaction::safeSender() const noexcept
 {
 	try
 	{
@@ -98,11 +102,11 @@ Address Transaction::safeSender() const noexcept
 	catch (...)
 	{
 		cwarn << "safeSender() did throw an exception: " <<  boost::current_exception_diagnostic_information();
-		return Address();
+		return NullAddress;
 	}
 }
 
-Address Transaction::sender() const
+Address const& Transaction::sender() const
 {
 	if (!m_sender)
 	{

libethereum/Transaction.h

@@ -75,16 +75,30 @@ TransactionException toTransactionException(VMException const& _e);
 struct ExecutionResult
 {
 	ExecutionResult() = default;
-	ExecutionResult(u256 _gasUsed, TransactionException _excepted, Address _newAddress, bytesConstRef _output, CodeDeposit _codeDeposit): gasUsed(_gasUsed), excepted(_excepted), newAddress(_newAddress), output(_output.toBytes()), codeDeposit(_codeDeposit) {}
+	ExecutionResult(u256 const& _gasUsed, TransactionException _excepted, Address const& _newAddress, bytesConstRef _output, CodeDeposit _codeDeposit, u256 const& _gasRefund, unsigned _depositSize, u256 const& _gasForDeposit):
-	u256 gasUsed;
+		gasUsed(_gasUsed),
+		excepted(_excepted),
+		newAddress(_newAddress),
+		output(_output.toBytes()),
+		codeDeposit(_codeDeposit),
+		gasRefunded(_gasRefund),
+		depositSize(_depositSize),
+		gasForDeposit(_gasForDeposit)
+	{}
+	u256 gasUsed = 0;
 	TransactionException excepted = TransactionException::Unknown;
 	Address newAddress;
 	bytes output;
 	CodeDeposit codeDeposit = CodeDeposit::None;
+	u256 gasRefunded = 0;
+	unsigned depositSize = 0;
+	u256 gasForDeposit;
 };
 
 std::ostream& operator<<(std::ostream& _out, ExecutionResult const& _er);
 
+static const Address NullAddress;
+
 /// Encodes a transaction, ready to be exported to or freshly imported from RLP.
 class Transaction
 {
@@ -93,16 +107,16 @@ public:
 	Transaction() {}
 
 	/// Constructs a signed message-call transaction.
-	Transaction(u256 _value, u256 _gasPrice, u256 _gas, Address const& _dest, bytes const& _data, u256 _nonce, Secret const& _secret): m_type(MessageCall), m_nonce(_nonce), m_value(_value), m_receiveAddress(_dest), m_gasPrice(_gasPrice), m_gas(_gas), m_data(_data) { sign(_secret); }
+	Transaction(u256 const& _value, u256 const& _gasPrice, u256 const& _gas, Address const& _dest, bytes const& _data, u256 const& _nonce, Secret const& _secret): m_type(MessageCall), m_nonce(_nonce), m_value(_value), m_receiveAddress(_dest), m_gasPrice(_gasPrice), m_gas(_gas), m_data(_data) { sign(_secret); }
 
 	/// Constructs a signed contract-creation transaction.
-	Transaction(u256 _value, u256 _gasPrice, u256 _gas, bytes const& _data, u256 _nonce, Secret const& _secret): m_type(ContractCreation), m_nonce(_nonce), m_value(_value), m_gasPrice(_gasPrice), m_gas(_gas), m_data(_data) { sign(_secret); }
+	Transaction(u256 const& _value, u256 const& _gasPrice, u256 const& _gas, bytes const& _data, u256 const& _nonce, Secret const& _secret): m_type(ContractCreation), m_nonce(_nonce), m_value(_value), m_gasPrice(_gasPrice), m_gas(_gas), m_data(_data) { sign(_secret); }
 
 	/// Constructs an unsigned message-call transaction.
-	Transaction(u256 _value, u256 _gasPrice, u256 _gas, Address const& _dest, bytes const& _data): m_type(MessageCall), m_value(_value), m_receiveAddress(_dest), m_gasPrice(_gasPrice), m_gas(_gas), m_data(_data) {}
+	Transaction(u256 const& _value, u256 const& _gasPrice, u256 const& _gas, Address const& _dest, bytes const& _data): m_type(MessageCall), m_value(_value), m_receiveAddress(_dest), m_gasPrice(_gasPrice), m_gas(_gas), m_data(_data) {}
 
 	/// Constructs an unsigned contract-creation transaction.
-	Transaction(u256 _value, u256 _gasPrice, u256 _gas, bytes const& _data): m_type(ContractCreation), m_value(_value), m_gasPrice(_gasPrice), m_gas(_gas), m_data(_data) {}
+	Transaction(u256 const& _value, u256 const& _gasPrice, u256 const& _gas, bytes const& _data): m_type(ContractCreation), m_value(_value), m_gasPrice(_gasPrice), m_gas(_gas), m_data(_data) {}
 
 	/// Constructs a transaction from the given RLP.
 	explicit Transaction(bytesConstRef _rlp, CheckSignature _checkSig);
@@ -117,9 +131,9 @@ public:
 	bool operator!=(Transaction const& _c) const { return !operator==(_c); }
 
 	/// @returns sender of the transaction from the signature (and hash).
-	Address sender() const;
+	Address const& sender() const;
 	/// Like sender() but will never throw. @returns a null Address if the signature is invalid.
-	Address safeSender() const noexcept;
+	Address const& safeSender() const noexcept;
 
 	/// @returns true if transaction is non-null.
 	explicit operator bool() const { return m_type != NullTransaction; }

libethereum/TransactionQueue.cpp

@@ -46,7 +46,7 @@ bool TransactionQueue::import(bytesConstRef _transactionRLP)
 
 		UpgradeGuard ul(l);
 		// If valid, append to blocks.
-		m_current[h] = _transactionRLP.toBytes();
+		m_current[h] = t;
 		m_known.insert(h);
 	}
 	catch (Exception const& _e)
@@ -63,20 +63,20 @@ bool TransactionQueue::import(bytesConstRef _transactionRLP)
 	return true;
 }
 
-void TransactionQueue::setFuture(std::pair<h256, bytes> const& _t)
+void TransactionQueue::setFuture(std::pair<h256, Transaction> const& _t)
 {
 	WriteGuard l(m_lock);
 	if (m_current.count(_t.first))
 	{
 		m_current.erase(_t.first);
-		m_unknown.insert(make_pair(Transaction(_t.second, CheckSignature::Sender).sender(), _t));
+		m_unknown.insert(make_pair(_t.second.sender(), _t));
 	}
 }
 
-void TransactionQueue::noteGood(std::pair<h256, bytes> const& _t)
+void TransactionQueue::noteGood(std::pair<h256, Transaction> const& _t)
 {
 	WriteGuard l(m_lock);
-	auto r = m_unknown.equal_range(Transaction(_t.second, CheckSignature::Sender).sender());
+	auto r = m_unknown.equal_range(_t.second.sender());
 	for (auto it = r.first; it != r.second; ++it)
 		m_current.insert(it->second);
 	m_unknown.erase(r.first, r.second);

libethereum/TransactionQueue.h

@@ -25,6 +25,7 @@
 #include <libdevcore/Common.h>
 #include "libethcore/Common.h"
 #include <libdevcore/Guards.h>
+#include "Transaction.h"
 
 namespace dev
 {
@@ -46,19 +47,19 @@ public:
 
 	void drop(h256 _txHash);
 
-	std::map<h256, bytes> transactions() const { ReadGuard l(m_lock); return m_current; }
+	std::map<h256, Transaction> transactions() const { ReadGuard l(m_lock); return m_current; }
 	std::pair<unsigned, unsigned> items() const { ReadGuard l(m_lock); return std::make_pair(m_current.size(), m_unknown.size()); }
 
-	void setFuture(std::pair<h256, bytes> const& _t);
+	void setFuture(std::pair<h256, Transaction> const& _t);
-	void noteGood(std::pair<h256, bytes> const& _t);
+	void noteGood(std::pair<h256, Transaction> const& _t);
 
 	void clear() { WriteGuard l(m_lock); m_known.clear(); m_current.clear(); m_unknown.clear(); }
 
 private:
 	mutable boost::shared_mutex m_lock;							///< General lock.
 	std::set<h256> m_known;										///< Hashes of transactions in both sets.
-	std::map<h256, bytes> m_current;							///< Map of SHA3(tx) to tx.
+	std::map<h256, Transaction> m_current;						///< Map of SHA3(tx) to tx.
-	std::multimap<Address, std::pair<h256, bytes>> m_unknown;	///< For transactions that have a future nonce; we map their sender address to the tx stuff, and insert once the sender has a valid TX.
+	std::multimap<Address, std::pair<h256, Transaction>> m_unknown;	///< For transactions that have a future nonce; we map their sender address to the tx stuff, and insert once the sender has a valid TX.
 };
 
 }

libevm/VM.cpp

@@ -614,10 +614,7 @@ bytesConstRef VM::go(ExtVMFace& _ext, OnOpFunc const& _onOp, uint64_t _steps)
 			m_stack.pop_back();
 
 			if (_ext.balance(_ext.myAddress) >= endowment && _ext.depth < 1024)
-			{
-				_ext.subBalance(endowment);
 				m_stack.push_back((u160)_ext.create(endowment, m_gas, bytesConstRef(m_temp.data() + initOff, initSize), _onOp));
-			}
 			else
 				m_stack.push_back(0);
 			break;
@@ -644,10 +641,7 @@ bytesConstRef VM::go(ExtVMFace& _ext, OnOpFunc const& _onOp, uint64_t _steps)
 			m_stack.pop_back();
 
 			if (_ext.balance(_ext.myAddress) >= value && _ext.depth < 1024)
-			{
-				_ext.subBalance(value);
 				m_stack.push_back(_ext.call(inst == Instruction::CALL ? receiveAddress : _ext.myAddress, value, bytesConstRef(m_temp.data() + inOff, inSize), gas, bytesRef(m_temp.data() + outOff, outSize), _onOp, {}, receiveAddress));
-			}
 			else
 				m_stack.push_back(0);
 

libevmcore/Assembly.cpp

@@ -20,140 +20,23 @@
  */
 
 #include "Assembly.h"
-
 #include <fstream>
-
 #include <libdevcore/Log.h>
+#include <libevmcore/CommonSubexpressionEliminator.h>
 
 using namespace std;
 using namespace dev;
 using namespace dev::eth;
 
-unsigned AssemblyItem::bytesRequired(unsigned _addressLength) const
-{
-	switch (m_type)
-	{
-	case Operation:
-	case Tag: // 1 byte for the JUMPDEST
-		return 1;
-	case PushString:
-		return 33;
-	case Push:
-		return 1 + max<unsigned>(1, dev::bytesRequired(m_data));
-	case PushSubSize:
-	case PushProgramSize:
-		return 4;		// worst case: a 16MB program
-	case PushTag:
-	case PushData:
-	case PushSub:
-		return 1 + _addressLength;
-	default:
-		break;
-	}
-	BOOST_THROW_EXCEPTION(InvalidOpcode());
-}
-
-int AssemblyItem::deposit() const
-{
-	switch (m_type)
-	{
-	case Operation:
-		return instructionInfo(instruction()).ret - instructionInfo(instruction()).args;
-	case Push:
-	case PushString:
-	case PushTag:
-	case PushData:
-	case PushSub:
-	case PushSubSize:
-	case PushProgramSize:
-		return 1;
-	case Tag:
-		return 0;
-	default:;
-	}
-	return 0;
-}
-
-string AssemblyItem::getJumpTypeAsString() const
-{
-	switch (m_jumpType)
-	{
-	case JumpType::IntoFunction:
-		return "[in]";
-	case JumpType::OutOfFunction:
-		return "[out]";
-	case JumpType::Ordinary:
-	default:
-		return "";
-	}
-}
-
-ostream& dev::eth::operator<<(ostream& _out, AssemblyItem const& _item)
-{
-	switch (_item.type())
-	{
-	case Operation:
-		_out << " " << instructionInfo(_item.instruction()).name;
-		if (_item.instruction() == eth::Instruction::JUMP || _item.instruction() == eth::Instruction::JUMPI)
-			_out << "\t" << _item.getJumpTypeAsString();
-		break;
-	case Push:
-		_out << " PUSH " << hex << _item.data();
-		break;
-	case PushString:
-		_out << " PushString"  << hex << (unsigned)_item.data();
-		break;
-	case PushTag:
-		_out << " PushTag " << _item.data();
-		break;
-	case Tag:
-		_out << " Tag " << _item.data();
-		break;
-	case PushData:
-		_out << " PushData " << hex << (unsigned)_item.data();
-		break;
-	case PushSub:
-		_out << " PushSub " << hex << h256(_item.data()).abridged();
-		break;
-	case PushSubSize:
-		_out << " PushSubSize " << hex << h256(_item.data()).abridged();
-		break;
-	case PushProgramSize:
-		_out << " PushProgramSize";
-		break;
-	case UndefinedItem:
-		_out << " ???";
-		break;
-	default:
-		BOOST_THROW_EXCEPTION(InvalidOpcode());
-	}
-	return _out;
-}
-
-unsigned Assembly::bytesRequired() const
-{
-	for (unsigned br = 1;; ++br)
-	{
-		unsigned ret = 1;
-		for (auto const& i: m_data)
-			ret += i.second.size();
-
-		for (AssemblyItem const& i: m_items)
-			ret += i.bytesRequired(br);
-		if (dev::bytesRequired(ret) <= br)
-			return ret;
-	}
-}
-
 void Assembly::append(Assembly const& _a)
 {
 	auto newDeposit = m_deposit + _a.deposit();
 	for (AssemblyItem i: _a.m_items)
 	{
 		if (i.type() == Tag || i.type() == PushTag)
-			i.m_data += m_usedTags;
+			i.setData(i.data() + m_usedTags);
 		else if (i.type() == PushSub || i.type() == PushSubSize)
-			i.m_data += m_subs.size();
+			i.setData(i.data() + m_usedTags);
 		append(i);
 	}
 	m_deposit = newDeposit;
@@ -181,11 +64,19 @@ void Assembly::append(Assembly const& _a, int _deposit)
 	}
 }
 
-ostream& dev::eth::operator<<(ostream& _out, AssemblyItemsConstRef _i)
+unsigned Assembly::bytesRequired() const
 {
-	for (AssemblyItem const& i: _i)
+	for (unsigned br = 1;; ++br)
-		_out << i;
+	{
-	return _out;
+		unsigned ret = 1;
+		for (auto const& i: m_data)
+			ret += i.second.size();
+
+		for (AssemblyItem const& i: m_items)
+			ret += i.bytesRequired(br);
+		if (dev::bytesRequired(ret) <= br)
+			return ret;
+	}
 }
 
 string Assembly::getLocationFromSources(StringMap const& _sourceCodes, SourceLocation const& _location) const
@@ -215,34 +106,34 @@ ostream& Assembly::stream(ostream& _out, string const& _prefix, StringMap const&
 	for (AssemblyItem const& i: m_items)
 	{
 		_out << _prefix;
-		switch (i.m_type)
+		switch (i.type())
 		{
 		case Operation:
 			_out << "  " << instructionInfo(i.instruction()).name  << "\t" << i.getJumpTypeAsString();
 			break;
 		case Push:
-			_out << "  PUSH " << i.m_data;
+			_out << "  PUSH " << i.data();
 			break;
 		case PushString:
-			_out << "  PUSH \"" << m_strings.at((h256)i.m_data) << "\"";
+			_out << "  PUSH \"" << m_strings.at((h256)i.data()) << "\"";
 			break;
 		case PushTag:
-			_out << "  PUSH [tag" << i.m_data << "]";
+			_out << "  PUSH [tag" << i.data() << "]";
 			break;
 		case PushSub:
-			_out << "  PUSH [$" << h256(i.m_data).abridged() << "]";
+			_out << "  PUSH [$" << h256(i.data()).abridged() << "]";
 			break;
 		case PushSubSize:
-			_out << "  PUSH #[$" << h256(i.m_data).abridged() << "]";
+			_out << "  PUSH #[$" << h256(i.data()).abridged() << "]";
 			break;
 		case PushProgramSize:
 			_out << "  PUSHSIZE";
 			break;
 		case Tag:
-			_out << "tag" << i.m_data << ": " << endl << _prefix << "  JUMPDEST";
+			_out << "tag" << i.data() << ": " << endl << _prefix << "  JUMPDEST";
 			break;
 		case PushData:
-			_out << "  PUSH [" << hex << (unsigned)i.m_data << "]";
+			_out << "  PUSH [" << hex << (unsigned)i.data() << "]";
 			break;
 		default:
 			BOOST_THROW_EXCEPTION(InvalidOpcode());
@@ -289,24 +180,6 @@ inline bool matches(AssemblyItemsConstRef _a, AssemblyItemsConstRef _b)
 	return true;
 }
 
-inline bool popCountIncreased(AssemblyItemsConstRef _pre, AssemblyItems const& _post)
-{
-	auto isPop = [](AssemblyItem const& _item) -> bool { return _item.match(AssemblyItem(Instruction::POP)); };
-	return count_if(begin(_post), end(_post), isPop) > count_if(begin(_pre), end(_pre), isPop);
-}
-
-//@todo this has to move to a special optimizer class soon
-template<class Iterator>
-unsigned bytesRequiredBySlice(Iterator _begin, Iterator _end)
-{
-	// this is only used in the optimizer, so we can provide a guess for the address length
-	unsigned addressLength = 4;
-	unsigned size = 0;
-	for (; _begin != _end; ++_begin)
-		size += _begin->bytesRequired(addressLength);
-	return size;
-}
-
 struct OptimiserChannel: public LogChannel { static const char* name() { return "OPT"; } static const int verbosity = 12; };
 #define copt dev::LogOutputStream<OptimiserChannel, true>()
 
@@ -314,107 +187,45 @@ Assembly& Assembly::optimise(bool _enable)
 {
 	if (!_enable)
 		return *this;
-	auto signextend = [](u256 a, u256 b) -> u256
+	std::vector<pair<AssemblyItems, function<AssemblyItems(AssemblyItemsConstRef)>>> rules;
-	{
-		if (a >= 31)
-			return b;
-		unsigned testBit = unsigned(a) * 8 + 7;
-		u256 mask = (u256(1) << testBit) - 1;
-		return boost::multiprecision::bit_test(b, testBit) ? b | ~mask : b & mask;
-	};
-	map<Instruction, function<u256(u256, u256)>> const c_simple =
-	{
-		{ Instruction::SUB, [](u256 a, u256 b)->u256{return a - b;} },
-		{ Instruction::DIV, [](u256 a, u256 b)->u256{return a / b;} },
-		{ Instruction::SDIV, [](u256 a, u256 b)->u256{return s2u(u2s(a) / u2s(b));} },
-		{ Instruction::MOD, [](u256 a, u256 b)->u256{return a % b;} },
-		{ Instruction::SMOD, [](u256 a, u256 b)->u256{return s2u(u2s(a) % u2s(b));} },
-		{ Instruction::EXP, [](u256 a, u256 b)->u256{return (u256)boost::multiprecision::powm((bigint)a, (bigint)b, bigint(1) << 256);} },
-		{ Instruction::SIGNEXTEND, signextend },
-		{ Instruction::LT, [](u256 a, u256 b)->u256{return a < b ? 1 : 0;} },
-		{ Instruction::GT, [](u256 a, u256 b)->u256{return a > b ? 1 : 0;} },
-		{ Instruction::SLT, [](u256 a, u256 b)->u256{return u2s(a) < u2s(b) ? 1 : 0;} },
-		{ Instruction::SGT, [](u256 a, u256 b)->u256{return u2s(a) > u2s(b) ? 1 : 0;} },
-		{ Instruction::EQ, [](u256 a, u256 b)->u256{return a == b ? 1 : 0;} },
-	};
-	map<Instruction, function<u256(u256, u256)>> const c_associative =
-	{
-		{ Instruction::ADD, [](u256 a, u256 b)->u256{return a + b;} },
-		{ Instruction::MUL, [](u256 a, u256 b)->u256{return a * b;} },
-		{ Instruction::AND, [](u256 a, u256 b)->u256{return a & b;} },
-		{ Instruction::OR, [](u256 a, u256 b)->u256{return a | b;} },
-		{ Instruction::XOR, [](u256 a, u256 b)->u256{return a ^ b;} },
-	};
-	std::vector<pair<AssemblyItem, u256>> const c_identities =
-	{ { Instruction::ADD, 0}, { Instruction::MUL, 1}, { Instruction::MOD, 0}, { Instruction::OR, 0}, { Instruction::XOR, 0} };
-	std::vector<pair<AssemblyItems, function<AssemblyItems(AssemblyItemsConstRef)>>> rules =
-	{
-		{ { Push, Instruction::POP }, [](AssemblyItemsConstRef) -> AssemblyItems { return {}; } },
-		{ { PushTag, Instruction::POP }, [](AssemblyItemsConstRef) -> AssemblyItems { return {}; } },
-		{ { PushString, Instruction::POP }, [](AssemblyItemsConstRef) -> AssemblyItems { return {}; } },
-		{ { PushSub, Instruction::POP }, [](AssemblyItemsConstRef) -> AssemblyItems { return {}; } },
-		{ { PushSubSize, Instruction::POP }, [](AssemblyItemsConstRef) -> AssemblyItems { return {}; } },
-		{ { PushProgramSize, Instruction::POP }, [](AssemblyItemsConstRef) -> AssemblyItems { return {}; } },
-		{ { Push, PushTag, Instruction::JUMPI }, [](AssemblyItemsConstRef m) -> AssemblyItems { if (m[0].data()) return { m[1], Instruction::JUMP }; else return {}; } },
-		{ { Instruction::ISZERO, Instruction::ISZERO }, [](AssemblyItemsConstRef) -> AssemblyItems { return {}; } },
-	};
-
-	for (auto const& i: c_simple)
-		rules.push_back({ { Push, Push, i.first }, [&](AssemblyItemsConstRef m) -> AssemblyItems { return { i.second(m[1].data(), m[0].data()) }; } });
-	for (auto const& i: c_associative)
-	{
-		rules.push_back({ { Push, Push, i.first }, [&](AssemblyItemsConstRef m) -> AssemblyItems { return { i.second(m[1].data(), m[0].data()) }; } });
-		rules.push_back({ { Push, i.first, Push, i.first }, [&](AssemblyItemsConstRef m) -> AssemblyItems { return { i.second(m[2].data(), m[0].data()), i.first }; } });
-	}
-	for (auto const& i: c_identities)
-		rules.push_back({{Push, i.first}, [&](AssemblyItemsConstRef m) -> AssemblyItems
-							{ return m[0].data() == i.second ? AssemblyItems() : m.toVector(); }});
 	// jump to next instruction
-	rules.push_back({ { PushTag, Instruction::JUMP, Tag }, [](AssemblyItemsConstRef m) -> AssemblyItems { if (m[0].m_data == m[2].m_data) return {m[2]}; else return m.toVector(); }});
+	rules.push_back({ { PushTag, Instruction::JUMP, Tag }, [](AssemblyItemsConstRef m) -> AssemblyItems { if (m[0].data() == m[2].data()) return {m[2]}; else return m.toVector(); }});
-
-	// pop optimization, do not compute values that are popped again anyway
-	rules.push_back({ { AssemblyItem(UndefinedItem), Instruction::POP }, [](AssemblyItemsConstRef m) -> AssemblyItems
-					  {
-						  if (m[0].type() != Operation)
-							return m.toVector();
-						  Instruction instr = m[0].instruction();
-						  if (Instruction::DUP1 <= instr && instr <= Instruction::DUP16)
-							return {};
-						  InstructionInfo info = instructionInfo(instr);
-						  if (info.sideEffects || info.additional != 0 || info.ret != 1)
-							  return m.toVector();
-						  return AssemblyItems(info.args, Instruction::POP);
-					  } });
-	// compute constants close to powers of two by expressions
-	auto computeConstants = [](AssemblyItemsConstRef m) -> AssemblyItems
-	{
-		u256 const& c = m[0].data();
-		unsigned const minBits = 4 * 8;
-		if (c < (bigint(1) << minBits))
-			return m.toVector(); // we need at least "PUSH1 <bits> PUSH1 <2> EXP"
-		if (c == u256(-1))
-			return {u256(0), Instruction::NOT};
-		for (unsigned bits = minBits; bits < 256; ++bits)
-		{
-			bigint const diff = c - (bigint(1) << bits);
-			if (abs(diff) > 0xff)
-				continue;
-			AssemblyItems powerOfTwo{u256(bits), u256(2), Instruction::EXP};
-			if (diff == 0)
-				return powerOfTwo;
-			return AssemblyItems{u256(abs(diff))} + powerOfTwo +
-				   AssemblyItems{diff > 0 ? Instruction::ADD : Instruction::SUB};
-		}
-		return m.toVector();
-	};
-	rules.push_back({{Push}, computeConstants});
-
-	copt << *this;
 
 	unsigned total = 0;
 	for (unsigned count = 1; count > 0; total += count)
 	{
+		copt << *this;
 		count = 0;
+
+		copt << "Performing common subexpression elimination...";
+		for (auto iter = m_items.begin(); iter != m_items.end();)
+		{
+			CommonSubexpressionEliminator eliminator;
+			auto orig = iter;
+			iter = eliminator.feedItems(iter, m_items.end());
+			AssemblyItems optItems;
+			bool shouldReplace = false;
+			try
+			{
+				optItems = eliminator.getOptimizedItems();
+				shouldReplace = (optItems.size() < size_t(iter - orig));
+			}
+			catch (StackTooDeepException const&)
+			{
+				// This might happen if the opcode reconstruction is not as efficient
+				// as the hand-crafted code.
+			}
+
+			if (shouldReplace)
+			{
+				copt << "Old size: " << (iter - orig) << ", new size: " << optItems.size();
+				count++;
+				for (auto moveIter = optItems.begin(); moveIter != optItems.end(); ++orig, ++moveIter)
+					*orig = move(*moveIter);
+				iter = m_items.erase(orig, iter);
+			}
+		}
+
 		for (unsigned i = 0; i < m_items.size(); ++i)
 		{
 			for (auto const& r: rules)
@@ -423,12 +234,10 @@ Assembly& Assembly::optimise(bool _enable)
 				if (matches(vr, &r.first))
 				{
 					auto rw = r.second(vr);
-					unsigned const vrSizeInBytes = bytesRequiredBySlice(vr.begin(), vr.end());
+					if (rw.size() < vr.size())
-					unsigned const rwSizeInBytes = bytesRequiredBySlice(rw.begin(), rw.end());
-					if (rwSizeInBytes < vrSizeInBytes || (rwSizeInBytes == vrSizeInBytes && popCountIncreased(vr, rw)))
 					{
-						copt << vr << "matches" << AssemblyItemsConstRef(&r.first) << "becomes...";
+						copt << "Rule " << vr << " matches " << AssemblyItemsConstRef(&r.first) << " becomes...";
-						copt << AssemblyItemsConstRef(&rw);
+						copt << AssemblyItemsConstRef(&rw) << "\n";
 						if (rw.size() > vr.size())
 						{
 							// create hole in the vector
@@ -442,7 +251,7 @@ Assembly& Assembly::optimise(bool _enable)
 						copy(rw.begin(), rw.end(), m_items.begin() + i);
 
 						count++;
-						copt << "Now:\n" << m_items;
+						copt << "Now:" << m_items;
 					}
 				}
 			}
@@ -522,16 +331,16 @@ bytes Assembly::assemble() const
 	// m_data must not change from here on
 
 	for (AssemblyItem const& i: m_items)
-		switch (i.m_type)
+		switch (i.type())
 		{
 		case Operation:
-			ret.push_back((byte)i.m_data);
+			ret.push_back((byte)i.data());
 			break;
 		case PushString:
 		{
 			ret.push_back((byte)Instruction::PUSH32);
 			unsigned ii = 0;
-			for (auto j: m_strings.at((h256)i.m_data))
+			for (auto j: m_strings.at((h256)i.data()))
 				if (++ii > 32)
 					break;
 				else
@@ -542,30 +351,30 @@ bytes Assembly::assemble() const
 		}
 		case Push:
 		{
-			byte b = max<unsigned>(1, dev::bytesRequired(i.m_data));
+			byte b = max<unsigned>(1, dev::bytesRequired(i.data()));
 			ret.push_back((byte)Instruction::PUSH1 - 1 + b);
 			ret.resize(ret.size() + b);
 			bytesRef byr(&ret.back() + 1 - b, b);
-			toBigEndian(i.m_data, byr);
+			toBigEndian(i.data(), byr);
 			break;
 		}
 		case PushTag:
 		{
 			ret.push_back(tagPush);
-			tagRef[ret.size()] = (unsigned)i.m_data;
+			tagRef[ret.size()] = (unsigned)i.data();
 			ret.resize(ret.size() + bytesPerTag);
 			break;
 		}
 		case PushData: case PushSub:
 		{
 			ret.push_back(dataRefPush);
-			dataRef.insert(make_pair((h256)i.m_data, ret.size()));
+			dataRef.insert(make_pair((h256)i.data(), ret.size()));
 			ret.resize(ret.size() + bytesPerDataRef);
 			break;
 		}
 		case PushSubSize:
 		{
-			auto s = m_data[i.m_data].size();
+			auto s = m_data[i.data()].size();
 			byte b = max<unsigned>(1, dev::bytesRequired(s));
 			ret.push_back((byte)Instruction::PUSH1 - 1 + b);
 			ret.resize(ret.size() + b);
@@ -581,7 +390,7 @@ bytes Assembly::assemble() const
 			break;
 		}
 		case Tag:
-			tagPos[(unsigned)i.m_data] = ret.size();
+			tagPos[(unsigned)i.data()] = ret.size();
 			ret.push_back((byte)Instruction::JUMPDEST);
 			break;
 		default:

libevmcore/Assembly.h

@@ -27,6 +27,7 @@
 #include <libdevcore/Assertions.h>
 #include <libevmcore/SourceLocation.h>
 #include <libevmcore/Instruction.h>
+#include <libevmcore/AssemblyItem.h>
 #include "Exceptions.h"
 
 namespace dev
@@ -34,60 +35,6 @@ namespace dev
 namespace eth
 {
 
-enum AssemblyItemType { UndefinedItem, Operation, Push, PushString, PushTag, PushSub, PushSubSize, PushProgramSize, Tag, PushData };
-
-class Assembly;
-
-class AssemblyItem
-{
-	friend class Assembly;
-
-public:
-	enum class JumpType { Ordinary, IntoFunction, OutOfFunction };
-
-	AssemblyItem(u256 _push): m_type(Push), m_data(_push) {}
-	AssemblyItem(Instruction _i): m_type(Operation), m_data((byte)_i) {}
-	AssemblyItem(AssemblyItemType _type, u256 _data = 0): m_type(_type), m_data(_data) {}
-
-	AssemblyItem tag() const { assertThrow(m_type == PushTag || m_type == Tag, Exception, ""); return AssemblyItem(Tag, m_data); }
-	AssemblyItem pushTag() const { assertThrow(m_type == PushTag || m_type == Tag, Exception, ""); return AssemblyItem(PushTag, m_data); }
-
-	AssemblyItemType type() const { return m_type; }
-	u256 const& data() const { return m_data; }
-	/// @returns the instruction of this item (only valid if type() == Operation)
-	Instruction instruction() const { return Instruction(byte(m_data)); }
-
-	/// @returns true iff the type and data of the items are equal.
-	bool operator==(AssemblyItem const& _other) const { return m_type == _other.m_type && m_data == _other.m_data; }
-	bool operator!=(AssemblyItem const& _other) const { return !operator==(_other); }
-
-	/// @returns an upper bound for the number of bytes required by this item, assuming that
-	/// the value of a jump tag takes @a _addressLength bytes.
-	unsigned bytesRequired(unsigned _addressLength) const;
-	int deposit() const;
-
-	bool match(AssemblyItem const& _i) const { return _i.m_type == UndefinedItem || (m_type == _i.m_type && (m_type != Operation || m_data == _i.m_data)); }
-	void setLocation(SourceLocation const& _location) { m_location = _location; }
-	SourceLocation const& getLocation() const { return m_location; }
-
-	void setJumpType(JumpType _jumpType) { m_jumpType = _jumpType; }
-	JumpType getJumpType() const { return m_jumpType; }
-	std::string getJumpTypeAsString() const;
-
-private:
-	AssemblyItemType m_type;
-	u256 m_data;
-	SourceLocation m_location;
-	JumpType m_jumpType = JumpType::Ordinary;
-};
-
-using AssemblyItems = std::vector<AssemblyItem>;
-using AssemblyItemsConstRef = vector_ref<AssemblyItem const>;
-
-std::ostream& operator<<(std::ostream& _out, AssemblyItem const& _item);
-std::ostream& operator<<(std::ostream& _out, AssemblyItemsConstRef _i);
-inline std::ostream& operator<<(std::ostream& _out, AssemblyItems const& _i) { return operator<<(_out, AssemblyItemsConstRef(&_i)); }
-
 class Assembly
 {
 public:
@@ -118,7 +65,7 @@ public:
 	template <class T> Assembly& operator<<(T const& _d) { append(_d); return *this; }
 	AssemblyItems const& getItems() const { return m_items; }
 	AssemblyItem const& back() const { return m_items.back(); }
-	std::string backString() const { return m_items.size() && m_items.back().m_type == PushString ? m_strings.at((h256)m_items.back().m_data) : std::string(); }
+	std::string backString() const { return m_items.size() && m_items.back().type() == PushString ? m_strings.at((h256)m_items.back().data()) : std::string(); }
 
 	void onePath() { if (asserts(!m_totalDeposit && !m_baseDeposit)) BOOST_THROW_EXCEPTION(InvalidDeposit()); m_baseDeposit = m_deposit; m_totalDeposit = INT_MAX; }
 	void otherPath() { donePath(); m_totalDeposit = m_deposit; m_deposit = m_baseDeposit; }

libevmcore/AssemblyItem.cpp

@@ -0,0 +1,135 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file Assembly.cpp
+ * @author Gav Wood <i@gavwood.com>
+ * @date 2014
+ */
+
+#include "AssemblyItem.h"
+#include <fstream>
+
+using namespace std;
+using namespace dev;
+using namespace dev::eth;
+
+unsigned AssemblyItem::bytesRequired(unsigned _addressLength) const
+{
+	switch (m_type)
+	{
+	case Operation:
+	case Tag: // 1 byte for the JUMPDEST
+		return 1;
+	case PushString:
+		return 33;
+	case Push:
+		return 1 + max<unsigned>(1, dev::bytesRequired(m_data));
+	case PushSubSize:
+	case PushProgramSize:
+		return 4;		// worst case: a 16MB program
+	case PushTag:
+	case PushData:
+	case PushSub:
+		return 1 + _addressLength;
+	default:
+		break;
+	}
+	BOOST_THROW_EXCEPTION(InvalidOpcode());
+}
+
+int AssemblyItem::deposit() const
+{
+	switch (m_type)
+	{
+	case Operation:
+		return instructionInfo(instruction()).ret - instructionInfo(instruction()).args;
+	case Push:
+	case PushString:
+	case PushTag:
+	case PushData:
+	case PushSub:
+	case PushSubSize:
+	case PushProgramSize:
+		return 1;
+	case Tag:
+		return 0;
+	default:;
+	}
+	return 0;
+}
+
+string AssemblyItem::getJumpTypeAsString() const
+{
+	switch (m_jumpType)
+	{
+	case JumpType::IntoFunction:
+		return "[in]";
+	case JumpType::OutOfFunction:
+		return "[out]";
+	case JumpType::Ordinary:
+	default:
+		return "";
+	}
+}
+
+ostream& dev::eth::operator<<(ostream& _out, AssemblyItem const& _item)
+{
+	switch (_item.type())
+	{
+	case Operation:
+		_out << " " << instructionInfo(_item.instruction()).name;
+		if (_item.instruction() == eth::Instruction::JUMP || _item.instruction() == eth::Instruction::JUMPI)
+			_out << "\t" << _item.getJumpTypeAsString();
+		break;
+	case Push:
+		_out << " PUSH " << hex << _item.data();
+		break;
+	case PushString:
+		_out << " PushString"  << hex << (unsigned)_item.data();
+		break;
+	case PushTag:
+		_out << " PushTag " << _item.data();
+		break;
+	case Tag:
+		_out << " Tag " << _item.data();
+		break;
+	case PushData:
+		_out << " PushData " << hex << (unsigned)_item.data();
+		break;
+	case PushSub:
+		_out << " PushSub " << hex << h256(_item.data()).abridged();
+		break;
+	case PushSubSize:
+		_out << " PushSubSize " << hex << h256(_item.data()).abridged();
+		break;
+	case PushProgramSize:
+		_out << " PushProgramSize";
+		break;
+	case UndefinedItem:
+		_out << " ???";
+		break;
+	default:
+		BOOST_THROW_EXCEPTION(InvalidOpcode());
+	}
+	return _out;
+}
+
+ostream& dev::eth::operator<<(ostream& _out, AssemblyItemsConstRef _i)
+{
+	for (AssemblyItem const& i: _i)
+		_out << i;
+	return _out;
+}

libevmcore/AssemblyItem.h

@@ -0,0 +1,93 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file Assembly.h
+ * @author Gav Wood <i@gavwood.com>
+ * @date 2014
+ */
+
+#pragma once
+
+#include <iostream>
+#include <sstream>
+#include <libdevcore/Common.h>
+#include <libdevcore/Assertions.h>
+#include <libevmcore/SourceLocation.h>
+#include <libevmcore/Instruction.h>
+#include "Exceptions.h"
+
+namespace dev
+{
+namespace eth
+{
+
+enum AssemblyItemType { UndefinedItem, Operation, Push, PushString, PushTag, PushSub, PushSubSize, PushProgramSize, Tag, PushData };
+
+class Assembly;
+
+class AssemblyItem
+{
+public:
+	enum class JumpType { Ordinary, IntoFunction, OutOfFunction };
+
+	AssemblyItem(u256 _push): m_type(Push), m_data(_push) {}
+	AssemblyItem(Instruction _i): m_type(Operation), m_data((byte)_i) {}
+	AssemblyItem(AssemblyItemType _type, u256 _data = 0): m_type(_type), m_data(_data) {}
+
+	AssemblyItem tag() const { assertThrow(m_type == PushTag || m_type == Tag, Exception, ""); return AssemblyItem(Tag, m_data); }
+	AssemblyItem pushTag() const { assertThrow(m_type == PushTag || m_type == Tag, Exception, ""); return AssemblyItem(PushTag, m_data); }
+
+	AssemblyItemType type() const { return m_type; }
+	u256 const& data() const { return m_data; }
+	void setType(AssemblyItemType const _type) { m_type = _type; }
+	void setData(u256 const& _data) { m_data = _data; }
+
+	/// @returns the instruction of this item (only valid if type() == Operation)
+	Instruction instruction() const { return Instruction(byte(m_data)); }
+
+	/// @returns true iff the type and data of the items are equal.
+	bool operator==(AssemblyItem const& _other) const { return m_type == _other.m_type && m_data == _other.m_data; }
+	bool operator!=(AssemblyItem const& _other) const { return !operator==(_other); }
+
+	/// @returns an upper bound for the number of bytes required by this item, assuming that
+	/// the value of a jump tag takes @a _addressLength bytes.
+	unsigned bytesRequired(unsigned _addressLength) const;
+	int deposit() const;
+
+	bool match(AssemblyItem const& _i) const { return _i.m_type == UndefinedItem || (m_type == _i.m_type && (m_type != Operation || m_data == _i.m_data)); }
+	void setLocation(SourceLocation const& _location) { m_location = _location; }
+	SourceLocation const& getLocation() const { return m_location; }
+
+	void setJumpType(JumpType _jumpType) { m_jumpType = _jumpType; }
+	JumpType getJumpType() const { return m_jumpType; }
+	std::string getJumpTypeAsString() const;
+
+private:
+	AssemblyItemType m_type;
+	u256 m_data;
+	SourceLocation m_location;
+	JumpType m_jumpType = JumpType::Ordinary;
+};
+
+using AssemblyItems = std::vector<AssemblyItem>;
+using AssemblyItemsConstRef = vector_ref<AssemblyItem const>;
+
+std::ostream& operator<<(std::ostream& _out, AssemblyItem const& _item);
+std::ostream& operator<<(std::ostream& _out, AssemblyItemsConstRef _i);
+inline std::ostream& operator<<(std::ostream& _out, AssemblyItems const& _i) { return operator<<(_out, AssemblyItemsConstRef(&_i)); }
+
+}
+}

libevmcore/CommonSubexpressionEliminator.cpp

@@ -0,0 +1,558 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @file CommonSubexpressionEliminator.cpp
+ * @author Christian <c@ethdev.com>
+ * @date 2015
+ * Optimizer step for common subexpression elimination and stack reorganisation.
+ */
+
+#include <functional>
+#include <boost/range/adaptor/reversed.hpp>
+#include <libevmcore/CommonSubexpressionEliminator.h>
+#include <libevmcore/AssemblyItem.h>
+
+using namespace std;
+using namespace dev;
+using namespace dev::eth;
+
+vector<AssemblyItem> CommonSubexpressionEliminator::getOptimizedItems()
+{
+	optimizeBreakingItem();
+
+	map<int, ExpressionClasses::Id> initialStackContents;
+	map<int, ExpressionClasses::Id> targetStackContents;
+	int minHeight = m_stackHeight + 1;
+	if (!m_stackElements.empty())
+		minHeight = min(minHeight, m_stackElements.begin()->first);
+	for (int height = minHeight; height <= 0; ++height)
+		initialStackContents[height] = initialStackElement(height);
+	for (int height = minHeight; height <= m_stackHeight; ++height)
+		targetStackContents[height] = stackElement(height);
+
+	// Debug info:
+	//stream(cout, initialStackContents, targetStackContents);
+
+	AssemblyItems items = CSECodeGenerator(m_expressionClasses, m_storeOperations).generateCode(
+		initialStackContents,
+		targetStackContents
+	);
+	if (m_breakingItem)
+		items.push_back(*m_breakingItem);
+	return items;
+}
+
+ostream& CommonSubexpressionEliminator::stream(
+	ostream& _out,
+	map<int, ExpressionClasses::Id> _initialStack,
+	map<int, ExpressionClasses::Id> _targetStack
+) const
+{
+	auto streamExpressionClass = [this](ostream& _out, ExpressionClasses::Id _id)
+	{
+		auto const& expr = m_expressionClasses.representative(_id);
+		_out << "  " << dec << _id << ": " << *expr.item;
+		if (expr.sequenceNumber)
+			_out << "@" << dec << expr.sequenceNumber;
+		_out << "(";
+		for (ExpressionClasses::Id arg: expr.arguments)
+			_out << dec << arg << ",";
+		_out << ")" << endl;
+	};
+
+	_out << "Optimizer analysis:" << endl;
+	_out << "Final stack height: " << dec << m_stackHeight << endl;
+	_out << "Equivalence classes: " << endl;
+	for (ExpressionClasses::Id eqClass = 0; eqClass < m_expressionClasses.size(); ++eqClass)
+		streamExpressionClass(_out, eqClass);
+
+	_out << "Initial stack: " << endl;
+	for (auto const& it: _initialStack)
+	{
+		_out << "  " << dec << it.first << ": ";
+		streamExpressionClass(_out, it.second);
+	}
+	_out << "Target stack: " << endl;
+	for (auto const& it: _targetStack)
+	{
+		_out << "  " << dec << it.first << ": ";
+		streamExpressionClass(_out, it.second);
+	}
+
+	return _out;
+}
+
+void CommonSubexpressionEliminator::feedItem(AssemblyItem const& _item, bool _copyItem)
+{
+	if (_item.type() != Operation)
+	{
+		assertThrow(_item.deposit() == 1, InvalidDeposit, "");
+		setStackElement(++m_stackHeight, m_expressionClasses.find(_item, {}, _copyItem));
+	}
+	else
+	{
+		Instruction instruction = _item.instruction();
+		InstructionInfo info = instructionInfo(instruction);
+		if (SemanticInformation::isDupInstruction(_item))
+			setStackElement(
+				m_stackHeight + 1,
+				stackElement(m_stackHeight - int(instruction) + int(Instruction::DUP1))
+			);
+		else if (SemanticInformation::isSwapInstruction(_item))
+			swapStackElements(
+				m_stackHeight,
+				m_stackHeight - 1 - int(instruction) + int(Instruction::SWAP1)
+			);
+		else if (instruction != Instruction::POP)
+		{
+			vector<ExpressionClasses::Id> arguments(info.args);
+			for (int i = 0; i < info.args; ++i)
+				arguments[i] = stackElement(m_stackHeight - i);
+			if (_item.instruction() == Instruction::SSTORE)
+				storeInStorage(arguments[0], arguments[1]);
+			else if (_item.instruction() == Instruction::SLOAD)
+				setStackElement(m_stackHeight + _item.deposit(), loadFromStorage(arguments[0]));
+			else if (_item.instruction() == Instruction::MSTORE)
+				storeInMemory(arguments[0], arguments[1]);
+			else if (_item.instruction() == Instruction::MLOAD)
+				setStackElement(m_stackHeight + _item.deposit(), loadFromMemory(arguments[0]));
+			else
+				setStackElement(m_stackHeight + _item.deposit(), m_expressionClasses.find(_item, arguments, _copyItem));
+		}
+		m_stackHeight += _item.deposit();
+	}
+}
+
+void CommonSubexpressionEliminator::optimizeBreakingItem()
+{
+	if (!m_breakingItem || *m_breakingItem != AssemblyItem(Instruction::JUMPI))
+		return;
+
+	using Id = ExpressionClasses::Id;
+	static AssemblyItem s_jump = Instruction::JUMP;
+
+	Id condition = stackElement(m_stackHeight - 1);
+	Id zero = m_expressionClasses.find(u256(0));
+	if (m_expressionClasses.knownToBeDifferent(condition, zero))
+	{
+		feedItem(Instruction::SWAP1, true);
+		feedItem(Instruction::POP, true);
+		m_breakingItem = &s_jump;
+		return;
+	}
+	Id negatedCondition = m_expressionClasses.find(Instruction::ISZERO, {condition});
+	if (m_expressionClasses.knownToBeDifferent(negatedCondition, zero))
+	{
+		feedItem(Instruction::POP, true);
+		feedItem(Instruction::POP, true);
+		m_breakingItem = nullptr;
+	}
+}
+
+void CommonSubexpressionEliminator::setStackElement(int _stackHeight, ExpressionClasses::Id _class)
+{
+	m_stackElements[_stackHeight] = _class;
+}
+
+void CommonSubexpressionEliminator::swapStackElements(int _stackHeightA, int _stackHeightB)
+{
+	assertThrow(_stackHeightA != _stackHeightB, OptimizerException, "Swap on same stack elements.");
+	// ensure they are created
+	stackElement(_stackHeightA);
+	stackElement(_stackHeightB);
+
+	swap(m_stackElements[_stackHeightA], m_stackElements[_stackHeightB]);
+}
+
+ExpressionClasses::Id CommonSubexpressionEliminator::stackElement(int _stackHeight)
+{
+	if (m_stackElements.count(_stackHeight))
+		return m_stackElements.at(_stackHeight);
+	// Stack element not found (not assigned yet), create new equivalence class.
+	return m_stackElements[_stackHeight] = initialStackElement(_stackHeight);
+}
+
+ExpressionClasses::Id CommonSubexpressionEliminator::initialStackElement(int _stackHeight)
+{
+	assertThrow(_stackHeight <= 0, OptimizerException, "Initial stack element of positive height requested.");
+	assertThrow(_stackHeight > -16, StackTooDeepException, "");
+	// This is a special assembly item that refers to elements pre-existing on the initial stack.
+	return m_expressionClasses.find(AssemblyItem(dupInstruction(1 - _stackHeight)));
+}
+
+void CommonSubexpressionEliminator::storeInStorage(ExpressionClasses::Id _slot, ExpressionClasses::Id _value)
+{
+	if (m_storageContent.count(_slot) && m_storageContent[_slot] == _value)
+		// do not execute the storage if we know that the value is already there
+		return;
+	m_sequenceNumber++;
+	decltype(m_storageContent) storageContents;
+	// copy over values at points where we know that they are different from _slot
+	for (auto const& storageItem: m_storageContent)
+		if (m_expressionClasses.knownToBeDifferent(storageItem.first, _slot))
+			storageContents.insert(storageItem);
+	m_storageContent = move(storageContents);
+	ExpressionClasses::Id id = m_expressionClasses.find(Instruction::SSTORE, {_slot, _value}, true, m_sequenceNumber);
+	m_storeOperations.push_back(StoreOperation(StoreOperation::Storage, _slot, m_sequenceNumber, id));
+	m_storageContent[_slot] = _value;
+	// increment a second time so that we get unique sequence numbers for writes
+	m_sequenceNumber++;
+}
+
+ExpressionClasses::Id CommonSubexpressionEliminator::loadFromStorage(ExpressionClasses::Id _slot)
+{
+	if (m_storageContent.count(_slot))
+		return m_storageContent.at(_slot);
+	else
+		return m_storageContent[_slot] = m_expressionClasses.find(Instruction::SLOAD, {_slot}, true, m_sequenceNumber);
+}
+
+void CommonSubexpressionEliminator::storeInMemory(ExpressionClasses::Id _slot, ExpressionClasses::Id _value)
+{
+	if (m_memoryContent.count(_slot) && m_memoryContent[_slot] == _value)
+		// do not execute the store if we know that the value is already there
+		return;
+	m_sequenceNumber++;
+	decltype(m_memoryContent) memoryContents;
+	// copy over values at points where we know that they are different from _slot by at least 32
+	for (auto const& memoryItem: m_memoryContent)
+		if (m_expressionClasses.knownToBeDifferentBy32(memoryItem.first, _slot))
+			memoryContents.insert(memoryItem);
+	m_memoryContent = move(memoryContents);
+	ExpressionClasses::Id id = m_expressionClasses.find(Instruction::MSTORE, {_slot, _value}, true, m_sequenceNumber);
+	m_storeOperations.push_back(StoreOperation(StoreOperation::Memory, _slot, m_sequenceNumber, id));
+	m_memoryContent[_slot] = _value;
+	// increment a second time so that we get unique sequence numbers for writes
+	m_sequenceNumber++;
+}
+
+ExpressionClasses::Id CommonSubexpressionEliminator::loadFromMemory(ExpressionClasses::Id _slot)
+{
+	if (m_memoryContent.count(_slot))
+		return m_memoryContent.at(_slot);
+	else
+		return m_memoryContent[_slot] = m_expressionClasses.find(Instruction::MLOAD, {_slot}, true, m_sequenceNumber);
+}
+
+CSECodeGenerator::CSECodeGenerator(
+	ExpressionClasses& _expressionClasses,
+	vector<CSECodeGenerator::StoreOperation> const& _storeOperations
+):
+	m_expressionClasses(_expressionClasses)
+{
+	for (auto const& store: _storeOperations)
+		m_storeOperations[make_pair(store.target, store.slot)].push_back(store);
+}
+
+AssemblyItems CSECodeGenerator::generateCode(
+	map<int, ExpressionClasses::Id> const& _initialStack,
+	map<int, ExpressionClasses::Id> const& _targetStackContents
+)
+{
+	m_stack = _initialStack;
+	for (auto const& item: m_stack)
+		if (!m_classPositions.count(item.second))
+			m_classPositions[item.second] = item.first;
+
+	// @todo: provide information about the positions of copies of class elements
+
+	// generate the dependency graph starting from final storage and memory writes and target stack contents
+	for (auto const& p: m_storeOperations)
+		addDependencies(p.second.back().expression);
+	for (auto const& targetItem: _targetStackContents)
+	{
+		m_finalClasses.insert(targetItem.second);
+		addDependencies(targetItem.second);
+	}
+
+	// store all needed sequenced expressions
+	set<pair<unsigned, ExpressionClasses::Id>> sequencedExpressions;
+	for (auto const& p: m_neededBy)
+		for (auto id: {p.first, p.second})
+			if (unsigned seqNr = m_expressionClasses.representative(id).sequenceNumber)
+				sequencedExpressions.insert(make_pair(seqNr, id));
+
+	// Perform all operations on storage and memory in order, if they are needed.
+	for (auto const& seqAndId: sequencedExpressions)
+		if (!m_classPositions.count(seqAndId.second))
+			generateClassElement(seqAndId.second, true);
+
+	// generate the target stack elements
+	for (auto const& targetItem: _targetStackContents)
+	{
+		int position = generateClassElement(targetItem.second);
+		assertThrow(position != c_invalidPosition, OptimizerException, "");
+		if (position == targetItem.first)
+			continue;
+		if (position < targetItem.first)
+			// it is already at its target, we need another copy
+			appendDup(position);
+		else
+			appendOrRemoveSwap(position);
+		appendOrRemoveSwap(targetItem.first);
+	}
+
+	// remove surplus elements
+	while (removeStackTopIfPossible())
+	{
+		// no-op
+	}
+
+	// check validity
+	int finalHeight = 0;
+	if (!_targetStackContents.empty())
+		// have target stack, so its height should be the final height
+		finalHeight = (--_targetStackContents.end())->first;
+	else if (!_initialStack.empty())
+		// no target stack, only erase the initial stack
+		finalHeight = _initialStack.begin()->first - 1;
+	else
+		// neither initial no target stack, no change in height
+		finalHeight = 0;
+	assertThrow(finalHeight == m_stackHeight, OptimizerException, "Incorrect final stack height.");
+	return m_generatedItems;
+}
+
+void CSECodeGenerator::addDependencies(ExpressionClasses::Id _c)
+{
+	if (m_neededBy.count(_c))
+		return; // we already computed the dependencies for _c
+	ExpressionClasses::Expression expr = m_expressionClasses.representative(_c);
+	for (ExpressionClasses::Id argument: expr.arguments)
+	{
+		addDependencies(argument);
+		m_neededBy.insert(make_pair(argument, _c));
+	}
+	if (expr.item->type() == Operation && (
+		expr.item->instruction() == Instruction::SLOAD ||
+		expr.item->instruction() == Instruction::MLOAD
+	))
+	{
+		// this loads an unknown value from storage or memory and thus, in addition to its
+		// arguments, depends on all store operations to addresses where we do not know that
+		// they are different that occur before this load
+		StoreOperation::Target target = expr.item->instruction() == Instruction::SLOAD ?
+			StoreOperation::Storage : StoreOperation::Memory;
+		ExpressionClasses::Id slotToLoadFrom = expr.arguments.at(0);
+		for (auto const& p: m_storeOperations)
+		{
+			if (p.first.first != target)
+				continue;
+			ExpressionClasses::Id slot = p.first.second;
+			StoreOperations const& storeOps = p.second;
+			if (storeOps.front().sequenceNumber > expr.sequenceNumber)
+				continue;
+			if (
+				(target == StoreOperation::Memory && m_expressionClasses.knownToBeDifferentBy32(slot, slotToLoadFrom)) ||
+				(target == StoreOperation::Storage && m_expressionClasses.knownToBeDifferent(slot, slotToLoadFrom))
+			)
+				continue;
+			// note that store and load never have the same sequence number
+			ExpressionClasses::Id latestStore = storeOps.front().expression;
+			for (auto it = ++storeOps.begin(); it != storeOps.end(); ++it)
+				if (it->sequenceNumber < expr.sequenceNumber)
+					latestStore = it->expression;
+			addDependencies(latestStore);
+			m_neededBy.insert(make_pair(latestStore, _c));
+		}
+	}
+}
+
+int CSECodeGenerator::generateClassElement(ExpressionClasses::Id _c, bool _allowSequenced)
+{
+	// do some cleanup
+	removeStackTopIfPossible();
+
+	if (m_classPositions.count(_c))
+	{
+		assertThrow(
+			m_classPositions[_c] != c_invalidPosition,
+			OptimizerException,
+			"Element already removed but still needed."
+		);
+		return m_classPositions[_c];
+	}
+	ExpressionClasses::Expression const& expr = m_expressionClasses.representative(_c);
+	assertThrow(
+		_allowSequenced || expr.sequenceNumber == 0,
+		OptimizerException,
+		"Sequence constrained operation requested out of sequence."
+	);
+	ExpressionClasses::Ids const& arguments = expr.arguments;
+	for (ExpressionClasses::Id arg: boost::adaptors::reverse(arguments))
+		generateClassElement(arg);
+
+	// The arguments are somewhere on the stack now, so it remains to move them at the correct place.
+	// This is quite difficult as sometimes, the values also have to removed in this process
+	// (if canBeRemoved() returns true) and the two arguments can be equal. For now, this is
+	// implemented for every single case for combinations of up to two arguments manually.
+	if (arguments.size() == 1)
+	{
+		if (canBeRemoved(arguments[0], _c))
+			appendOrRemoveSwap(classElementPosition(arguments[0]));
+		else
+			appendDup(classElementPosition(arguments[0]));
+	}
+	else if (arguments.size() == 2)
+	{
+		if (canBeRemoved(arguments[1], _c))
+		{
+			appendOrRemoveSwap(classElementPosition(arguments[1]));
+			if (arguments[0] == arguments[1])
+				appendDup(m_stackHeight);
+			else if (canBeRemoved(arguments[0], _c))
+			{
+				appendOrRemoveSwap(m_stackHeight - 1);
+				appendOrRemoveSwap(classElementPosition(arguments[0]));
+			}
+			else
+				appendDup(classElementPosition(arguments[0]));
+		}
+		else
+		{
+			if (arguments[0] == arguments[1])
+			{
+				appendDup(classElementPosition(arguments[0]));
+				appendDup(m_stackHeight);
+			}
+			else if (canBeRemoved(arguments[0], _c))
+			{
+				appendOrRemoveSwap(classElementPosition(arguments[0]));
+				appendDup(classElementPosition(arguments[1]));
+				appendOrRemoveSwap(m_stackHeight - 1);
+			}
+			else
+			{
+				appendDup(classElementPosition(arguments[1]));
+				appendDup(classElementPosition(arguments[0]));
+			}
+		}
+	}
+	else
+		assertThrow(
+			arguments.size() <= 2,
+			OptimizerException,
+			"Opcodes with more than two arguments not implemented yet."
+		);
+	for (size_t i = 0; i < arguments.size(); ++i)
+		assertThrow(m_stack[m_stackHeight - i] == arguments[i], OptimizerException, "Expected arguments not present." );
+
+	while (SemanticInformation::isCommutativeOperation(*expr.item) &&
+			!m_generatedItems.empty() &&
+			m_generatedItems.back() == AssemblyItem(Instruction::SWAP1))
+		// this will not append a swap but remove the one that is already there
+		appendOrRemoveSwap(m_stackHeight - 1);
+	for (auto arg: arguments)
+		if (canBeRemoved(arg, _c))
+			m_classPositions[arg] = c_invalidPosition;
+	for (size_t i = 0; i < arguments.size(); ++i)
+		m_stack.erase(m_stackHeight - i);
+	appendItem(*expr.item);
+	if (expr.item->type() != Operation || instructionInfo(expr.item->instruction()).ret == 1)
+	{
+		m_stack[m_stackHeight] = _c;
+		return m_classPositions[_c] = m_stackHeight;
+	}
+	else
+	{
+		assertThrow(
+			instructionInfo(expr.item->instruction()).ret == 0,
+			OptimizerException,
+			"Invalid number of return values."
+		);
+		return m_classPositions[_c] = c_invalidPosition;
+	}
+}
+
+int CSECodeGenerator::classElementPosition(ExpressionClasses::Id _id) const
+{
+	assertThrow(
+		m_classPositions.count(_id) && m_classPositions.at(_id) != c_invalidPosition,
+		OptimizerException,
+		"Element requested but is not present."
+	);
+	return m_classPositions.at(_id);
+}
+
+bool CSECodeGenerator::canBeRemoved(ExpressionClasses::Id _element, ExpressionClasses::Id _result)
+{
+	// Returns false if _element is finally needed or is needed by a class that has not been
+	// computed yet. Note that m_classPositions also includes classes that were deleted in the meantime.
+	if (m_finalClasses.count(_element))
+		return false;
+
+	auto range = m_neededBy.equal_range(_element);
+	for (auto it = range.first; it != range.second; ++it)
+		if (it->second != _result && !m_classPositions.count(it->second))
+			return false;
+	return true;
+}
+
+bool CSECodeGenerator::removeStackTopIfPossible()
+{
+	if (m_stack.empty())
+		return false;
+	assertThrow(m_stack.count(m_stackHeight), OptimizerException, "");
+	ExpressionClasses::Id top = m_stack[m_stackHeight];
+	if (!canBeRemoved(top))
+		return false;
+	m_generatedItems.push_back(AssemblyItem(Instruction::POP));
+	m_stack.erase(m_stackHeight);
+	m_stackHeight--;
+	return true;
+}
+
+void CSECodeGenerator::appendDup(int _fromPosition)
+{
+	assertThrow(_fromPosition != c_invalidPosition, OptimizerException, "");
+	int instructionNum = 1 + m_stackHeight - _fromPosition;
+	assertThrow(instructionNum <= 16, StackTooDeepException, "Stack too deep.");
+	assertThrow(1 <= instructionNum, OptimizerException, "Invalid stack access.");
+	appendItem(AssemblyItem(dupInstruction(instructionNum)));
+	m_stack[m_stackHeight] = m_stack[_fromPosition];
+}
+
+void CSECodeGenerator::appendOrRemoveSwap(int _fromPosition)
+{
+	assertThrow(_fromPosition != c_invalidPosition, OptimizerException, "");
+	if (_fromPosition == m_stackHeight)
+		return;
+	int instructionNum = m_stackHeight - _fromPosition;
+	assertThrow(instructionNum <= 16, StackTooDeepException, "Stack too deep.");
+	assertThrow(1 <= instructionNum, OptimizerException, "Invalid stack access.");
+	appendItem(AssemblyItem(swapInstruction(instructionNum)));
+	// The value of a class can be present in multiple locations on the stack. We only update the
+	// "canonical" one that is tracked by m_classPositions
+	if (m_classPositions[m_stack[m_stackHeight]] == m_stackHeight)
+		m_classPositions[m_stack[m_stackHeight]] = _fromPosition;
+	if (m_classPositions[m_stack[_fromPosition]] == _fromPosition)
+		m_classPositions[m_stack[_fromPosition]] = m_stackHeight;
+	swap(m_stack[m_stackHeight], m_stack[_fromPosition]);
+	if (m_generatedItems.size() >= 2 &&
+		SemanticInformation::isSwapInstruction(m_generatedItems.back()) &&
+		*(m_generatedItems.end() - 2) == m_generatedItems.back())
+	{
+		m_generatedItems.pop_back();
+		m_generatedItems.pop_back();
+	}
+}
+
+void CSECodeGenerator::appendItem(AssemblyItem const& _item)
+{
+	m_generatedItems.push_back(_item);
+	m_stackHeight += _item.deposit();
+}

libevmcore/CommonSubexpressionEliminator.h

@@ -0,0 +1,228 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @file CommonSubexpressionEliminator.h
+ * @author Christian <c@ethdev.com>
+ * @date 2015
+ * Optimizer step for common subexpression elimination and stack reorganisation.
+ */
+
+#pragma once
+
+#include <vector>
+#include <map>
+#include <set>
+#include <tuple>
+#include <ostream>
+#include <libdevcore/CommonIO.h>
+#include <libdevcore/Exceptions.h>
+#include <libevmcore/ExpressionClasses.h>
+#include <libevmcore/SemanticInformation.h>
+
+namespace dev
+{
+namespace eth
+{
+
+class AssemblyItem;
+using AssemblyItems = std::vector<AssemblyItem>;
+
+/**
+ * Optimizer step that performs common subexpression elimination and stack reorganisation,
+ * i.e. it tries to infer equality among expressions and compute the values of two expressions
+ * known to be equal only once.
+ *
+ * The general workings are that for each assembly item that is fed into the eliminator, an
+ * equivalence class is derived from the operation and the equivalence class of its arguments.
+ * DUPi, SWAPi and some arithmetic instructions are used to infer equivalences while these
+ * classes are determined.
+ *
+ * When the list of optimized items is requested, they are generated in a bottom-up fashion,
+ * adding code for equivalence classes that were not yet computed.
+ */
+class CommonSubexpressionEliminator
+{
+public:
+	struct StoreOperation
+	{
+		enum Target { Memory, Storage };
+		StoreOperation(
+			Target _target,
+			ExpressionClasses::Id _slot,
+			unsigned _sequenceNumber,
+			ExpressionClasses::Id _expression
+		): target(_target), slot(_slot), sequenceNumber(_sequenceNumber), expression(_expression) {}
+		Target target;
+		ExpressionClasses::Id slot;
+		unsigned sequenceNumber;
+		ExpressionClasses::Id expression;
+	};
+
+	/// Feeds AssemblyItems into the eliminator and @returns the iterator pointing at the first
+	/// item that must be fed into a new instance of the eliminator.
+	template <class _AssemblyItemIterator>
+	_AssemblyItemIterator feedItems(_AssemblyItemIterator _iterator, _AssemblyItemIterator _end);
+
+	/// @returns the resulting items after optimization.
+	AssemblyItems getOptimizedItems();
+
+	/// Streams debugging information to @a _out.
+	std::ostream& stream(
+		std::ostream& _out,
+		std::map<int, ExpressionClasses::Id> _initialStack = std::map<int, ExpressionClasses::Id>(),
+		std::map<int, ExpressionClasses::Id> _targetStack = std::map<int, ExpressionClasses::Id>()
+	) const;
+
+private:
+	/// Feeds the item into the system for analysis.
+	void feedItem(AssemblyItem const& _item, bool _copyItem = false);
+
+	/// Tries to optimize the item that breaks the basic block at the end.
+	void optimizeBreakingItem();
+
+	/// Simplifies the given item using
+	/// Assigns a new equivalence class to the next sequence number of the given stack element.
+	void setStackElement(int _stackHeight, ExpressionClasses::Id _class);
+	/// Swaps the given stack elements in their next sequence number.
+	void swapStackElements(int _stackHeightA, int _stackHeightB);
+	/// Retrieves the current equivalence class fo the given stack element (or generates a new
+	/// one if it does not exist yet).
+	ExpressionClasses::Id stackElement(int _stackHeight);
+	/// @returns the equivalence class id of the special initial stack element at the given height
+	/// (must not be positive).
+	ExpressionClasses::Id initialStackElement(int _stackHeight);
+
+	/// Increments the sequence number, deletes all storage information that might be overwritten
+	/// and stores the new value at the given slot.
+	void storeInStorage(ExpressionClasses::Id _slot, ExpressionClasses::Id _value);
+	/// Retrieves the current value at the given slot in storage or creates a new special sload class.
+	ExpressionClasses::Id loadFromStorage(ExpressionClasses::Id _slot);
+	/// Increments the sequence number, deletes all memory information that might be overwritten
+	/// and stores the new value at the given slot.
+	void storeInMemory(ExpressionClasses::Id _slot, ExpressionClasses::Id _value);
+	/// Retrieves the current value at the given slot in memory or creates a new special mload class.
+	ExpressionClasses::Id loadFromMemory(ExpressionClasses::Id _slot);
+
+
+	/// Current stack height, can be negative.
+	int m_stackHeight = 0;
+	/// Current stack layout, mapping stack height -> equivalence class
+	std::map<int, ExpressionClasses::Id> m_stackElements;
+	/// Current sequence number, this is incremented with each modification to storage or memory.
+	unsigned m_sequenceNumber = 1;
+	/// Knowledge about storage content.
+	std::map<ExpressionClasses::Id, ExpressionClasses::Id> m_storageContent;
+	/// Knowledge about memory content. Keys are memory addresses, note that the values overlap
+	/// and are not contained here if they are not completely known.
+	std::map<ExpressionClasses::Id, ExpressionClasses::Id> m_memoryContent;
+	/// Keeps information about which storage or memory slots were written to at which sequence
+	/// number with what instruction.
+	std::vector<StoreOperation> m_storeOperations;
+	/// Structure containing the classes of equivalent expressions.
+	ExpressionClasses m_expressionClasses;
+
+	/// The item that breaks the basic block, can be nullptr.
+	/// It is usually appended to the block but can be optimized in some cases.
+	AssemblyItem const* m_breakingItem = nullptr;
+};
+
+/**
+ * Unit that generates code from current stack layout, target stack layout and information about
+ * the equivalence classes.
+ */
+class CSECodeGenerator
+{
+public:
+	using StoreOperation = CommonSubexpressionEliminator::StoreOperation;
+	using StoreOperations = std::vector<StoreOperation>;
+
+	/// Initializes the code generator with the given classes and store operations.
+	/// The store operations have to be sorted by sequence number in ascending order.
+	CSECodeGenerator(ExpressionClasses& _expressionClasses, StoreOperations const& _storeOperations);
+
+	/// @returns the assembly items generated from the given requirements
+	/// @param _initialStack current contents of the stack (up to stack height of zero)
+	/// @param _targetStackContents final contents of the stack, by stack height relative to initial
+	/// @note should only be called once on each object.
+	AssemblyItems generateCode(
+		std::map<int, ExpressionClasses::Id> const& _initialStack,
+		std::map<int, ExpressionClasses::Id> const& _targetStackContents
+	);
+
+private:
+	/// Recursively discovers all dependencies to @a m_requests.
+	void addDependencies(ExpressionClasses::Id _c);
+
+	/// Produce code that generates the given element if it is not yet present.
+	/// @returns the stack position of the element or c_invalidPosition if it does not actually
+	/// generate a value on the stack.
+	/// @param _allowSequenced indicates that sequence-constrained operations are allowed
+	int generateClassElement(ExpressionClasses::Id _c, bool _allowSequenced = false);
+	/// @returns the position of the representative of the given id on the stack.
+	/// @note throws an exception if it is not on the stack.
+	int classElementPosition(ExpressionClasses::Id _id) const;
+
+	/// @returns true if @a _element can be removed - in general or, if given, while computing @a _result.
+	bool canBeRemoved(ExpressionClasses::Id _element, ExpressionClasses::Id _result = ExpressionClasses::Id(-1));
+
+	/// Appends code to remove the topmost stack element if it can be removed.
+	bool removeStackTopIfPossible();
+
+	/// Appends a dup instruction to m_generatedItems to retrieve the element at the given stack position.
+	void appendDup(int _fromPosition);
+	/// Appends a swap instruction to m_generatedItems to retrieve the element at the given stack position.
+	/// @note this might also remove the last item if it exactly the same swap instruction.
+	void appendOrRemoveSwap(int _fromPosition);
+	/// Appends the given assembly item.
+	void appendItem(AssemblyItem const& _item);
+
+	static const int c_invalidPosition = -0x7fffffff;
+
+	AssemblyItems m_generatedItems;
+	/// Current height of the stack relative to the start.
+	int m_stackHeight = 0;
+	/// If (b, a) is in m_requests then b is needed to compute a.
+	std::multimap<ExpressionClasses::Id, ExpressionClasses::Id> m_neededBy;
+	/// Current content of the stack.
+	std::map<int, ExpressionClasses::Id> m_stack;
+	/// Current positions of equivalence classes, equal to c_invalidPosition if already deleted.
+	std::map<ExpressionClasses::Id, int> m_classPositions;
+
+	/// The actual eqivalence class items and how to compute them.
+	ExpressionClasses& m_expressionClasses;
+	/// Keeps information about which storage or memory slots were written to by which operations.
+	/// The operations are sorted ascendingly by sequence number.
+	std::map<std::pair<StoreOperation::Target, ExpressionClasses::Id>, StoreOperations> m_storeOperations;
+	/// The set of equivalence classes that should be present on the stack at the end.
+	std::set<ExpressionClasses::Id> m_finalClasses;
+};
+
+template <class _AssemblyItemIterator>
+_AssemblyItemIterator CommonSubexpressionEliminator::feedItems(
+	_AssemblyItemIterator _iterator,
+	_AssemblyItemIterator _end
+)
+{
+	for (; _iterator != _end && !SemanticInformation::breaksCSEAnalysisBlock(*_iterator); ++_iterator)
+		feedItem(*_iterator);
+	if (_iterator != _end)
+		m_breakingItem = &(*_iterator++);
+	return _iterator;
+}
+
+}
+}

libevmcore/Exceptions.h

@@ -31,6 +31,8 @@ namespace eth
 struct AssemblyException: virtual Exception {};
 struct InvalidDeposit: virtual AssemblyException {};
 struct InvalidOpcode: virtual AssemblyException {};
+struct OptimizerException: virtual AssemblyException {};
+struct StackTooDeepException: virtual OptimizerException {};
 
 }
 }

libevmcore/ExpressionClasses.cpp

@@ -0,0 +1,419 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @file ExpressionClasses.cpp
+ * @author Christian <c@ethdev.com>
+ * @date 2015
+ * Container for equivalence classes of expressions for use in common subexpression elimination.
+ */
+
+#include <libevmcore/ExpressionClasses.h>
+#include <utility>
+#include <tuple>
+#include <functional>
+#include <boost/range/adaptor/reversed.hpp>
+#include <boost/noncopyable.hpp>
+#include <libevmcore/Assembly.h>
+#include <libevmcore/CommonSubexpressionEliminator.h>
+
+using namespace std;
+using namespace dev;
+using namespace dev::eth;
+
+
+bool ExpressionClasses::Expression::operator<(ExpressionClasses::Expression const& _other) const
+{
+	auto type = item->type();
+	auto otherType = _other.item->type();
+	return std::tie(type, item->data(), arguments, sequenceNumber) <
+		std::tie(otherType, _other.item->data(), _other.arguments, _other.sequenceNumber);
+}
+
+ExpressionClasses::Id ExpressionClasses::find(
+	AssemblyItem const& _item,
+	Ids const& _arguments,
+	bool _copyItem,
+	unsigned _sequenceNumber
+)
+{
+	Expression exp;
+	exp.id = Id(-1);
+	exp.item = &_item;
+	exp.arguments = _arguments;
+	exp.sequenceNumber = _sequenceNumber;
+
+	if (SemanticInformation::isCommutativeOperation(_item))
+		sort(exp.arguments.begin(), exp.arguments.end());
+
+	auto it = m_expressions.find(exp);
+	if (it != m_expressions.end())
+		return it->id;
+
+	if (_copyItem)
+	{
+		m_spareAssemblyItem.push_back(make_shared<AssemblyItem>(_item));
+		exp.item = m_spareAssemblyItem.back().get();
+	}
+
+	ExpressionClasses::Id id = tryToSimplify(exp);
+	if (id < m_representatives.size())
+		exp.id = id;
+	else
+	{
+		exp.id = m_representatives.size();
+		m_representatives.push_back(exp);
+	}
+	m_expressions.insert(exp);
+	return exp.id;
+}
+
+bool ExpressionClasses::knownToBeDifferent(ExpressionClasses::Id _a, ExpressionClasses::Id _b)
+{
+	// Try to simplify "_a - _b" and return true iff the value is a non-zero constant.
+	map<unsigned, Expression const*> matchGroups;
+	Pattern constant(Push);
+	constant.setMatchGroup(1, matchGroups);
+	Id difference = find(Instruction::SUB, {_a, _b});
+	return constant.matches(representative(difference), *this) && constant.d() != u256(0);
+}
+
+bool ExpressionClasses::knownToBeDifferentBy32(ExpressionClasses::Id _a, ExpressionClasses::Id _b)
+{
+	// Try to simplify "_a - _b" and return true iff the value is at least 32 away from zero.
+	map<unsigned, Expression const*> matchGroups;
+	Pattern constant(Push);
+	constant.setMatchGroup(1, matchGroups);
+	Id difference = find(Instruction::SUB, {_a, _b});
+	if (!constant.matches(representative(difference), *this))
+		return false;
+	// forbidden interval is ["-31", 31]
+	return constant.d() + 31 > u256(62);
+}
+
+string ExpressionClasses::fullDAGToString(ExpressionClasses::Id _id) const
+{
+	Expression const& expr = representative(_id);
+	stringstream str;
+	str << dec << expr.id << ":" << *expr.item << "(";
+	for (Id arg: expr.arguments)
+		str << fullDAGToString(arg) << ",";
+	str << ")";
+	return str.str();
+}
+
+class Rules: public boost::noncopyable
+{
+public:
+	Rules();
+	void resetMatchGroups() { m_matchGroups.clear(); }
+	vector<pair<Pattern, function<Pattern()>>> rules() const { return m_rules; }
+
+private:
+	using Expression = ExpressionClasses::Expression;
+	map<unsigned, Expression const*> m_matchGroups;
+	vector<pair<Pattern, function<Pattern()>>> m_rules;
+};
+
+Rules::Rules()
+{
+	// Multiple occurences of one of these inside one rule must match the same equivalence class.
+	// Constants.
+	Pattern A(Push);
+	Pattern B(Push);
+	Pattern C(Push);
+	// Anything.
+	Pattern X;
+	Pattern Y;
+	Pattern Z;
+	A.setMatchGroup(1, m_matchGroups);
+	B.setMatchGroup(2, m_matchGroups);
+	C.setMatchGroup(3, m_matchGroups);
+	X.setMatchGroup(4, m_matchGroups);
+	Y.setMatchGroup(5, m_matchGroups);
+	Z.setMatchGroup(6, m_matchGroups);
+
+	m_rules = vector<pair<Pattern, function<Pattern()>>>{
+		// arithmetics on constants
+		{{Instruction::ADD, {A, B}}, [=]{ return A.d() + B.d(); }},
+		{{Instruction::MUL, {A, B}}, [=]{ return A.d() * B.d(); }},
+		{{Instruction::SUB, {A, B}}, [=]{ return A.d() - B.d(); }},
+		{{Instruction::DIV, {A, B}}, [=]{ return B.d() == 0 ? 0 : A.d() / B.d(); }},
+		{{Instruction::SDIV, {A, B}}, [=]{ return B.d() == 0 ? 0 : s2u(u2s(A.d()) / u2s(B.d())); }},
+		{{Instruction::MOD, {A, B}}, [=]{ return B.d() == 0 ? 0 : A.d() % B.d(); }},
+		{{Instruction::SMOD, {A, B}}, [=]{ return B.d() == 0 ? 0 : s2u(u2s(A.d()) % u2s(B.d())); }},
+		{{Instruction::EXP, {A, B}}, [=]{ return u256(boost::multiprecision::powm(bigint(A.d()), bigint(B.d()), bigint(1) << 256)); }},
+		{{Instruction::NOT, {A}}, [=]{ return ~A.d(); }},
+		{{Instruction::LT, {A, B}}, [=]() { return A.d() < B.d() ? u256(1) : 0; }},
+		{{Instruction::GT, {A, B}}, [=]() -> u256 { return A.d() > B.d() ? 1 : 0; }},
+		{{Instruction::SLT, {A, B}}, [=]() -> u256 { return u2s(A.d()) < u2s(B.d()) ? 1 : 0; }},
+		{{Instruction::SGT, {A, B}}, [=]() -> u256 { return u2s(A.d()) > u2s(B.d()) ? 1 : 0; }},
+		{{Instruction::EQ, {A, B}}, [=]() -> u256 { return A.d() == B.d() ? 1 : 0; }},
+		{{Instruction::ISZERO, {A}}, [=]() -> u256 { return A.d() == 0 ? 1 : 0; }},
+		{{Instruction::AND, {A, B}}, [=]{ return A.d() & B.d(); }},
+		{{Instruction::OR, {A, B}}, [=]{ return A.d() | B.d(); }},
+		{{Instruction::XOR, {A, B}}, [=]{ return A.d() ^ B.d(); }},
+		{{Instruction::BYTE, {A, B}}, [=]{ return A.d() >= 32 ? 0 : (B.d() >> unsigned(8 * (31 - A.d()))) & 0xff; }},
+		{{Instruction::ADDMOD, {A, B, C}}, [=]{ return C.d() == 0 ? 0 : u256((bigint(A.d()) + bigint(B.d())) % C.d()); }},
+		{{Instruction::MULMOD, {A, B, C}}, [=]{ return C.d() == 0 ? 0 : u256((bigint(A.d()) * bigint(B.d())) % C.d()); }},
+		{{Instruction::MULMOD, {A, B, C}}, [=]{ return A.d() * B.d(); }},
+		{{Instruction::SIGNEXTEND, {A, B}}, [=]() -> u256 {
+			if (A.d() >= 31)
+				return B.d();
+			unsigned testBit = unsigned(A.d()) * 8 + 7;
+			u256 mask = (u256(1) << testBit) - 1;
+			return u256(boost::multiprecision::bit_test(B.d(), testBit) ? B.d() | ~mask : B.d() & mask);
+		}},
+
+		// invariants involving known constants
+		{{Instruction::ADD, {X, 0}}, [=]{ return X; }},
+		{{Instruction::MUL, {X, 1}}, [=]{ return X; }},
+		{{Instruction::DIV, {X, 1}}, [=]{ return X; }},
+		{{Instruction::SDIV, {X, 1}}, [=]{ return X; }},
+		{{Instruction::OR, {X, 0}}, [=]{ return X; }},
+		{{Instruction::XOR, {X, 0}}, [=]{ return X; }},
+		{{Instruction::AND, {X, ~u256(0)}}, [=]{ return X; }},
+		{{Instruction::MUL, {X, 0}}, [=]{ return u256(0); }},
+		{{Instruction::DIV, {X, 0}}, [=]{ return u256(0); }},
+		{{Instruction::MOD, {X, 0}}, [=]{ return u256(0); }},
+		{{Instruction::MOD, {0, X}}, [=]{ return u256(0); }},
+		{{Instruction::AND, {X, 0}}, [=]{ return u256(0); }},
+		{{Instruction::OR, {X, ~u256(0)}}, [=]{ return ~u256(0); }},
+		// operations involving an expression and itself
+		{{Instruction::AND, {X, X}}, [=]{ return X; }},
+		{{Instruction::OR, {X, X}}, [=]{ return X; }},
+		{{Instruction::SUB, {X, X}}, [=]{ return u256(0); }},
+		{{Instruction::EQ, {X, X}}, [=]{ return u256(1); }},
+		{{Instruction::LT, {X, X}}, [=]{ return u256(0); }},
+		{{Instruction::SLT, {X, X}}, [=]{ return u256(0); }},
+		{{Instruction::GT, {X, X}}, [=]{ return u256(0); }},
+		{{Instruction::SGT, {X, X}}, [=]{ return u256(0); }},
+		{{Instruction::MOD, {X, X}}, [=]{ return u256(0); }},
+
+		{{Instruction::NOT, {{Instruction::NOT, {X}}}}, [=]{ return X; }},
+	};
+	// Associative operations
+	for (auto const& opFun: vector<pair<Instruction,function<u256(u256 const&,u256 const&)>>>{
+		{Instruction::ADD, plus<u256>()},
+		{Instruction::MUL, multiplies<u256>()},
+		{Instruction::AND, bit_and<u256>()},
+		{Instruction::OR, bit_or<u256>()},
+		{Instruction::XOR, bit_xor<u256>()}
+	})
+	{
+		auto op = opFun.first;
+		auto fun = opFun.second;
+		// Moving constants to the outside, order matters here!
+		// we need actions that return expressions (or patterns?) here, and we need also reversed rules
+		// (X+A)+B -> X+(A+B)
+		m_rules += vector<pair<Pattern, function<Pattern()>>>{{
+			{op, {{op, {X, A}}, B}},
+			[=]() -> Pattern { return {op, {X, fun(A.d(), B.d())}}; }
+		}, {
+		// X+(Y+A) -> (X+Y)+A
+			{op, {{op, {X, A}}, Y}},
+			[=]() -> Pattern { return {op, {{op, {X, Y}}, A}}; }
+		}, {
+		// For now, we still need explicit commutativity for the inner pattern
+			{op, {{op, {A, X}}, B}},
+			[=]() -> Pattern { return {op, {X, fun(A.d(), B.d())}}; }
+		}, {
+			{op, {{op, {A, X}}, Y}},
+			[=]() -> Pattern { return {op, {{op, {X, Y}}, A}}; }
+		}};
+	}
+	// move constants across subtractions
+	m_rules += vector<pair<Pattern, function<Pattern()>>>{
+		{
+			// X - A -> X + (-A)
+			{Instruction::SUB, {X, A}},
+			[=]() -> Pattern { return {Instruction::ADD, {X, 0 - A.d()}}; }
+		}, {
+			// (X + A) - Y -> (X - Y) + A
+			{Instruction::SUB, {{Instruction::ADD, {X, A}}, Y}},
+			[=]() -> Pattern { return {Instruction::ADD, {{Instruction::SUB, {X, Y}}, A}}; }
+		}, {
+			// (A + X) - Y -> (X - Y) + A
+			{Instruction::SUB, {{Instruction::ADD, {A, X}}, Y}},
+			[=]() -> Pattern { return {Instruction::ADD, {{Instruction::SUB, {X, Y}}, A}}; }
+		}, {
+			// X - (Y + A) -> (X - Y) + (-A)
+			{Instruction::SUB, {X, {Instruction::ADD, {Y, A}}}},
+			[=]() -> Pattern { return {Instruction::ADD, {{Instruction::SUB, {X, Y}}, 0 - A.d()}}; }
+		}, {
+			// X - (A + Y) -> (X - Y) + (-A)
+			{Instruction::SUB, {X, {Instruction::ADD, {A, Y}}}},
+			[=]() -> Pattern { return {Instruction::ADD, {{Instruction::SUB, {X, Y}}, 0 - A.d()}}; }
+		}
+	};
+}
+
+ExpressionClasses::Id ExpressionClasses::tryToSimplify(Expression const& _expr, bool _secondRun)
+{
+	static Rules rules;
+
+	if (_expr.item->type() != Operation)
+		return -1;
+
+	for (auto const& rule: rules.rules())
+	{
+		rules.resetMatchGroups();
+		if (rule.first.matches(_expr, *this))
+		{
+			// Debug info
+			//cout << "Simplifying " << *_expr.item << "(";
+			//for (Id arg: _expr.arguments)
+			//	cout << fullDAGToString(arg) << ", ";
+			//cout << ")" << endl;
+			//cout << "with rule " << rule.first.toString() << endl;
+			//ExpressionTemplate t(rule.second());
+			//cout << "to " << rule.second().toString() << endl;
+			return rebuildExpression(ExpressionTemplate(rule.second()));
+		}
+	}
+
+	if (!_secondRun && _expr.arguments.size() == 2 && SemanticInformation::isCommutativeOperation(*_expr.item))
+	{
+		Expression expr = _expr;
+		swap(expr.arguments[0], expr.arguments[1]);
+		return tryToSimplify(expr, true);
+	}
+
+	return -1;
+}
+
+ExpressionClasses::Id ExpressionClasses::rebuildExpression(ExpressionTemplate const& _template)
+{
+	if (_template.hasId)
+		return _template.id;
+
+	Ids arguments;
+	for (ExpressionTemplate const& t: _template.arguments)
+		arguments.push_back(rebuildExpression(t));
+	return find(_template.item, arguments);
+}
+
+
+Pattern::Pattern(Instruction _instruction, std::vector<Pattern> const& _arguments):
+	m_type(Operation),
+	m_requireDataMatch(true),
+	m_data(_instruction),
+	m_arguments(_arguments)
+{
+}
+
+void Pattern::setMatchGroup(unsigned _group, map<unsigned, Expression const*>& _matchGroups)
+{
+	m_matchGroup = _group;
+	m_matchGroups = &_matchGroups;
+}
+
+bool Pattern::matches(Expression const& _expr, ExpressionClasses const& _classes) const
+{
+	if (!matchesBaseItem(*_expr.item))
+		return false;
+	if (m_matchGroup)
+	{
+		if (!m_matchGroups->count(m_matchGroup))
+			(*m_matchGroups)[m_matchGroup] = &_expr;
+		else if ((*m_matchGroups)[m_matchGroup]->id != _expr.id)
+			return false;
+	}
+	assertThrow(m_arguments.size() == 0 || _expr.arguments.size() == m_arguments.size(), OptimizerException, "");
+	for (size_t i = 0; i < m_arguments.size(); ++i)
+		if (!m_arguments[i].matches(_classes.representative(_expr.arguments[i]), _classes))
+			return false;
+	return true;
+}
+
+string Pattern::toString() const
+{
+	stringstream s;
+	switch (m_type)
+	{
+	case Operation:
+		s << instructionInfo(Instruction(unsigned(m_data))).name;
+		break;
+	case Push:
+		s << "PUSH " << hex << m_data;
+		break;
+	case UndefinedItem:
+		s << "ANY";
+		break;
+	default:
+		s << "t=" << dec << m_type << " d=" << hex << m_data;
+		break;
+	}
+	if (!m_requireDataMatch)
+		s << " ~";
+	if (m_matchGroup)
+		s << "[" << dec << m_matchGroup << "]";
+	s << "(";
+	for (Pattern const& p: m_arguments)
+		s << p.toString() << ", ";
+	s << ")";
+	return s.str();
+}
+
+bool Pattern::matchesBaseItem(AssemblyItem const& _item) const
+{
+	if (m_type == UndefinedItem)
+		return true;
+	if (m_type != _item.type())
+		return false;
+	if (m_requireDataMatch && m_data != _item.data())
+		return false;
+	return true;
+}
+
+Pattern::Expression const& Pattern::matchGroupValue() const
+{
+	assertThrow(m_matchGroup > 0, OptimizerException, "");
+	assertThrow(!!m_matchGroups, OptimizerException, "");
+	assertThrow((*m_matchGroups)[m_matchGroup], OptimizerException, "");
+	return *(*m_matchGroups)[m_matchGroup];
+}
+
+
+ExpressionTemplate::ExpressionTemplate(Pattern const& _pattern)
+{
+	if (_pattern.matchGroup())
+	{
+		hasId = true;
+		id = _pattern.id();
+	}
+	else
+	{
+		hasId = false;
+		item = _pattern.toAssemblyItem();
+	}
+	for (auto const& arg: _pattern.arguments())
+		arguments.push_back(ExpressionTemplate(arg));
+}
+
+string ExpressionTemplate::toString() const
+{
+	stringstream s;
+	if (hasId)
+		s << id;
+	else
+		s << item;
+	s << "(";
+	for (auto const& arg: arguments)
+		s << arg.toString();
+	s << ")";
+	return s.str();
+}

libevmcore/ExpressionClasses.h

@@ -0,0 +1,168 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @file ExpressionClasses.h
+ * @author Christian <c@ethdev.com>
+ * @date 2015
+ * Container for equivalence classes of expressions for use in common subexpression elimination.
+ */
+
+#pragma once
+
+#include <vector>
+#include <map>
+#include <memory>
+#include <libdevcore/Common.h>
+#include <libevmcore/AssemblyItem.h>
+
+namespace dev
+{
+namespace eth
+{
+
+class Pattern;
+struct ExpressionTemplate;
+
+/**
+ * Collection of classes of equivalent expressions that can also determine the class of an expression.
+ * Identifiers are contiguously assigned to new classes starting from zero.
+ */
+class ExpressionClasses
+{
+public:
+	using Id = unsigned;
+	using Ids = std::vector<Id>;
+
+	struct Expression
+	{
+		Id id;
+		AssemblyItem const* item;
+		Ids arguments;
+		unsigned sequenceNumber; ///< Storage modification sequence, only used for SLOAD/SSTORE instructions.
+		/// Behaves as if this was a tuple of (item->type(), item->data(), arguments, sequenceNumber).
+		bool operator<(Expression const& _other) const;
+	};
+
+	/// Retrieves the id of the expression equivalence class resulting from the given item applied to the
+	/// given classes, might also create a new one.
+	/// @param _copyItem if true, copies the assembly item to an internal storage instead of just
+	/// keeping a pointer.
+	/// The @a _sequenceNumber indicates the current storage or memory access sequence.
+	Id find(
+		AssemblyItem const& _item,
+		Ids const& _arguments = {},
+		bool _copyItem = true,
+		unsigned _sequenceNumber = 0
+	);
+	/// @returns the canonical representative of an expression class.
+	Expression const& representative(Id _id) const { return m_representatives.at(_id); }
+	/// @returns the number of classes.
+	Id size() const { return m_representatives.size(); }
+
+	/// @returns true if the values of the given classes are known to be different (on every input).
+	/// @note that this function might still return false for some different inputs.
+	bool knownToBeDifferent(Id _a, Id _b);
+	/// Similar to @a knownToBeDifferent but require that abs(_a - b) >= 32.
+	bool knownToBeDifferentBy32(Id _a, Id _b);
+
+	std::string fullDAGToString(Id _id) const;
+
+private:
+	/// Tries to simplify the given expression.
+	/// @returns its class if it possible or Id(-1) otherwise.
+	/// @param _secondRun is set to true for the second run where arguments of commutative expressions are reversed
+	Id tryToSimplify(Expression const& _expr, bool _secondRun = false);
+
+	/// Rebuilds an expression from a (matched) pattern.
+	Id rebuildExpression(ExpressionTemplate const& _template);
+
+	std::vector<std::pair<Pattern, std::function<Pattern()>>> createRules() const;
+
+	/// Expression equivalence class representatives - we only store one item of an equivalence.
+	std::vector<Expression> m_representatives;
+	/// All expression ever encountered.
+	std::set<Expression> m_expressions;
+	std::vector<std::shared_ptr<AssemblyItem>> m_spareAssemblyItem;
+};
+
+/**
+ * Pattern to match against an expression.
+ * Also stores matched expressions to retrieve them later, for constructing new expressions using
+ * ExpressionTemplate.
+ */
+class Pattern
+{
+public:
+	using Expression = ExpressionClasses::Expression;
+	using Id = ExpressionClasses::Id;
+
+	// Matches a specific constant value.
+	Pattern(unsigned _value): Pattern(u256(_value)) {}
+	// Matches a specific constant value.
+	Pattern(u256 const& _value): m_type(Push), m_requireDataMatch(true), m_data(_value) {}
+	// Matches a specific assembly item type or anything if not given.
+	Pattern(AssemblyItemType _type = UndefinedItem): m_type(_type) {}
+	// Matches a given instruction with given arguments
+	Pattern(Instruction _instruction, std::vector<Pattern> const& _arguments = {});
+	/// Sets this pattern to be part of the match group with the identifier @a _group.
+	/// Inside one rule, all patterns in the same match group have to match expressions from the
+	/// same expression equivalence class.
+	void setMatchGroup(unsigned _group, std::map<unsigned, Expression const*>& _matchGroups);
+	unsigned matchGroup() const { return m_matchGroup; }
+	bool matches(Expression const& _expr, ExpressionClasses const& _classes) const;
+
+	AssemblyItem toAssemblyItem() const { return AssemblyItem(m_type, m_data); }
+	std::vector<Pattern> arguments() const { return m_arguments; }
+
+	/// @returns the id of the matched expression if this pattern is part of a match group.
+	Id id() const { return matchGroupValue().id; }
+	/// @returns the data of the matched expression if this pattern is part of a match group.
+	u256 d() const { return matchGroupValue().item->data(); }
+
+	std::string toString() const;
+
+private:
+	bool matchesBaseItem(AssemblyItem const& _item) const;
+	Expression const& matchGroupValue() const;
+
+	AssemblyItemType m_type;
+	bool m_requireDataMatch = false;
+	u256 m_data = 0;
+	std::vector<Pattern> m_arguments;
+	unsigned m_matchGroup = 0;
+	std::map<unsigned, Expression const*>* m_matchGroups = nullptr;
+};
+
+/**
+ * Template for a new expression that can be built from matched patterns.
+ */
+struct ExpressionTemplate
+{
+	using Expression = ExpressionClasses::Expression;
+	using Id = ExpressionClasses::Id;
+	explicit ExpressionTemplate(Pattern const& _pattern);
+	std::string toString() const;
+	bool hasId = false;
+	/// Id of the matched expression, if available.
+	Id id = Id(-1);
+	// Otherwise, assembly item.
+	AssemblyItem item = UndefinedItem;
+	std::vector<ExpressionTemplate> arguments;
+};
+
+}
+}

libevmcore/SemanticInformation.cpp

@@ -0,0 +1,107 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @file SemanticInformation.cpp
+ * @author Christian <c@ethdev.com>
+ * @date 2015
+ * Helper to provide semantic information about assembly items.
+ */
+
+#include <libevmcore/SemanticInformation.h>
+#include <libevmcore/AssemblyItem.h>
+
+using namespace std;
+using namespace dev;
+using namespace dev::eth;
+
+bool SemanticInformation::breaksCSEAnalysisBlock(AssemblyItem const& _item)
+{
+	switch (_item.type())
+	{
+	default:
+	case UndefinedItem:
+	case Tag:
+		return true;
+	case Push:
+	case PushString:
+	case PushTag:
+	case PushSub:
+	case PushSubSize:
+	case PushProgramSize:
+	case PushData:
+		return false;
+	case Operation:
+	{
+		if (isSwapInstruction(_item) || isDupInstruction(_item))
+			return false;
+		if (_item.instruction() == Instruction::GAS || _item.instruction() == Instruction::PC)
+			return true; // GAS and PC assume a specific order of opcodes
+		if (_item.instruction() == Instruction::MSIZE)
+			return true; // msize is modified already by memory access, avoid that for now
+		if (_item.instruction() == Instruction::SHA3)
+			return true; //@todo: we have to compare sha3's not based on their memory addresses but on the memory content.
+		InstructionInfo info = instructionInfo(_item.instruction());
+		if (_item.instruction() == Instruction::SSTORE)
+			return false;
+		if (_item.instruction() == Instruction::MSTORE)
+			return false;
+		//@todo: We do not handle the following memory instructions for now:
+		// calldatacopy, codecopy, extcodecopy, mstore8,
+		// msize (note that msize also depends on memory read access)
+
+		// the second requirement will be lifted once it is implemented
+		return info.sideEffects || info.args > 2;
+	}
+	}
+}
+
+bool SemanticInformation::isCommutativeOperation(AssemblyItem const& _item)
+{
+	if (_item.type() != Operation)
+		return false;
+	switch (_item.instruction())
+	{
+	case Instruction::ADD:
+	case Instruction::MUL:
+	case Instruction::EQ:
+	case Instruction::AND:
+	case Instruction::OR:
+	case Instruction::XOR:
+		return true;
+	default:
+		return false;
+	}
+}
+
+bool SemanticInformation::isDupInstruction(AssemblyItem const& _item)
+{
+	if (_item.type() != Operation)
+		return false;
+	return Instruction::DUP1 <= _item.instruction() && _item.instruction() <= Instruction::DUP16;
+}
+
+bool SemanticInformation::isSwapInstruction(AssemblyItem const& _item)
+{
+	if (_item.type() != Operation)
+		return false;
+	return Instruction::SWAP1 <= _item.instruction() && _item.instruction() <= Instruction::SWAP16;
+}
+
+bool SemanticInformation::isJumpInstruction(AssemblyItem const& _item)
+{
+	return _item == AssemblyItem(Instruction::JUMP) || _item == AssemblyItem(Instruction::JUMPI);
+}

libevmcore/SemanticInformation.h

@@ -0,0 +1,50 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @file SemanticInformation.h
+ * @author Christian <c@ethdev.com>
+ * @date 2015
+ * Helper to provide semantic information about assembly items.
+ */
+
+#pragma once
+
+
+namespace dev
+{
+namespace eth
+{
+
+class AssemblyItem;
+
+/**
+ * Helper functions to provide context-independent information about assembly items.
+ */
+struct SemanticInformation
+{
+	/// @returns true if the given items starts a new block for common subexpression analysis.
+	static bool breaksCSEAnalysisBlock(AssemblyItem const& _item);
+	/// @returns true if the item is a two-argument operation whose value does not depend on the
+	/// order of its arguments.
+	static bool isCommutativeOperation(AssemblyItem const& _item);
+	static bool isDupInstruction(AssemblyItem const& _item);
+	static bool isSwapInstruction(AssemblyItem const& _item);
+	static bool isJumpInstruction(AssemblyItem const& _item);
+};
+
+}
+}

libnatspec/NatspecExpressionEvaluator.h

@@ -21,7 +21,6 @@
 
 #pragma once
 
-#include <QtCore/QObject>
 #include <QtCore/QtCore>
 #include <QtQml/QJSEngine>
 

libp2p/Common.cpp

@@ -25,6 +25,17 @@ using namespace dev;
 using namespace dev::p2p;
 
 const unsigned dev::p2p::c_protocolVersion = 3;
+const unsigned dev::p2p::c_defaultIPPort = 30303;
+
+bool p2p::isPublicAddress(std::string const& _addressToCheck)
+{
+	return _addressToCheck.empty() ? false : isPublicAddress(bi::address::from_string(_addressToCheck));
+}
+
+bool p2p::isPublicAddress(bi::address const& _addressToCheck)
+{
+	return !(isPrivateAddress(_addressToCheck) || isLocalHostAddress(_addressToCheck));
+}
 
 // Helper function to determine if an address falls within one of the reserved ranges
 // For V4:
@@ -55,6 +66,11 @@ bool p2p::isPrivateAddress(bi::address const& _addressToCheck)
 	return false;
 }
 
+bool p2p::isPrivateAddress(std::string const& _addressToCheck)
+{
+	return _addressToCheck.empty() ? false : isPrivateAddress(bi::address::from_string(_addressToCheck));
+}
+
 // Helper function to determine if an address is localhost
 bool p2p::isLocalHostAddress(bi::address const& _addressToCheck)
 {
@@ -69,6 +85,11 @@ bool p2p::isLocalHostAddress(bi::address const& _addressToCheck)
 	return find(c_rejectAddresses.begin(), c_rejectAddresses.end(), _addressToCheck) != c_rejectAddresses.end();
 }
 
+bool p2p::isLocalHostAddress(std::string const& _addressToCheck)
+{
+	return _addressToCheck.empty() ? false : isLocalHostAddress(bi::address::from_string(_addressToCheck));
+}
+
 std::string p2p::reasonOf(DisconnectReason _r)
 {
 	switch (_r)
@@ -89,3 +110,9 @@ std::string p2p::reasonOf(DisconnectReason _r)
 	default: return "Unknown reason.";
 	}
 }
+
+void Node::cullEndpoint()
+{
+	if (!isPublicAddress(endpoint.tcp.address()) && isPublicAddress(endpoint.udp.address()))
+		endpoint.tcp.address(endpoint.udp.address());
+}

libp2p/Common.h

@@ -50,11 +50,16 @@ namespace p2p
 
 /// Peer network protocol version.
 extern const unsigned c_protocolVersion;
+extern const unsigned c_defaultIPPort;
 
 using NodeId = h512;
 
 bool isPrivateAddress(bi::address const& _addressToCheck);
+bool isPrivateAddress(std::string const& _addressToCheck);
 bool isLocalHostAddress(bi::address const& _addressToCheck);
+bool isLocalHostAddress(std::string const& _addressToCheck);
+bool isPublicAddress(bi::address const& _addressToCheck);
+bool isPublicAddress(std::string const& _addressToCheck);
 
 class UPnP;
 class Capability;
@@ -62,6 +67,8 @@ class Host;
 class Session;
 
 struct NetworkStartRequired: virtual dev::Exception {};
+struct InvalidPublicIPAddress: virtual dev::Exception {};
+struct InvalidHostIPAddress: virtual dev::Exception {};
 
 struct NetWarn: public LogChannel { static const char* name() { return "!N!"; } static const int verbosity = 0; };
 struct NetNote: public LogChannel { static const char* name() { return "*N*"; } static const int verbosity = 1; };
@@ -169,6 +176,9 @@ struct Node
 	virtual NodeId const& address() const { return id; }
 	virtual Public const& publicKey() const { return id; }
 	
+	/// Adopt UDP address for TCP if TCP isn't public and UDP is. (to be removed when protocol is updated for nat)
+	void cullEndpoint();
+	
 	NodeId id;
 	
 	/// Endpoints by which we expect to reach node.

libp2p/Host.cpp

@@ -66,10 +66,6 @@ Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n, byte
 	m_alias(networkAlias(_restoreNetwork)),
 	m_lastPing(chrono::steady_clock::time_point::min())
 {
-	for (auto address: m_ifAddresses)
-		if (address.is_v4())
-			clog(NetNote) << "IP Address: " << address << " = " << (isPrivateAddress(address) ? "[LOCAL]" : "[PEER]");
-
 	clog(NetNote) << "Id:" << id();
 }
 
@@ -287,67 +283,50 @@ void Host::onNodeTableEvent(NodeId const& _n, NodeTableEventType const& _e)
 	}
 }
 
-void Host::determinePublic(string const& _publicAddress, bool _upnp)
+void Host::determinePublic()
 {
-	m_peerAddresses.clear();
+	// set m_tcpPublic := listenIP (if public) > public > upnp > unspecified address.
 	
-	// no point continuing if there are no interface addresses or valid listen port
+	auto ifAddresses = Network::getInterfaceAddresses();
-	if (!m_ifAddresses.size() || m_listenPort < 1)
+	auto laddr = m_netPrefs.listenIPAddress.empty() ? bi::address() : bi::address::from_string(m_netPrefs.listenIPAddress);
-		return;
+	auto lset = !laddr.is_unspecified();
+	auto paddr = m_netPrefs.publicIPAddress.empty() ? bi::address() : bi::address::from_string(m_netPrefs.publicIPAddress);
+	auto pset = !paddr.is_unspecified();
 	
-	// populate interfaces we'll listen on (eth listens on all interfaces); ignores local
+	bool listenIsPublic = lset && isPublicAddress(laddr);
-	for (auto addr: m_ifAddresses)
+	bool publicIsHost = !lset && pset && ifAddresses.count(paddr);
-		if ((m_netPrefs.localNetworking || !isPrivateAddress(addr)) && !isLocalHostAddress(addr))
-			m_peerAddresses.insert(addr);
-
-	// if user supplied address is a public address then we use it
-	// if user supplied address is private, and localnetworking is enabled, we use it
-	bi::address reqPublicAddr(bi::address(_publicAddress.empty() ? bi::address() : bi::address::from_string(_publicAddress)));
-	bi::tcp::endpoint reqPublic(reqPublicAddr, m_listenPort);
-	bool isprivate = isPrivateAddress(reqPublicAddr);
-	bool ispublic = !isprivate && !isLocalHostAddress(reqPublicAddr);
-	if (!reqPublicAddr.is_unspecified() && (ispublic || (isprivate && m_netPrefs.localNetworking)))
-	{
-		if (!m_peerAddresses.count(reqPublicAddr))
-			m_peerAddresses.insert(reqPublicAddr);
-		m_tcpPublic = reqPublic;
-		return;
-	}
 	
-	// if address wasn't provided, then use first public ipv4 address found
+	bi::tcp::endpoint ep(bi::address(), m_netPrefs.listenPort);
-	for (auto addr: m_peerAddresses)
+	if (m_netPrefs.traverseNAT && listenIsPublic)
-		if (addr.is_v4() && !isPrivateAddress(addr))
 	{
-			m_tcpPublic = bi::tcp::endpoint(*m_peerAddresses.begin(), m_listenPort);
+		clog(NetNote) << "Listen address set to Public address:" << laddr << ". UPnP disabled.";
-			return;
+		ep.address(laddr);
 	}
-
+	else if (m_netPrefs.traverseNAT && publicIsHost)
-	// or find address via upnp
-	if (_upnp)
 	{
-		bi::address upnpifaddr;
+		clog(NetNote) << "Public address set to Host configured address:" << paddr << ". UPnP disabled.";
-		bi::tcp::endpoint upnpep = Network::traverseNAT(m_ifAddresses, m_listenPort, upnpifaddr);
+		ep.address(paddr);
-		if (!upnpep.address().is_unspecified() && !upnpifaddr.is_unspecified())
-		{
-			if (!m_peerAddresses.count(upnpep.address()))
-				m_peerAddresses.insert(upnpep.address());
-			m_tcpPublic = upnpep;
-			return;
-		}
 	}
+	else if (m_netPrefs.traverseNAT)
+	{
+		bi::address natIFAddr;
+		ep = Network::traverseNAT(lset && ifAddresses.count(laddr) ? std::set<bi::address>({laddr}) : ifAddresses, m_netPrefs.listenPort, natIFAddr);
 		
-	// or if no address provided, use private ipv4 address if local networking is enabled
+		if (lset && natIFAddr != laddr)
-	if (reqPublicAddr.is_unspecified())
+			// if listen address is set, Host will use it, even if upnp returns different
-		if (m_netPrefs.localNetworking)
+			clog(NetWarn) << "Listen address" << laddr << "differs from local address" << natIFAddr << "returned by UPnP!";
-			for (auto addr: m_peerAddresses)
+		
-				if (addr.is_v4() && isPrivateAddress(addr))
+		if (pset && ep.address() != paddr)
 		{
-					m_tcpPublic = bi::tcp::endpoint(addr, m_listenPort);
+			// if public address is set, Host will advertise it, even if upnp returns different
-					return;
+			clog(NetWarn) << "Specified public address" << paddr << "differs from external address" << ep.address() << "returned by UPnP!";
+			ep.address(paddr);
 		}
+	}
+	else if (pset)
+		ep.address(paddr);
 
-	// otherwise address is unspecified
+	m_tcpPublic = ep;
-	m_tcpPublic = bi::tcp::endpoint(bi::address(), m_listenPort);
 }
 
 void Host::runAcceptor()
@@ -401,7 +380,7 @@ string Host::pocHost()
 	return "poc-" + strs[1] + ".ethdev.com";
 }
 
-void Host::addNode(NodeId const& _node, std::string const& _addr, unsigned short _tcpPeerPort, unsigned short _udpNodePort)
+void Host::addNode(NodeId const& _node, bi::address const& _addr, unsigned short _udpNodePort, unsigned short _tcpPeerPort)
 {
 	// TODO: p2p clean this up (bring tested acceptor code over from network branch)
 	while (isWorking() && !m_run)
@@ -418,23 +397,58 @@ void Host::addNode(NodeId const& _node, std::string const& _addr, unsigned short
 		_tcpPeerPort = 0;
 	}
 	
-	boost::system::error_code ec;
+	if (m_nodeTable)
-	bi::address addr = bi::address::from_string(_addr, ec);
+		m_nodeTable->addNode(Node(_node, NodeIPEndpoint(bi::udp::endpoint(_addr, _udpNodePort), bi::tcp::endpoint(_addr, _tcpPeerPort))));
-	if (ec)
+}
+
+void Host::requirePeer(NodeId const& _n, bi::address const& _udpAddr, unsigned short _udpPort, bi::address const& _tcpAddr, unsigned short _tcpPort)
 {
-		bi::tcp::resolver *r = new bi::tcp::resolver(m_ioService);
+	auto naddr = _udpAddr;
-		r->async_resolve({_addr, toString(_tcpPeerPort)}, [=](boost::system::error_code const& _ec, bi::tcp::resolver::iterator _epIt)
+	auto paddr = _tcpAddr.is_unspecified() ? naddr : _tcpAddr;
+	auto udp = bi::udp::endpoint(naddr, _udpPort);
+	auto tcp = bi::tcp::endpoint(paddr, _tcpPort ? _tcpPort : _udpPort);
+	Node node(_n, NodeIPEndpoint(udp, tcp));
+	if (_n)
 	{
-			if (!_ec)
+		// add or replace peer
+		shared_ptr<Peer> p;
+		{
+			RecursiveGuard l(x_sessions);
+			if (m_peers.count(_n))
+				p = m_peers[_n];
+			else
 			{
-				bi::tcp::endpoint tcp = *_epIt;
+				p.reset(new Peer());
-				if (m_nodeTable) m_nodeTable->addNode(Node(_node, NodeIPEndpoint(bi::udp::endpoint(tcp.address(), _udpNodePort), tcp)));
+				p->id = _n;
+				p->required = true;
+				m_peers[_n] = p;
+			}
+			p->endpoint.udp = node.endpoint.udp;
+			p->endpoint.tcp = node.endpoint.tcp;
+		}
+		connect(p);
 	}
-			delete r;
+	else if (m_nodeTable)
+	{
+		shared_ptr<boost::asio::deadline_timer> t(new boost::asio::deadline_timer(m_ioService));
+		m_timers.push_back(t);
+		
+		m_nodeTable->addNode(node);
+		t->expires_from_now(boost::posix_time::milliseconds(600));
+		t->async_wait([this, _n](boost::system::error_code const& _ec)
+		{
+			if (!_ec && m_nodeTable)
+				if (auto n = m_nodeTable->node(_n))
+					requirePeer(n.id, n.endpoint.udp.address(), n.endpoint.udp.port(), n.endpoint.tcp.address(), n.endpoint.tcp.port());
 		});
 	}
-	else
+}
-		if (m_nodeTable) m_nodeTable->addNode(Node(_node, NodeIPEndpoint(bi::udp::endpoint(addr, _udpNodePort), bi::tcp::endpoint(addr, _tcpPeerPort))));
+
+void Host::relinquishPeer(NodeId const& _node)
+{
+	Guard l(x_requiredPeers);
+	if (m_requiredPeers.count(_node))
+		m_requiredPeers.erase(_node);
 }
 
 void Host::connect(std::shared_ptr<Peer> const& _p)
@@ -485,6 +499,9 @@ void Host::connect(std::shared_ptr<Peer> const& _p)
 				Guard l(x_connecting);
 				m_connecting.push_back(handshake);
 			}
+			
+			// preempt setting failedAttempts; this value is cleared upon success
+			_p->m_failedAttempts++;
 			handshake->start();
 		}
 		
@@ -541,6 +558,13 @@ void Host::run(boost::system::error_code const&)
 		Guard l(x_connecting);
 		m_connecting.remove_if([](std::weak_ptr<RLPXHandshake> h){ return h.lock(); });
 	}
+	{
+		Guard l(x_timers);
+		m_timers.remove_if([](std::shared_ptr<boost::asio::deadline_timer> t)
+		{
+			return t->expires_from_now().total_milliseconds() > 0;
+		});
+	}
 	
 	for (auto p: m_sessions)
 		if (auto pp = p.second.lock())
@@ -560,7 +584,7 @@ void Host::run(boost::system::error_code const&)
 		{
 			RecursiveGuard l(x_sessions);
 			for (auto p: m_peers)
-				if (p.second->shouldReconnect())
+				if (p.second->shouldReconnect() && !havePeerSession(p.second->id))
 					toConnect.push_back(p.second);
 		}
 		
@@ -592,26 +616,26 @@ void Host::startedWorking()
 		m_run = true;
 	}
 
-	// try to open acceptor (todo: ipv6)
+	// start capability threads (ready for incoming connections)
-	m_listenPort = Network::tcp4Listen(m_tcp4Acceptor, m_netPrefs.listenPort);
-
-	// start capability threads
 	for (auto const& h: m_capabilities)
 		h.second->onStarting();
 	
+	// try to open acceptor (todo: ipv6)
+	m_listenPort = Network::tcp4Listen(m_tcp4Acceptor, m_netPrefs);
+
 	// determine public IP, but only if we're able to listen for connections
 	// todo: GUI when listen is unavailable in UI
 	if (m_listenPort)
 	{
-		determinePublic(m_netPrefs.publicIP, m_netPrefs.upnp);
+		determinePublic();
 
 		if (m_listenPort > 0)
 			runAcceptor();
 	}
 	else
-		clog(NetNote) << "p2p.start.notice id:" << id().abridged() << "Listen port is invalid or unavailable. Node Table using default port (30303).";
+		clog(NetNote) << "p2p.start.notice id:" << id().abridged() << "TCP Listen port is invalid or unavailable.";
 
-	m_nodeTable.reset(new NodeTable(m_ioService, m_alias, bi::address::from_string(listenAddress()), listenPort() > 0 ? listenPort() : 30303));
+	m_nodeTable.reset(new NodeTable(m_ioService, m_alias, bi::address::from_string(listenAddress()), listenPort()));
 	m_nodeTable->setEventHandler(new HostNodeTableHandler(*this));
 	restoreNetwork(&m_restoreNetwork);
 
@@ -654,9 +678,6 @@ void Host::disconnectLatePeers()
 
 bytes Host::saveNetwork() const
 {
-	if (!m_nodeTable)
-		return bytes();
-
 	std::list<Peer> peers;
 	{
 		RecursiveGuard l(x_sessions);
@@ -668,29 +689,24 @@ bytes Host::saveNetwork() const
 
 	RLPStream network;
 	int count = 0;
-	{
-		RecursiveGuard l(x_sessions);
 	for (auto const& p: peers)
 	{
-			// TODO: alpha: Figure out why it ever shares these ports.//p.address.port() >= 30300 && p.address.port() <= 30305 &&
-			// TODO: alpha: if/how to save private addresses
 		// Only save peers which have connected within 2 days, with properly-advertised port and public IP address
-			if (chrono::system_clock::now() - p.m_lastConnected < chrono::seconds(3600 * 48) && p.peerEndpoint().port() > 0 && p.peerEndpoint().port() < /*49152*/32768 && p.id != id() && !isPrivateAddress(p.endpoint.udp.address()) && !isPrivateAddress(p.endpoint.tcp.address()))
+		// todo: e2e ipv6 support
+		bi::tcp::endpoint endpoint(p.peerEndpoint());
+		if (!endpoint.address().is_v4())
+			continue;
+		
+		if (chrono::system_clock::now() - p.m_lastConnected < chrono::seconds(3600 * 48) && endpoint.port() > 0 && endpoint.port() < /*49152*/32768 && p.id != id() && !isPrivateAddress(p.endpoint.udp.address()) && !isPrivateAddress(endpoint.address()))
 		{
 			network.appendList(10);
-				if (p.peerEndpoint().address().is_v4())
+			network << endpoint.port() << p.id << p.required
-					network << p.peerEndpoint().address().to_v4().to_bytes();
-				else
-					network << p.peerEndpoint().address().to_v6().to_bytes();
-				// TODO: alpha: replace 0 with trust-state of node
-				network << p.peerEndpoint().port() << p.id << 0
 				<< chrono::duration_cast<chrono::seconds>(p.m_lastConnected.time_since_epoch()).count()
 				<< chrono::duration_cast<chrono::seconds>(p.m_lastAttempted.time_since_epoch()).count()
 				<< p.m_failedAttempts << (unsigned)p.m_lastDisconnect << p.m_score << p.m_rating;
 			count++;
 		}
 	}
-	}
 
 	if (!!m_nodeTable)
 	{
@@ -707,10 +723,13 @@ bytes Host::saveNetwork() const
 			count++;
 		}
 	}
+	// else: TODO: use previous configuration if available
 
 	RLPStream ret(3);
 	ret << dev::p2p::c_protocolVersion << m_alias.secret();
-	ret.appendList(count).appendRaw(network.out(), count);
+	ret.appendList(count);
+	if (!!count)
+		ret.appendRaw(network.out(), count);
 	return ret.out();
 }
 
@@ -720,6 +739,9 @@ void Host::restoreNetwork(bytesConstRef _b)
 	if (!isStarted())
 		BOOST_THROW_EXCEPTION(NetworkStartRequired());
 
+	if (m_dropPeers)
+		return;
+	
 	RecursiveGuard l(x_sessions);
 	RLP r(_b);
 	if (r.itemCount() > 0 && r[0].isInt() && r[0].toInt<unsigned>() == dev::p2p::c_protocolVersion)
@@ -730,22 +752,14 @@ void Host::restoreNetwork(bytesConstRef _b)
 
 		for (auto i: r[2])
 		{
+			// todo: e2e ipv6 support
+			// bi::tcp::endpoint(bi::address_v6(i[0].toArray<byte, 16>()), i[1].toInt<short>());
+			if (i[0].itemCount() != 4)
+				continue;
 			bi::tcp::endpoint tcp;
 			bi::udp::endpoint udp;
-			if (i[0].itemCount() == 4)
-			{
 			tcp = bi::tcp::endpoint(bi::address_v4(i[0].toArray<byte, 4>()), i[1].toInt<short>());
 			udp = bi::udp::endpoint(bi::address_v4(i[0].toArray<byte, 4>()), i[1].toInt<short>());
-			}
-			else
-			{
-				tcp = bi::tcp::endpoint(bi::address_v6(i[0].toArray<byte, 16>()), i[1].toInt<short>());
-				udp = bi::udp::endpoint(bi::address_v6(i[0].toArray<byte, 16>()), i[1].toInt<short>());
-			}
-			
-			// skip private addresses
-			// todo: to support private addresseses entries must be stored
-			//       and managed externally by host rather than nodetable.
 			if (isPrivateAddress(tcp.address()) || isPrivateAddress(udp.address()))
 				continue;
 			
@@ -756,6 +770,7 @@ void Host::restoreNetwork(bytesConstRef _b)
 			{
 				shared_ptr<Peer> p = make_shared<Peer>();
 				p->id = id;
+				p->required = i[3].toInt<bool>();
 				p->m_lastConnected = chrono::system_clock::time_point(chrono::seconds(i[4].toInt<unsigned>()));
 				p->m_lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[5].toInt<unsigned>()));
 				p->m_failedAttempts = i[6].toInt<unsigned>();
@@ -765,6 +780,9 @@ void Host::restoreNetwork(bytesConstRef _b)
 				p->endpoint.tcp = tcp;
 				p->endpoint.udp = udp;
 				m_peers[p->id] = p;
+				if (p->required)
+					requirePeer(p->id, p->endpoint.udp.address(), p->endpoint.udp.port());
+				else
 					m_nodeTable->addNode(*p.get());
 			}
 		}
@@ -774,7 +792,7 @@ void Host::restoreNetwork(bytesConstRef _b)
 KeyPair Host::networkAlias(bytesConstRef _b)
 {
 	RLP r(_b);
-	if (r.itemCount() == 3 && r[0].isInt() && r[0].toInt<int>() == 1)
+	if (r.itemCount() == 3 && r[0].isInt() && r[0].toInt<unsigned>() == dev::p2p::c_protocolVersion)
 		return move(KeyPair(move(Secret(r[1].toBytes()))));
 	else
 		return move(KeyPair::create());

libp2p/Host.h

@@ -70,9 +70,7 @@ private:
  * @brief The Host class
  * Capabilities should be registered prior to startNetwork, since m_capabilities is not thread-safe.
  *
- * @todo cleanup startPeerSession
  * @todo determinePublic: ipv6, udp
- * @todo handle conflict if addNode/requireNode called and Node already exists w/conflicting tcp or udp port
  * @todo per-session keepalive/ping instead of broadcast; set ping-timeout via median-latency
  */
 class Host: public Worker
@@ -98,14 +96,20 @@ public:
 	static std::string pocHost();
 
 	/// Register a peer-capability; all new peer connections will have this capability.
-	template <class T> std::shared_ptr<T> registerCapability(T* _t) { _t->m_host = this; auto ret = std::shared_ptr<T>(_t); m_capabilities[std::make_pair(T::staticName(), T::staticVersion())] = ret; return ret; }
+	template <class T> std::shared_ptr<T> registerCapability(T* _t) { _t->m_host = this; std::shared_ptr<T> ret(_t); m_capabilities[std::make_pair(T::staticName(), T::staticVersion())] = ret; return ret; }
 
 	bool haveCapability(CapDesc const& _name) const { return m_capabilities.count(_name) != 0; }
 	CapDescs caps() const { CapDescs ret; for (auto const& i: m_capabilities) ret.push_back(i.first); return ret; }
 	template <class T> std::shared_ptr<T> cap() const { try { return std::static_pointer_cast<T>(m_capabilities.at(std::make_pair(T::staticName(), T::staticVersion()))); } catch (...) { return nullptr; } }
 
 	/// Add node as a peer candidate. Node is added if discovery ping is successful and table has capacity.
-	void addNode(NodeId const& _node, std::string const& _addr, unsigned short _tcpPort, unsigned short _udpPort);
+	void addNode(NodeId const& _node, bi::address const& _addr, unsigned short _udpPort, unsigned short _tcpPort);
+	
+	/// Create Peer and attempt keeping peer connected.
+	void requirePeer(NodeId const& _node, bi::address const& _udpAddr, unsigned short _udpPort, bi::address const& _tcpAddr = bi::address(), unsigned short _tcpPort = 0);
+
+	/// Note peer as no longer being required.
+	void relinquishPeer(NodeId const& _node);
 	
 	/// Set ideal number of peers.
 	void setIdealPeerCount(unsigned _n) { m_idealPeerCount = _n; }
@@ -117,10 +121,10 @@ public:
 	size_t peerCount() const;
 
 	/// Get the address we're listening on currently.
-	std::string listenAddress() const { return m_tcpPublic.address().to_string(); }
+	std::string listenAddress() const { return m_netPrefs.listenIPAddress.empty() ? "0.0.0.0" : m_netPrefs.listenIPAddress; }
 
 	/// Get the port we're listening on currently.
-	unsigned short listenPort() const { return m_tcpPublic.port(); }
+	unsigned short listenPort() const { return m_netPrefs.listenPort; }
 
 	/// Serialise the set of known peers.
 	bytes saveNetwork() const;
@@ -128,7 +132,7 @@ public:
 	// TODO: P2P this should be combined with peers into a HostStat object of some kind; coalesce data, as it's only used for status information.
 	Peers getPeers() const { RecursiveGuard l(x_sessions); Peers ret; for (auto const& i: m_peers) ret.push_back(*i.second); return ret; }
 
-	void setNetworkPreferences(NetworkPreferences const& _p) { auto had = isStarted(); if (had) stop(); m_netPrefs = _p; if (had) start(); }
+	void setNetworkPreferences(NetworkPreferences const& _p, bool _dropPeers = false) { m_dropPeers = _dropPeers; auto had = isStarted(); if (had) stop(); m_netPrefs = _p; if (had) start(); }
 
 	/// Start network. @threadsafe
 	void start();
@@ -154,8 +158,8 @@ protected:
 private:
 	bool havePeerSession(NodeId _id) { RecursiveGuard l(x_sessions); return m_sessions.count(_id) ? !!m_sessions[_id].lock() : false; }
 	
-	/// Populate m_peerAddresses with available public addresses.
+	/// Determines and sets m_tcpPublic to publicly advertised address.
-	void determinePublic(std::string const& _publicAddress, bool _upnp);
+	void determinePublic();
 
 	void connect(std::shared_ptr<Peer> const& _p);
 
@@ -192,7 +196,7 @@ private:
 	NetworkPreferences m_netPrefs;										///< Network settings.
 
 	/// Interface addresses (private, public)
-	std::vector<bi::address> m_ifAddresses;								///< Interface addresses.
+	std::set<bi::address> m_ifAddresses;								///< Interface addresses.
 
 	int m_listenPort = -1;												///< What port are we listening on. -1 means binding failed or acceptor hasn't been initialized.
 
@@ -212,6 +216,10 @@ private:
 	/// Shared storage of Peer objects. Peers are created or destroyed on demand by the Host. Active sessions maintain a shared_ptr to a Peer;
 	std::map<NodeId, std::shared_ptr<Peer>> m_peers;
 	
+	/// Peers we try to connect regardless of p2p network.
+	std::set<NodeId> m_requiredPeers;
+	Mutex x_requiredPeers;
+
 	/// The nodes to which we are currently connected. Used by host to service peer requests and keepAlivePeers and for shutdown. (see run())
 	/// Mutable because we flush zombie entries (null-weakptrs) as regular maintenance from a const method.
 	mutable std::map<NodeId, std::weak_ptr<Session>> m_sessions;
@@ -222,12 +230,15 @@ private:
 
 	unsigned m_idealPeerCount = 5;										///< Ideal number of peers to be connected to.
 
-	std::set<bi::address> m_peerAddresses;									///< Public addresses that peers (can) know us by.
-
 	std::map<CapDesc, std::shared_ptr<HostCapabilityFace>> m_capabilities;	///< Each of the capabilities we support.
 	
+	/// Deadline timers used for isolated network events. GC'd by run.
+	std::list<std::shared_ptr<boost::asio::deadline_timer>> m_timers;
+	Mutex x_timers;
+
 	std::chrono::steady_clock::time_point m_lastPing;						///< Time we sent the last ping to all peers.
 	bool m_accepting = false;
+	bool m_dropPeers = false;
 };
 
 }

libp2p/Network.cpp

@@ -27,6 +27,7 @@
 #endif
 
 #include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/split.hpp>
 
 #include <libdevcore/Common.h>
 #include <libdevcore/Assertions.h>
@@ -40,9 +41,9 @@ using namespace std;
 using namespace dev;
 using namespace dev::p2p;
 
-std::vector<bi::address> Network::getInterfaceAddresses()
+std::set<bi::address> Network::getInterfaceAddresses()
 {
-	std::vector<bi::address> addresses;
+	std::set<bi::address> addresses;
 
 #ifdef _WIN32
 	WSAData wsaData;
@@ -72,7 +73,7 @@ std::vector<bi::address> Network::getInterfaceAddresses()
 		char *addrStr = inet_ntoa(addr);
 		bi::address address(bi::address::from_string(addrStr));
 		if (!isLocalHostAddress(address))
-			addresses.push_back(address.to_v4());
+			addresses.insert(address.to_v4());
 	}
 
 	WSACleanup();
@@ -91,7 +92,7 @@ std::vector<bi::address> Network::getInterfaceAddresses()
 			in_addr addr = ((struct sockaddr_in *)ifa->ifa_addr)->sin_addr;
 			boost::asio::ip::address_v4 address(boost::asio::detail::socket_ops::network_to_host_long(addr.s_addr));
 			if (!isLocalHostAddress(address))
-				addresses.push_back(address);
+				addresses.insert(address);
 		}
 		else if (ifa->ifa_addr->sa_family == AF_INET6)
 		{
@@ -101,7 +102,7 @@ std::vector<bi::address> Network::getInterfaceAddresses()
 			memcpy(&bytes[0], addr.s6_addr, 16);
 			boost::asio::ip::address_v6 address(bytes, sockaddr->sin6_scope_id);
 			if (!isLocalHostAddress(address))
-				addresses.push_back(address);
+				addresses.insert(address);
 		}
 	}
 
@@ -113,13 +114,14 @@ std::vector<bi::address> Network::getInterfaceAddresses()
 	return std::move(addresses);
 }
 
-int Network::tcp4Listen(bi::tcp::acceptor& _acceptor, unsigned short _listenPort)
+int Network::tcp4Listen(bi::tcp::acceptor& _acceptor, NetworkPreferences const& _netPrefs)
 {
 	int retport = -1;
+	if (_netPrefs.listenIPAddress.empty())
 		for (unsigned i = 0; i < 2; ++i)
 		{
 			// try to connect w/listenPort, else attempt net-allocated port
-		bi::tcp::endpoint endpoint(bi::tcp::v4(), i ? 0 : _listenPort);
+			bi::tcp::endpoint endpoint(bi::tcp::v4(), i ? 0 : _netPrefs.listenPort);
 			try
 			{
 				_acceptor.open(endpoint.protocol());
@@ -142,10 +144,28 @@ int Network::tcp4Listen(bi::tcp::acceptor& _acceptor, unsigned short _listenPort
 				continue;
 			}
 		}
+	else
+	{
+		bi::tcp::endpoint endpoint(bi::address::from_string(_netPrefs.listenIPAddress), _netPrefs.listenPort);
+		try
+		{
+			_acceptor.open(endpoint.protocol());
+			_acceptor.set_option(ba::socket_base::reuse_address(true));
+			_acceptor.bind(endpoint);
+			_acceptor.listen();
+			retport = _acceptor.local_endpoint().port();
+		}
+		catch (...)
+		{
+			clog(NetWarn) << "Couldn't start accepting connections on host. Failed to accept socket.\n" << boost::current_exception_diagnostic_information();
+		}
+		assert(retport == _netPrefs.listenPort);
+		return retport;
+	}
 	return retport;
 }
 
-bi::tcp::endpoint Network::traverseNAT(std::vector<bi::address> const& _ifAddresses, unsigned short _listenPort, bi::address& o_upnpifaddr)
+bi::tcp::endpoint Network::traverseNAT(std::set<bi::address> const& _ifAddresses, unsigned short _listenPort, bi::address& o_upnpInterfaceAddr)
 {
 	asserts(_listenPort != 0);
 
@@ -157,26 +177,26 @@ bi::tcp::endpoint Network::traverseNAT(std::vector<bi::address> const& _ifAddres
 	// let m_upnp continue as null - we handle it properly.
 	catch (...) {}
 
-	bi::tcp::endpoint upnpep;
+	bi::tcp::endpoint upnpEP;
 	if (upnp && upnp->isValid())
 	{
-		bi::address paddr;
+		bi::address pAddr;
 		int extPort = 0;
 		for (auto const& addr: _ifAddresses)
 			if (addr.is_v4() && isPrivateAddress(addr) && (extPort = upnp->addRedirect(addr.to_string().c_str(), _listenPort)))
 			{
-				paddr = addr;
+				pAddr = addr;
 				break;
 			}
 
-		auto eip = upnp->externalIP();
+		auto eIP = upnp->externalIP();
-		bi::address eipaddr(bi::address::from_string(eip));
+		bi::address eIPAddr(bi::address::from_string(eIP));
-		if (extPort && eip != string("0.0.0.0") && !isPrivateAddress(eipaddr))
+		if (extPort && eIP != string("0.0.0.0") && !isPrivateAddress(eIPAddr))
 		{
 			clog(NetNote) << "Punched through NAT and mapped local port" << _listenPort << "onto external port" << extPort << ".";
-			clog(NetNote) << "External addr:" << eip;
+			clog(NetNote) << "External addr:" << eIP;
-			o_upnpifaddr = paddr;
+			o_upnpInterfaceAddr = pAddr;
-			upnpep = bi::tcp::endpoint(eipaddr, (unsigned short)extPort);
+			upnpEP = bi::tcp::endpoint(eIPAddr, (unsigned short)extPort);
 		}
 		else
 			clog(NetWarn) << "Couldn't punch through NAT (or no NAT in place).";
@@ -185,5 +205,33 @@ bi::tcp::endpoint Network::traverseNAT(std::vector<bi::address> const& _ifAddres
 			delete upnp;
 	}
 
-	return upnpep;
+	return upnpEP;
+}
+
+bi::tcp::endpoint Network::resolveHost(string const& _addr)
+{
+	static boost::asio::io_service s_resolverIoService;
+	
+	vector<string> split;
+	boost::split(split, _addr, boost::is_any_of(":"));
+	unsigned port = split.size() > 1 ? stoi(split[1]) : dev::p2p::c_defaultIPPort;
+
+	bi::tcp::endpoint ep(bi::address(), port);
+	boost::system::error_code ec;
+	bi::address address = bi::address::from_string(split[0], ec);
+	if (!ec)
+		ep.address(address);
+	else
+	{
+		boost::system::error_code ec;
+		// resolve returns an iterator (host can resolve to multiple addresses)
+		bi::tcp::resolver r(s_resolverIoService);
+		auto it = r.resolve({split[0], toString(port)}, ec);
+		if (ec)
+			clog(NetWarn) << "Error resolving host address " << _addr << ":" << ec.message();
+		else
+			ep = *it;
+	}
+	return ep;
 }
+

libp2p/Network.h

@@ -39,12 +39,19 @@ namespace p2p
 
 struct NetworkPreferences
 {
-	NetworkPreferences(unsigned short p = 30303, std::string i = std::string(), bool u = true, bool l = false): listenPort(p), publicIP(i), upnp(u), localNetworking(l) {}
+	// Default Network Preferences
+	NetworkPreferences(unsigned short lp = 30303): listenPort(lp) {}
 	
+	// Network Preferences with specific Listen IP
+	NetworkPreferences(std::string const& l, unsigned short lp = 30303, bool u = true): publicIPAddress(), listenIPAddress(l), listenPort(lp), traverseNAT(u) {}
+	
+	// Network Preferences with intended Public IP
+	NetworkPreferences(std::string const& publicIP, std::string const& l = std::string(), unsigned short lp = 30303, bool u = true): publicIPAddress(publicIP), listenIPAddress(l), listenPort(lp), traverseNAT(u) { if (!publicIPAddress.empty() && !isPublicAddress(publicIPAddress)) BOOST_THROW_EXCEPTION(InvalidPublicIPAddress()); }
+
+	std::string publicIPAddress;
+	std::string listenIPAddress;
 	unsigned short listenPort = 30303;
-	std::string publicIP;
+	bool traverseNAT = true;
-	bool upnp = true;
-	bool localNetworking = false;
 };
 
 /**
@@ -55,13 +62,16 @@ class Network
 {
 public:
 	/// @returns public and private interface addresses
-	static std::vector<bi::address> getInterfaceAddresses();
+	static std::set<bi::address> getInterfaceAddresses();
 	
 	/// Try to bind and listen on _listenPort, else attempt net-allocated port.
-	static int tcp4Listen(bi::tcp::acceptor& _acceptor, unsigned short _listenPort);
+	static int tcp4Listen(bi::tcp::acceptor& _acceptor, NetworkPreferences const& _netPrefs);
 
 	/// Return public endpoint of upnp interface. If successful o_upnpifaddr will be a private interface address and endpoint will contain public address and port.
-	static bi::tcp::endpoint traverseNAT(std::vector<bi::address> const& _ifAddresses, unsigned short _listenPort, bi::address& o_upnpifaddr);
+	static bi::tcp::endpoint traverseNAT(std::set<bi::address> const& _ifAddresses, unsigned short _listenPort, bi::address& o_upnpInterfaceAddr);
+	
+	/// Resolve "host:port" string as TCP endpoint. Returns unspecified endpoint on failure.
+	static bi::tcp::endpoint resolveHost(std::string const& _host);
 };
 	
 }

libp2p/NodeTable.cpp

@@ -70,24 +70,22 @@ shared_ptr<NodeEntry> NodeTable::addNode(Public const& _pubk, bi::udp::endpoint
 
 shared_ptr<NodeEntry> NodeTable::addNode(Node const& _node)
 {
-	if (_node.endpoint.udp.address().to_string() == "0.0.0.0" || _node.endpoint.tcp.address().to_string() == "0.0.0.0")
+	// re-enable tcp checks when NAT hosts are handled by discover
+	// we handle when tcp endpoint is 0 below
+	if (_node.endpoint.udp.address().to_string() == "0.0.0.0")
 	{
-		string ptype;
+		clog(NodeTableWarn) << "addNode Failed. Invalid UDP address 0.0.0.0 for" << _node.id.abridged();
-		if (_node.endpoint.udp.address().to_string() != "0.0.0.0")
-			ptype = "TCP";
-		else if (_node.endpoint.tcp.address().to_string() != "0.0.0.0")
-			ptype = "UDP";
-		else
-			ptype = "TCP,UDP";
-		
-		clog(NodeTableWarn) << "addNode Failed. Invalid" << ptype << "address 0.0.0.0 for" << _node.id.abridged();
 		return move(shared_ptr<NodeEntry>());
 	}
 	
-	// ping address if nodeid is empty
+	// ping address to recover nodeid if nodeid is empty
 	if (!_node.id)
 	{
-		m_pubkDiscoverPings[m_node.endpoint.udp.address()] = std::chrono::steady_clock::now();
+		clog(NodeTableConnect) << "Sending public key discovery Ping to" << _node.endpoint.udp << "(Advertising:" << m_node.endpoint.udp << ")";
+		{
+			Guard l(x_pubkDiscoverPings);
+			m_pubkDiscoverPings[_node.endpoint.udp.address()] = std::chrono::steady_clock::now();
+		}
 		PingNode p(_node.endpoint.udp, m_node.endpoint.udp.address().to_string(), m_node.endpoint.udp.port());
 		p.sign(m_secret);
 		m_socketPointer->send(p);
@@ -102,6 +100,8 @@ shared_ptr<NodeEntry> NodeTable::addNode(Node const& _node)
 	
 	shared_ptr<NodeEntry> ret(new NodeEntry(m_node, _node.id, NodeIPEndpoint(_node.endpoint.udp, _node.endpoint.tcp)));
 	m_nodes[_node.id] = ret;
+	ret->cullEndpoint();
+	clog(NodeTableConnect) << "addNode pending for" << m_node.endpoint.udp << m_node.endpoint.tcp;
 	PingNode p(_node.endpoint.udp, m_node.endpoint.udp.address().to_string(), m_node.endpoint.udp.port());
 	p.sign(m_secret);
 	m_socketPointer->send(p);
@@ -313,10 +313,9 @@ void NodeTable::noteActiveNode(Public const& _pubk, bi::udp::endpoint const& _en
 	if (!!node && !node->pending)
 	{
 		clog(NodeTableConnect) << "Noting active node:" << _pubk.abridged() << _endpoint.address().to_string() << ":" << _endpoint.port();
-		
-		// update udp endpoint
 		node->endpoint.udp.address(_endpoint.address());
 		node->endpoint.udp.port(_endpoint.port());
+		node->cullEndpoint();
 		
 		shared_ptr<NodeEntry> contested;
 		{
@@ -398,7 +397,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes
 	
 	bytesConstRef signedBytes(hashedBytes.cropped(Signature::size, hashedBytes.size() - Signature::size));
 
-	// todo: verify sig via known-nodeid and MDC, or, do ping/pong auth if node/endpoint is unknown/untrusted
+	// todo: verify sig via known-nodeid and MDC
 	
 	bytesConstRef sigBytes(_packet.cropped(h256::size, Signature::size));
 	Public nodeid(dev::recover(*(Signature const*)sigBytes.data(), sha3(signedBytes)));
@@ -429,7 +428,6 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes
 							dropNode(n);
 						
 						if (auto n = nodeEntry(it->first.first))
-							if (m_nodeEventHandler && n->pending)
 							n->pending = false;
 						
 						it = m_evictions.erase(it);
@@ -440,12 +438,20 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes
 				{
 					if (auto n = nodeEntry(nodeid))
 						n->pending = false;
-				}
 					else if (m_pubkDiscoverPings.count(_from.address()))
+					{
+						{
+							Guard l(x_pubkDiscoverPings);
 							m_pubkDiscoverPings.erase(_from.address());
+						}
+						if (!haveNode(nodeid))
+							addNode(nodeid, _from, bi::tcp::endpoint(_from.address(), _from.port()));
+					}
 					else
 						return; // unsolicited pong; don't note node as active
+				}
 				
+				clog(NodeTableConnect) << "PONG from " << nodeid.abridged() << _from;
 				break;
 			}
 				
@@ -546,31 +552,42 @@ void NodeTable::doRefreshBuckets(boost::system::error_code const& _ec)
 
 	clog(NodeTableEvent) << "refreshing buckets";
 	bool connected = m_socketPointer->isOpen();
-	bool refreshed = false;
 	if (connected)
 	{
-		Guard l(x_state);
+		NodeId randNodeId;
-		for (auto& d: m_state)
+		crypto::Nonce::get().ref().copyTo(randNodeId.ref().cropped(0, h256::size));
-			if (chrono::steady_clock::now() - d.modified > c_bucketRefresh)
+		crypto::Nonce::get().ref().copyTo(randNodeId.ref().cropped(h256::size, h256::size));
-			{
+		discover(randNodeId);
-				d.touch();
-				while (!d.nodes.empty())
-				{
-					auto n = d.nodes.front();
-					if (auto p = n.lock())
-					{
-						refreshed = true;
-						ping(p.get());
-						break;
-					}
-					d.nodes.pop_front();
-				}
-			}
 	}
 
-	unsigned nextRefresh = connected ? (refreshed ? 200 : c_bucketRefresh.count()*1000) : 10000;
 	auto runcb = [this](boost::system::error_code const& error) { doRefreshBuckets(error); };
-	m_bucketRefreshTimer.expires_from_now(boost::posix_time::milliseconds(nextRefresh));
+	m_bucketRefreshTimer.expires_from_now(boost::posix_time::milliseconds(c_bucketRefresh.count()));
 	m_bucketRefreshTimer.async_wait(runcb);
 }
 
+void PingNode::streamRLP(RLPStream& _s) const
+{
+	_s.appendList(4);
+	_s << dev::p2p::c_protocolVersion << ipAddress << port << expiration;
+}
+
+void PingNode::interpretRLP(bytesConstRef _bytes)
+{
+	RLP r(_bytes);
+	if (r.itemCountStrict() == 3)
+	{
+		version = 2;
+		ipAddress = r[0].toString();
+		port = r[1].toInt<unsigned>(RLP::Strict);
+		expiration = r[2].toInt<unsigned>(RLP::Strict);
+	}
+	else if (r.itemCountStrict() == 4)
+	{
+		version = r[0].toInt<unsigned>(RLP::Strict);
+		ipAddress = r[1].toString();
+		port = r[2].toInt<unsigned>(RLP::Strict);
+		expiration = r[3].toInt<unsigned>(RLP::Strict);
+	}
+	else
+		BOOST_THROW_EXCEPTION(InvalidRLP());
+}

libp2p/NodeTable.h

@@ -195,7 +195,7 @@ private:
 	/* todo: replace boost::posix_time; change constants to upper camelcase */
 	boost::posix_time::milliseconds const c_evictionCheckInterval = boost::posix_time::milliseconds(75);	///< Interval at which eviction timeouts are checked.
 	std::chrono::milliseconds const c_reqTimeout = std::chrono::milliseconds(300);						///< How long to wait for requests (evict, find iterations).
-	std::chrono::seconds const c_bucketRefresh = std::chrono::seconds(3600);							///< Refresh interval prevents bucket from becoming stale. [Kademlia]
+	std::chrono::milliseconds const c_bucketRefresh = std::chrono::milliseconds(112500);							///< Refresh interval prevents bucket from becoming stale. [Kademlia]
 
 	struct NodeBucket
 	{
@@ -288,6 +288,8 @@ inline std::ostream& operator<<(std::ostream& _out, NodeTable const& _nodeTable)
 	return _out;
 }
 
+struct InvalidRLP: public Exception {};
+
 /**
  * Ping packet: Sent to check if node is alive.
  * PingNode is cached and regenerated after expiration - t, where t is timeout.
@@ -316,13 +318,13 @@ struct PingNode: RLPXDatagram<PingNode>
 
 	static const uint8_t type = 1;
 
-	unsigned version = dev::p2p::c_protocolVersion;
+	unsigned version = 0;
 	std::string ipAddress;
 	unsigned port;
 	unsigned expiration;
 
-	void streamRLP(RLPStream& _s) const { _s.appendList(3); _s << ipAddress << port << expiration; }
+	void streamRLP(RLPStream& _s) const override;
-	void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); ipAddress = r[0].toString(); port = r[1].toInt<unsigned>(); expiration = r[2].toInt<unsigned>(); }
+	void interpretRLP(bytesConstRef _bytes) override;
 };
 
 /**

libp2p/Peer.h

@@ -47,7 +47,6 @@ namespace p2p
  * those peers. Modifying these properties via a storage backend alleviates
  * Host of the responsibility. (&& remove save/restoreNetwork)
  * @todo reimplement recording of historical session information on per-transport basis
- * @todo rebuild nodetable when localNetworking is enabled/disabled
  * @todo move attributes into protected
  */
 class Peer: public Node

libp2p/RLPxHandshake.cpp

@@ -131,7 +131,14 @@ void RLPXHandshake::readAck()
 
 void RLPXHandshake::error()
 {
+	m_idleTimer.cancel();
+	
+	auto connected = m_socket->isConnected();
+	if (connected && !m_socket->remoteEndpoint().address().is_unspecified())
 		clog(NetConnect) << "Disconnecting " << m_socket->remoteEndpoint() << " (Handshake Failed)";
+	else
+		clog(NetConnect) << "Handshake Failed (Connection reset by peer)";
+
 	m_socket->close();
 	if (m_io != nullptr)
 		delete m_io;
@@ -140,7 +147,10 @@ void RLPXHandshake::error()
 void RLPXHandshake::transition(boost::system::error_code _ech)
 {
 	if (_ech || m_nextState == Error || m_cancel)
+	{
+		clog(NetConnect) << "Handshake Failed (I/O Error:" << _ech.message() << ")";
 		return error();
+	}
 	
 	auto self(shared_from_this());
 	if (m_nextState == New)
@@ -252,7 +262,7 @@ void RLPXHandshake::transition(boost::system::error_code _ech)
 						}
 
 						clog(NetNote) << (m_originated ? "p2p.connect.egress" : "p2p.connect.ingress") << "hello frame: success. starting session.";
-						RLP rlp(frame.cropped(1));
+						RLP rlp(frame.cropped(1), RLP::ThrowOnFail | RLP::FailIfTooSmall);
 						m_host->startPeerSession(m_remote, rlp, m_io, m_socket->remoteEndpoint());
 					}
 				});
@@ -265,6 +275,7 @@ void RLPXHandshake::transition(boost::system::error_code _ech)
 	{
 		if (!_ec)
 		{
+			if (!m_socket->remoteEndpoint().address().is_unspecified())
 				clog(NetWarn) << "Disconnecting " << m_socket->remoteEndpoint() << " (Handshake Timeout)";
 			cancel();
 		}

libp2p/RLPxHandshake.h

@@ -95,7 +95,7 @@ protected:
 	void transition(boost::system::error_code _ech = boost::system::error_code());
 
 	/// Timeout for remote to respond to transition events. Enforced by m_idleTimer and refreshed by transition().
-	boost::posix_time::milliseconds const c_timeout = boost::posix_time::milliseconds(1000);
+	boost::posix_time::milliseconds const c_timeout = boost::posix_time::milliseconds(1800);
 
 	State m_nextState = New;			///< Current or expected state of transition.
 	bool m_cancel = false;			///< Will be set to true if connection was canceled.

libp2p/Session.cpp

@@ -216,12 +216,8 @@ bool Session::interpret(PacketType _t, RLP const& _r)
 			NodeId id = _r[i][2].toHash<NodeId>();
 			
 			clogS(NetAllDetail) << "Checking: " << ep << "(" << id.abridged() << ")";
-//			clogS(NetAllDetail) << "Checking: " << ep << "(" << id.abridged() << ")" << isPrivateAddress(peerAddress) << this->id().abridged() << isPrivateAddress(endpoint().address()) << m_server->m_peers.count(id) << (m_server->m_peers.count(id) ? isPrivateAddress(m_server->m_peers.at(id)->address.address()) : -1);
 
-			// todo: draft spec: ignore if dist(us,item) - dist(us,them) > 1
+			if (!isPublicAddress(peerAddress))
-			
-			// TODO: isPrivate
-			if (!m_server->m_netPrefs.localNetworking && isPrivateAddress(peerAddress))
 				goto CONTINUE;	// Private address. Ignore.
 
 			if (!id)
@@ -241,7 +237,7 @@ bool Session::interpret(PacketType _t, RLP const& _r)
 
 			// OK passed all our checks. Assume it's good.
 			addRating(1000);
-			m_server->addNode(id, ep.address().to_string(), ep.port(), ep.port());
+			m_server->addNode(id, ep.address(), ep.port(), ep.port());
 			clogS(NetTriviaDetail) << "New peer: " << ep << "(" << id .abridged()<< ")";
 			CONTINUE:;
 			LAMEPEER:;

libp2p/UDP.h

@@ -77,7 +77,7 @@ template <class T>
 struct RLPXDatagram: public RLPXDatagramFace
 {
 	RLPXDatagram(bi::udp::endpoint const& _ep): RLPXDatagramFace(_ep) {}
-	static T fromBytesConstRef(bi::udp::endpoint const& _ep, bytesConstRef _bytes) { T t(_ep); t.interpretRLP(_bytes); return std::move(t); }
+	static T fromBytesConstRef(bi::udp::endpoint const& _ep, bytesConstRef _bytes) { try { T t(_ep); t.interpretRLP(_bytes); return std::move(t); } catch(...) { T t(_ep); return std::move(t); } }
 	uint8_t packetType() { return T::type; }
 };
 
@@ -163,7 +163,14 @@ void UDPSocket<Handler, MaxDatagramSize>::connect()
 		return;
 
 	m_socket.open(bi::udp::v4());
+	try
+	{
 		m_socket.bind(m_endpoint);
+	}
+	catch (...)
+	{
+		m_socket.bind(bi::udp::endpoint(bi::udp::v4(), m_endpoint.port()));
+	}
 
 	// clear write queue so reconnect doesn't send stale messages
 	Guard l(x_sendQ);
@@ -196,7 +203,10 @@ void UDPSocket<Handler, MaxDatagramSize>::doRead()
 	auto self(UDPSocket<Handler, MaxDatagramSize>::shared_from_this());
 	m_socket.async_receive_from(boost::asio::buffer(m_recvData), m_recvEndpoint, [this, self](boost::system::error_code _ec, size_t _len)
 	{
-		if (_ec)
+		// ASIO Safety: It is possible that ASIO will call lambda w/o an error
+		// and after the socket has been disconnected. Checking m_closed
+		// guarantees that m_host will not be called after disconnect().
+		if (_ec || m_closed)
 			return disconnectWithError(_ec);
 
 		assert(_len);
@@ -215,7 +225,7 @@ void UDPSocket<Handler, MaxDatagramSize>::doWrite()
 	auto self(UDPSocket<Handler, MaxDatagramSize>::shared_from_this());
 	m_socket.async_send_to(boost::asio::buffer(datagram.data), datagram.endpoint(), [this, self](boost::system::error_code _ec, std::size_t)
 	{
-		if (_ec)
+		if (_ec || m_closed)
 			return disconnectWithError(_ec);
 		else
 		{

libsolidity/AST.cpp

@@ -21,6 +21,7 @@
  */
 
 #include <algorithm>
+#include <boost/range/adaptor/reversed.hpp>
 #include <libsolidity/Utils.h>
 #include <libsolidity/AST.h>
 #include <libsolidity/ASTVisitor.h>
@@ -52,6 +53,7 @@ void ContractDefinition::checkTypeRequirements()
 		baseSpecifier->checkTypeRequirements();
 
 	checkIllegalOverrides();
+	checkAbstractFunctions();
 
 	FunctionDefinition const* constructor = getConstructor();
 	if (constructor && !constructor->getReturnParameters().empty())
@@ -60,6 +62,7 @@ void ContractDefinition::checkTypeRequirements()
 
 	FunctionDefinition const* fallbackFunction = nullptr;
 	for (ASTPointer<FunctionDefinition> const& function: getDefinedFunctions())
+	{
 		if (function->getName().empty())
 		{
 			if (fallbackFunction)
@@ -71,6 +74,9 @@ void ContractDefinition::checkTypeRequirements()
 					BOOST_THROW_EXCEPTION(fallbackFunction->getParameterList().createTypeError("Fallback function cannot take parameters."));
 			}
 		}
+		if (!function->isFullyImplemented())
+			setFullyImplemented(false);
+	}
 	for (ASTPointer<ModifierDefinition> const& modifier: getFunctionModifiers())
 		modifier->checkTypeRequirements();
 
@@ -88,7 +94,7 @@ void ContractDefinition::checkTypeRequirements()
 		if (hashes.count(hash))
 			BOOST_THROW_EXCEPTION(createTypeError(
 									  std::string("Function signature hash collision for ") +
-									  it.second->getCanonicalSignature()));
+									  it.second->externalSignature()));
 		hashes.insert(hash);
 	}
 }
@@ -124,6 +130,28 @@ FunctionDefinition const* ContractDefinition::getFallbackFunction() const
 	return nullptr;
 }
 
+void ContractDefinition::checkAbstractFunctions()
+{
+	map<string, bool> functions;
+
+	// Search from base to derived
+	for (ContractDefinition const* contract: boost::adaptors::reverse(getLinearizedBaseContracts()))
+		for (ASTPointer<FunctionDefinition> const& function: contract->getDefinedFunctions())
+		{
+			string const& name = function->getName();
+			if (!function->isFullyImplemented() && functions.count(name) && functions[name])
+				BOOST_THROW_EXCEPTION(function->createTypeError("Redeclaring an already implemented function as abstract"));
+			functions[name] = function->isFullyImplemented();
+		}
+
+	for (auto const& it: functions)
+		if (!it.second)
+		{
+			setFullyImplemented(false);
+			break;
+		}
+}
+
 void ContractDefinition::checkIllegalOverrides() const
 {
 	// TODO unify this at a later point. for this we need to put the constness and the access specifier
@@ -192,7 +220,7 @@ vector<pair<FixedHash<4>, FunctionTypePointer>> const& ContractDefinition::getIn
 				if (functionsSeen.count(f->getName()) == 0 && f->isPartOfExternalInterface())
 				{
 					functionsSeen.insert(f->getName());
-					FixedHash<4> hash(dev::sha3(f->getCanonicalSignature()));
+					FixedHash<4> hash(dev::sha3(f->externalSignature()));
 					m_interfaceFunctionList->push_back(make_pair(hash, make_shared<FunctionType>(*f, false)));
 				}
 
@@ -200,8 +228,9 @@ vector<pair<FixedHash<4>, FunctionTypePointer>> const& ContractDefinition::getIn
 				if (functionsSeen.count(v->getName()) == 0 && v->isPartOfExternalInterface())
 				{
 					FunctionType ftype(*v);
+					solAssert(v->getType().get(), "");
 					functionsSeen.insert(v->getName());
-					FixedHash<4> hash(dev::sha3(ftype.getCanonicalSignature(v->getName())));
+					FixedHash<4> hash(dev::sha3(ftype.externalSignature(v->getName())));
 					m_interfaceFunctionList->push_back(make_pair(hash, make_shared<FunctionType>(*v)));
 				}
 		}
@@ -305,19 +334,29 @@ TypePointer FunctionDefinition::getType(ContractDefinition const*) const
 void FunctionDefinition::checkTypeRequirements()
 {
 	for (ASTPointer<VariableDeclaration> const& var: getParameters() + getReturnParameters())
+	{
 		if (!var->getType()->canLiveOutsideStorage())
 			BOOST_THROW_EXCEPTION(var->createTypeError("Type is required to live outside storage."));
+		if (getVisibility() >= Visibility::Public && !(var->getType()->externalType()))
+		{
+			// todo delete when will be implemented arrays as parameter type in internal functions
+			if (getVisibility() == Visibility::Public && var->getType()->getCategory() == Type::Category::Array)
+				BOOST_THROW_EXCEPTION(var->createTypeError("Arrays only implemented for external functions."));
+			else
+				BOOST_THROW_EXCEPTION(var->createTypeError("Internal type is not allowed for public and external functions."));
+		}
+	}
 	for (ASTPointer<ModifierInvocation> const& modifier: m_functionModifiers)
 		modifier->checkTypeRequirements(isConstructor() ?
 			dynamic_cast<ContractDefinition const&>(*getScope()).getBaseContracts() :
 			vector<ASTPointer<InheritanceSpecifier>>());
-
+	if (m_body)
 		m_body->checkTypeRequirements();
 }
 
-string FunctionDefinition::getCanonicalSignature() const
+string FunctionDefinition::externalSignature() const
 {
-	return FunctionType(*this).getCanonicalSignature(getName());
+	return FunctionType(*this).externalSignature(getName());
 }
 
 bool VariableDeclaration::isLValue() const
@@ -342,7 +381,14 @@ void VariableDeclaration::checkTypeRequirements()
 	if (!m_value)
 		return;
 	if (m_type)
+	{
 		m_value->expectType(*m_type);
+		if (m_isStateVariable && !m_type->externalType() && getVisibility() >= Visibility::Public)
+			BOOST_THROW_EXCEPTION(createTypeError("Internal type is not allowed for state variables."));
+
+		if (!FunctionType(*this).externalType())
+			BOOST_THROW_EXCEPTION(createTypeError("Internal type is not allowed for public state variables."));
+	}
 	else
 	{
 		// no type declared and no previous assignment, infer the type
@@ -422,6 +468,8 @@ void EventDefinition::checkTypeRequirements()
 			numIndexed++;
 		if (!var->getType()->canLiveOutsideStorage())
 			BOOST_THROW_EXCEPTION(var->createTypeError("Type is required to live outside storage."));
+		if (!var->getType()->externalType())
+			BOOST_THROW_EXCEPTION(var->createTypeError("Internal type is not allowed as event parameter type."));
 	}
 	if (numIndexed > 3)
 		BOOST_THROW_EXCEPTION(createTypeError("More than 3 indexed arguments for event."));
@@ -639,6 +687,8 @@ void NewExpression::checkTypeRequirements()
 	m_contract = dynamic_cast<ContractDefinition const*>(m_contractName->getReferencedDeclaration());
 	if (!m_contract)
 		BOOST_THROW_EXCEPTION(createTypeError("Identifier is not a contract."));
+	if (!m_contract->isFullyImplemented())
+		BOOST_THROW_EXCEPTION(m_contract->createTypeError("Trying to create an instance of an abstract contract."));
 	shared_ptr<ContractType const> contractType = make_shared<ContractType>(*m_contract);
 	TypePointers const& parameterTypes = contractType->getConstructorType()->getParameterTypes();
 	m_type = make_shared<FunctionType>(parameterTypes, TypePointers{contractType},

libsolidity/AST.h

@@ -196,6 +196,22 @@ protected:
 	ASTPointer<ASTString> m_documentation;
 };
 
+/**
+ * Abstract class that is added to AST nodes that can be marked as not being fully implemented
+ */
+class ImplementationOptional
+{
+public:
+	explicit ImplementationOptional(bool _implemented): m_implemented(_implemented) {}
+
+	/// @return whether this node is fully implemented or not
+	bool isFullyImplemented() const { return m_implemented; }
+	void setFullyImplemented(bool _implemented) {  m_implemented = _implemented; }
+
+protected:
+	bool m_implemented;
+};
+
 /// @}
 
 /**
@@ -203,10 +219,11 @@ protected:
  * document order. It first visits all struct declarations, then all variable declarations and
  * finally all function declarations.
  */
-class ContractDefinition: public Declaration, public Documented
+class ContractDefinition: public Declaration, public Documented, public ImplementationOptional
 {
 public:
-	ContractDefinition(SourceLocation const& _location,
+	ContractDefinition(
+		SourceLocation const& _location,
 		ASTPointer<ASTString> const& _name,
 		ASTPointer<ASTString> const& _documentation,
 		std::vector<ASTPointer<InheritanceSpecifier>> const& _baseContracts,
@@ -215,8 +232,11 @@ public:
 		std::vector<ASTPointer<VariableDeclaration>> const& _stateVariables,
 		std::vector<ASTPointer<FunctionDefinition>> const& _definedFunctions,
 		std::vector<ASTPointer<ModifierDefinition>> const& _functionModifiers,
-					   std::vector<ASTPointer<EventDefinition>> const& _events):
+		std::vector<ASTPointer<EventDefinition>> const& _events
-		Declaration(_location, _name), Documented(_documentation),
+	):
+		Declaration(_location, _name),
+		Documented(_documentation),
+		ImplementationOptional(true),
 		m_baseContracts(_baseContracts),
 		m_definedStructs(_definedStructs),
 		m_definedEnums(_definedEnums),
@@ -263,6 +283,7 @@ public:
 
 private:
 	void checkIllegalOverrides() const;
+	void checkAbstractFunctions();
 
 	std::vector<std::pair<FixedHash<4>, FunctionTypePointer>> const& getInterfaceFunctionList() const;
 
@@ -378,18 +399,23 @@ private:
 	std::vector<ASTPointer<VariableDeclaration>> m_parameters;
 };
 
-class FunctionDefinition: public Declaration, public VariableScope, public Documented
+class FunctionDefinition: public Declaration, public VariableScope, public Documented, public ImplementationOptional
 {
 public:
-	FunctionDefinition(SourceLocation const& _location, ASTPointer<ASTString> const& _name,
+	FunctionDefinition(
+		SourceLocation const& _location,
+		ASTPointer<ASTString> const& _name,
 		Declaration::Visibility _visibility, bool _isConstructor,
 		ASTPointer<ASTString> const& _documentation,
 		ASTPointer<ParameterList> const& _parameters,
 		bool _isDeclaredConst,
 		std::vector<ASTPointer<ModifierInvocation>> const& _modifiers,
 		ASTPointer<ParameterList> const& _returnParameters,
-					ASTPointer<Block> const& _body):
+		ASTPointer<Block> const& _body
-	Declaration(_location, _name, _visibility), Documented(_documentation),
+	):
+		Declaration(_location, _name, _visibility),
+		Documented(_documentation),
+		ImplementationOptional(_body != nullptr),
 		m_isConstructor(_isConstructor),
 		m_parameters(_parameters),
 		m_isDeclaredConst(_isDeclaredConst),
@@ -421,10 +447,10 @@ public:
 	/// Checks that all parameters have allowed types and calls checkTypeRequirements on the body.
 	void checkTypeRequirements();
 
-	/// @returns the canonical signature of the function
+	/// @returns the external signature of the function
 	/// That consists of the name of the function followed by the types of the
 	/// arguments separated by commas all enclosed in parentheses without any spaces.
-	std::string getCanonicalSignature() const;
+	std::string externalSignature() const;
 
 private:
 	bool m_isConstructor;

libsolidity/AST_accept.h

@@ -175,6 +175,7 @@ void FunctionDefinition::accept(ASTVisitor& _visitor)
 		if (m_returnParameters)
 			m_returnParameters->accept(_visitor);
 		listAccept(m_functionModifiers, _visitor);
+		if (m_body)
 			m_body->accept(_visitor);
 	}
 	_visitor.endVisit(*this);
@@ -188,6 +189,7 @@ void FunctionDefinition::accept(ASTConstVisitor& _visitor) const
 		if (m_returnParameters)
 			m_returnParameters->accept(_visitor);
 		listAccept(m_functionModifiers, _visitor);
+		if (m_body)
 			m_body->accept(_visitor);
 	}
 	_visitor.endVisit(*this);

libsolidity/ArrayUtils.cpp

@@ -440,6 +440,110 @@ void ArrayUtils::retrieveLength(ArrayType const& _arrayType) const
 	}
 }
 
+void ArrayUtils::accessIndex(ArrayType const& _arrayType) const
+{
+	ArrayType::Location location = _arrayType.getLocation();
+	eth::Instruction load =
+		location == ArrayType::Location::Storage ? eth::Instruction::SLOAD :
+		location == ArrayType::Location::Memory ? eth::Instruction::MLOAD :
+		eth::Instruction::CALLDATALOAD;
+
+	// retrieve length
+	if (!_arrayType.isDynamicallySized())
+		m_context << _arrayType.getLength();
+	else if (location == ArrayType::Location::CallData)
+		// length is stored on the stack
+		m_context << eth::Instruction::SWAP1;
+	else
+		m_context << eth::Instruction::DUP2 << load;
+	// stack: <base_ref> <index> <length>
+	// check out-of-bounds access
+	m_context << eth::Instruction::DUP2 << eth::Instruction::LT;
+	eth::AssemblyItem legalAccess = m_context.appendConditionalJump();
+	// out-of-bounds access throws exception (just STOP for now)
+	m_context << eth::Instruction::STOP;
+
+	m_context << legalAccess;
+	// stack: <base_ref> <index>
+	if (_arrayType.isByteArray())
+		switch (location)
+		{
+		case ArrayType::Location::Storage:
+			// byte array index storage lvalue on stack (goal):
+			// <ref> <byte_number> = <base_ref + index / 32> <index % 32>
+			m_context << u256(32) << eth::Instruction::SWAP2;
+			CompilerUtils(m_context).computeHashStatic();
+			// stack: 32 index data_ref
+			m_context
+				<< eth::Instruction::DUP3 << eth::Instruction::DUP3
+				<< eth::Instruction::DIV << eth::Instruction::ADD
+			// stack: 32 index (data_ref + index / 32)
+				<< eth::Instruction::SWAP2 << eth::Instruction::SWAP1
+				<< eth::Instruction::MOD;
+			break;
+		case ArrayType::Location::CallData:
+			// no lvalue, just retrieve the value
+			m_context
+				<< eth::Instruction::ADD << eth::Instruction::CALLDATALOAD
+				<< ((u256(0xff) << (256 - 8)))  << eth::Instruction::AND;
+			break;
+		case ArrayType::Location::Memory:
+			solAssert(false, "Memory lvalues not yet implemented.");
+		}
+	else
+	{
+		// stack: <base_ref> <index>
+		m_context << eth::Instruction::SWAP1;
+		if (_arrayType.isDynamicallySized())
+		{
+			if (location == ArrayType::Location::Storage)
+				CompilerUtils(m_context).computeHashStatic();
+			else if (location == ArrayType::Location::Memory)
+				m_context << u256(32) << eth::Instruction::ADD;
+		}
+		// stack: <index> <data_ref>
+		switch (location)
+		{
+		case ArrayType::Location::CallData:
+			m_context
+				<< eth::Instruction::SWAP1 << _arrayType.getBaseType()->getCalldataEncodedSize()
+				<< eth::Instruction::MUL << eth::Instruction::ADD;
+			if (_arrayType.getBaseType()->isValueType())
+				CompilerUtils(m_context).loadFromMemoryDynamic(*_arrayType.getBaseType(), true, true, false);
+			break;
+		case ArrayType::Location::Storage:
+			m_context << eth::Instruction::SWAP1;
+			if (_arrayType.getBaseType()->getStorageBytes() <= 16)
+			{
+				// stack: <data_ref> <index>
+				// goal:
+				// <ref> <byte_number> = <base_ref + index / itemsPerSlot> <(index % itemsPerSlot) * byteSize>
+				unsigned byteSize = _arrayType.getBaseType()->getStorageBytes();
+				solAssert(byteSize != 0, "");
+				unsigned itemsPerSlot = 32 / byteSize;
+				m_context << u256(itemsPerSlot) << eth::Instruction::SWAP2;
+				// stack: itemsPerSlot index data_ref
+				m_context
+					<< eth::Instruction::DUP3 << eth::Instruction::DUP3
+					<< eth::Instruction::DIV << eth::Instruction::ADD
+				// stack: itemsPerSlot index (data_ref + index / itemsPerSlot)
+					<< eth::Instruction::SWAP2 << eth::Instruction::SWAP1
+					<< eth::Instruction::MOD
+					<< u256(byteSize) << eth::Instruction::MUL;
+			}
+			else
+			{
+				if (_arrayType.getBaseType()->getStorageSize() != 1)
+					m_context << _arrayType.getBaseType()->getStorageSize() << eth::Instruction::MUL;
+				m_context << eth::Instruction::ADD << u256(0);
+			}
+			break;
+		case ArrayType::Location::Memory:
+			solAssert(false, "Memory lvalues not yet implemented.");
+		}
+	}
+}
+
 void ArrayUtils::incrementByteOffset(unsigned _byteSize, unsigned _byteOffsetPosition, unsigned _storageOffsetPosition) const
 {
 	solAssert(_byteSize < 32, "");

libsolidity/ArrayUtils.h

@@ -70,6 +70,12 @@ public:
 	/// Stack pre: reference (excludes byte offset for dynamic storage arrays)
 	/// Stack post: reference length
 	void retrieveLength(ArrayType const& _arrayType) const;
+	/// Retrieves the value at a specific index. If the location is storage, only retrieves the
+	/// position.
+	/// Stack pre: reference [length] index
+	/// Stack post for storage: slot byte_offset
+	/// Stack post for calldata: value
+	void accessIndex(ArrayType const& _arrayType) const;
 
 private:
 	/// Adds the given number of bytes to a storage byte offset counter and also increments

libsolidity/CompilerStack.cpp

@@ -138,6 +138,8 @@ void CompilerStack::compile(bool _optimize)
 		for (ASTPointer<ASTNode> const& node: source->ast->getNodes())
 			if (ContractDefinition* contract = dynamic_cast<ContractDefinition*>(node.get()))
 			{
+				if (!contract->isFullyImplemented())
+					continue;
 				shared_ptr<Compiler> compiler = make_shared<Compiler>(_optimize);
 				compiler->compileContract(*contract, contractBytecode);
 				Contract& compiledContract = m_contracts[contract->getName()];

libsolidity/ExpressionCompiler.cpp

@@ -544,7 +544,7 @@ bool ExpressionCompiler::visit(FunctionCall const& _functionCall)
 				}
 			if (!event.isAnonymous())
 			{
-				m_context << u256(h256::Arith(dev::sha3(function.getCanonicalSignature(event.getName()))));
+				m_context << u256(h256::Arith(dev::sha3(function.externalSignature(event.getName()))));
 				++numIndexed;
 			}
 			solAssert(numIndexed <= 4, "Too many indexed arguments.");
@@ -755,113 +755,20 @@ bool ExpressionCompiler::visit(IndexAccess const& _indexAccess)
 	{
 		ArrayType const& arrayType = dynamic_cast<ArrayType const&>(baseType);
 		solAssert(_indexAccess.getIndexExpression(), "Index expression expected.");
-		ArrayType::Location location = arrayType.getLocation();
-		eth::Instruction load =
-			location == ArrayType::Location::Storage ? eth::Instruction::SLOAD :
-			location == ArrayType::Location::Memory ? eth::Instruction::MLOAD :
-			eth::Instruction::CALLDATALOAD;
 
 		// remove storage byte offset
-		if (location == ArrayType::Location::Storage)
+		if (arrayType.getLocation() == ArrayType::Location::Storage)
 			m_context << eth::Instruction::POP;
 
-		// stack layout: <base_ref> [<length>] <index>
 		_indexAccess.getIndexExpression()->accept(*this);
-		// retrieve length
+		// stack layout: <base_ref> [<length>] <index>
-		if (!arrayType.isDynamicallySized())
+		ArrayUtils(m_context).accessIndex(arrayType);
-			m_context << arrayType.getLength();
+		if (arrayType.getLocation() == ArrayType::Location::Storage)
-		else if (location == ArrayType::Location::CallData)
-			// length is stored on the stack
-			m_context << eth::Instruction::SWAP1;
-		else
-			m_context << eth::Instruction::DUP2 << load;
-		// stack: <base_ref> <index> <length>
-		// check out-of-bounds access
-		m_context << eth::Instruction::DUP2 << eth::Instruction::LT;
-		eth::AssemblyItem legalAccess = m_context.appendConditionalJump();
-		// out-of-bounds access throws exception (just STOP for now)
-		m_context << eth::Instruction::STOP;
-
-		m_context << legalAccess;
-		// stack: <base_ref> <index>
-		if (arrayType.isByteArray())
-			switch (location)
 		{
-			case ArrayType::Location::Storage:
+			if (arrayType.isByteArray())
-				// byte array index storage lvalue on stack (goal):
-				// <ref> <byte_number> = <base_ref + index / 32> <index % 32>
-				m_context << u256(32) << eth::Instruction::SWAP2;
-				CompilerUtils(m_context).computeHashStatic();
-				// stack: 32 index data_ref
-				m_context
-					<< eth::Instruction::DUP3 << eth::Instruction::DUP3
-					<< eth::Instruction::DIV << eth::Instruction::ADD
-				// stack: 32 index (data_ref + index / 32)
-					<< eth::Instruction::SWAP2 << eth::Instruction::SWAP1
-					<< eth::Instruction::MOD;
 				setLValue<StorageByteArrayElement>(_indexAccess);
-				break;
-			case ArrayType::Location::CallData:
-				// no lvalue, just retrieve the value
-				m_context
-					<< eth::Instruction::ADD << eth::Instruction::CALLDATALOAD
-					<< ((u256(0xff) << (256 - 8)))  << eth::Instruction::AND;
-				break;
-			case ArrayType::Location::Memory:
-				solAssert(false, "Memory lvalues not yet implemented.");
-			}
-		else
-		{
-			// stack: <base_ref> <index>
-			m_context << eth::Instruction::SWAP1;
-			if (arrayType.isDynamicallySized())
-			{
-				if (location == ArrayType::Location::Storage)
-					CompilerUtils(m_context).computeHashStatic();
-				else if (location == ArrayType::Location::Memory)
-					m_context << u256(32) << eth::Instruction::ADD;
-			}
-			// stack: <index> <data_ref>
-			switch (location)
-			{
-			case ArrayType::Location::CallData:
-				m_context
-					<< eth::Instruction::SWAP1 << arrayType.getBaseType()->getCalldataEncodedSize()
-					<< eth::Instruction::MUL << eth::Instruction::ADD;
-				if (arrayType.getBaseType()->isValueType())
-					CompilerUtils(m_context).loadFromMemoryDynamic(*arrayType.getBaseType(), true, true, false);
-				break;
-			case ArrayType::Location::Storage:
-				m_context << eth::Instruction::SWAP1;
-				if (arrayType.getBaseType()->getStorageBytes() <= 16)
-				{
-					// stack: <data_ref> <index>
-					// goal:
-					// <ref> <byte_number> = <base_ref + index / itemsPerSlot> <(index % itemsPerSlot) * byteSize>
-					unsigned byteSize = arrayType.getBaseType()->getStorageBytes();
-					solAssert(byteSize != 0, "");
-					unsigned itemsPerSlot = 32 / byteSize;
-					m_context << u256(itemsPerSlot) << eth::Instruction::SWAP2;
-					// stack: itemsPerSlot index data_ref
-					m_context
-						<< eth::Instruction::DUP3 << eth::Instruction::DUP3
-						<< eth::Instruction::DIV << eth::Instruction::ADD
-					// stack: itemsPerSlot index (data_ref + index / itemsPerSlot)
-						<< eth::Instruction::SWAP2 << eth::Instruction::SWAP1
-						<< eth::Instruction::MOD
-						<< u256(byteSize) << eth::Instruction::MUL;
-				}
 			else
-				{
-					if (arrayType.getBaseType()->getStorageSize() != 1)
-						m_context << arrayType.getBaseType()->getStorageSize() << eth::Instruction::MUL;
-					m_context << eth::Instruction::ADD << u256(0);
-				}
 				setLValueToStorageItem(_indexAccess);
-				break;
-			case ArrayType::Location::Memory:
-				solAssert(false, "Memory lvalues not yet implemented.");
-			}
 		}
 	}
 	else

libsolidity/ExpressionCompiler.h

@@ -42,7 +42,6 @@ class CompilerContext;
 class Type;
 class IntegerType;
 class ArrayType;
-class StaticStringType;
 
 /**
  * Compiler for expressions, i.e. converts an AST tree whose root is an Expression into a stream

libsolidity/InterfaceHandler.cpp

@@ -129,7 +129,7 @@ std::unique_ptr<std::string> InterfaceHandler::getUserDocumentation(ContractDefi
 			if (!m_notice.empty())
 			{// since @notice is the only user tag if missing function should not appear
 				user["notice"] = Json::Value(m_notice);
-				methods[it.second->getCanonicalSignature()] = user;
+				methods[it.second->externalSignature()] = user;
 			}
 		}
 	}
@@ -175,8 +175,17 @@ std::unique_ptr<std::string> InterfaceHandler::getDevDocumentation(ContractDefin
 				method["author"] = m_author;
 
 			Json::Value params(Json::objectValue);
+			std::vector<std::string> paramNames = it.second->getParameterNames();
 			for (auto const& pair: m_params)
+			{
+				if (find(paramNames.begin(), paramNames.end(), pair.first) == paramNames.end())
+					// LTODO: mismatching parameter name, throw some form of warning and not just an exception
+					BOOST_THROW_EXCEPTION(
+						DocstringParsingError() <<
+						errinfo_comment("documented parameter \"" + pair.first + "\" not found found in the function")
+					);					
 				params[pair.first] = pair.second;
+			}
 
 			if (!m_params.empty())
 				method["params"] = params;
@@ -185,7 +194,7 @@ std::unique_ptr<std::string> InterfaceHandler::getDevDocumentation(ContractDefin
 				method["return"] = m_return;
 
 			if (!method.empty()) // add the function, only if we have any documentation to add
-				methods[it.second->getCanonicalSignature()] = method;
+				methods[it.second->externalSignature()] = method;
 		}
 	}
 	doc["methods"] = methods;

libsolidity/Parser.cpp

@@ -164,8 +164,17 @@ ASTPointer<ContractDefinition> Parser::parseContractDefinition()
 	}
 	nodeFactory.markEndPosition();
 	expectToken(Token::RBrace);
-	return nodeFactory.createNode<ContractDefinition>(name, docString, baseContracts, structs, enums,
+	return nodeFactory.createNode<ContractDefinition>(
-													  stateVariables, functions, modifiers, events);
+		name,
+		docString,
+		baseContracts,
+		structs,
+		enums,
+		stateVariables,
+		functions,
+		modifiers,
+		events
+	);
 }
 
 ASTPointer<InheritanceSpecifier> Parser::parseInheritanceSpecifier()
@@ -247,8 +256,15 @@ ASTPointer<FunctionDefinition> Parser::parseFunctionDefinition(ASTString const*
 	}
 	else
 		returnParameters = createEmptyParameterList();
-	ASTPointer<Block> block = parseBlock();
+	ASTPointer<Block> block = ASTPointer<Block>();
+	nodeFactory.markEndPosition();
+	if (m_scanner->getCurrentToken() != Token::Semicolon)
+	{
+		block = parseBlock();
 		nodeFactory.setEndPositionFromNode(block);
+	}
+	else
+		m_scanner->next(); // just consume the ';'
 	bool const c_isConstructor = (_contractName && *name == *_contractName);
 	return nodeFactory.createNode<FunctionDefinition>(name, visibility, c_isConstructor, docstring,
 													  parameters, isDeclaredConst, modifiers,