Merge branch 'develop' into client_ref

CMakeLists.txt

@@ -397,7 +397,10 @@ if (JSCONSOLE)
 	add_subdirectory(ethconsole)
 endif ()
 
-add_subdirectory(secp256k1)
+if (NOT WIN32)
+	add_subdirectory(secp256k1)
+endif ()
+
 add_subdirectory(libscrypt)
 add_subdirectory(libdevcrypto)
 

alethzero/CMakeLists.txt

@@ -52,7 +52,6 @@ target_link_libraries(${EXECUTABLE} ethereum)
 target_link_libraries(${EXECUTABLE} evm)
 target_link_libraries(${EXECUTABLE} ethcore)
 target_link_libraries(${EXECUTABLE} devcrypto)
-target_link_libraries(${EXECUTABLE} secp256k1)
 target_link_libraries(${EXECUTABLE} lll)
 if (SOLIDITY)
 	target_link_libraries(${EXECUTABLE} solidity)

alethzero/DappLoader.cpp

@@ -129,7 +129,7 @@ void DappLoader::downloadComplete(QNetworkReply* _reply)
 
 		h256 expected = m_uriHashes[requestUrl];
 		bytes package(reinterpret_cast<unsigned char const*>(data.constData()), reinterpret_cast<unsigned char const*>(data.constData() + data.size()));
-		Secp256k1 dec;
+		Secp256k1PP dec;
 		dec.decrypt(expected, package);
 		h256 got = sha3(package);
 		if (got != expected)

eth/main.cpp

@@ -703,12 +703,24 @@ int main(int argc, char** argv)
 
 	string logbuf;
 	std::string additional;
-	g_logPost = [&](std::string const& a, char const*){
-		if (g_silence)
-			logbuf += a + "\n";
-		else
-			cout << "\r           \r" << a << endl << additional << flush;
-	};
+	if (interactive)
+		g_logPost = [&](std::string const& a, char const*){
+			static SpinLock s_lock;
+			SpinGuard l(s_lock);
+			
+			if (g_silence)
+				logbuf += a + "\n";
+			else
+				cout << "\r           \r" << a << endl << additional << flush;
+
+			// helpful to use OutputDebugString on windows
+	#ifdef _WIN32
+			{
+				OutputDebugStringA(a.data());
+				OutputDebugStringA("\n");
+			}
+	#endif
+		};
 
 	auto getPassword = [&](string const& prompt){
 		auto s = g_silence;
@@ -820,7 +832,7 @@ int main(int argc, char** argv)
 
 		while (web3.ethereum()->blockQueue().items().first + web3.ethereum()->blockQueue().items().second > 0)
 		{
-			sleep(1);
+			this_thread::sleep_for(chrono::seconds(1));
 			web3.ethereum()->syncQueue(100000);
 		}
 		double e = chrono::duration_cast<chrono::milliseconds>(chrono::steady_clock::now() - t).count() / 1000.0;

ethminer/MinerAux.h

@@ -265,7 +265,6 @@ public:
 			ProofOfWork::CPUMiner::setNumInstances(m_miningThreads);
 		else if (m_minerType == MinerType::GPU)
 		{
-			ProofOfWork::GPUMiner::setNumInstances(m_miningThreads);
 			if (!ProofOfWork::GPUMiner::configureGPU(
 					m_openclPlatform,
 					m_openclDevice,
@@ -277,6 +276,7 @@ public:
 				cout << "No GPU device with sufficient memory was found. Can't GPU mine. Remove the -G argument" << endl;
 				exit(1);
 			}
+			ProofOfWork::GPUMiner::setNumInstances(m_miningThreads);
 		}
 		if (mode == OperationMode::DAGInit)
 			doInitDAG(m_initDAG);

libdevcore/CommonData.h

@@ -334,4 +334,10 @@ std::vector<T> keysOf(std::unordered_map<T, U> const& _m)
 	return ret;
 }
 
+template <class T, class V>
+bool contains(T const& _t, V const& _v)
+{
+	return std::end(_t) != std::find(std::begin(_t), std::end(_t), _v);
+}
+
 }

libdevcrypto/CMakeLists.txt

@@ -24,6 +24,10 @@ target_link_libraries(${EXECUTABLE} ${DB_LIBRARIES})
 target_link_libraries(${EXECUTABLE} ${CRYPTOPP_LIBRARIES})
 target_link_libraries(${EXECUTABLE} scrypt)
 target_link_libraries(${EXECUTABLE} devcore)
+if (NOT WIN32)
+	add_definitions(-DETH_HAVE_SECP256K1)
+	target_link_libraries(${EXECUTABLE} secp256k1)
+endif ()
 
 install( TARGETS ${EXECUTABLE} RUNTIME DESTINATION bin ARCHIVE DESTINATION lib LIBRARY DESTINATION lib )
 install( FILES ${HEADERS} DESTINATION include/${EXECUTABLE} )

libdevcrypto/Common.cpp

@@ -29,6 +29,9 @@
 #include <libdevcore/Guards.h>
 #include <libdevcore/SHA3.h>
 #include <libdevcore/FileSystem.h>
+#if ETH_HAVE_SECP256K1
+#include <secp256k1/secp256k1.h>
+#endif
 #include "AES.h"
 #include "CryptoPP.h"
 #include "Exceptions.h"
@@ -36,7 +39,17 @@ using namespace std;
 using namespace dev;
 using namespace dev::crypto;
 
-static Secp256k1 s_secp256k1;
+#ifdef ETH_HAVE_SECP256K1
+struct Secp256k1Context
+{
+	Secp256k1Context() { secp256k1_start(); }
+	~Secp256k1Context() { secp256k1_stop(); }
+};
+static Secp256k1Context s_secp256k1;
+void dev::crypto::secp256k1Init() { (void)s_secp256k1; }
+#endif
+
+static Secp256k1PP s_secp256k1pp;
 
 bool dev::SignatureStruct::isValid() const noexcept
 {
@@ -53,34 +66,42 @@ Address dev::ZeroAddress = Address();
 
 Public dev::toPublic(Secret const& _secret)
 {
+#ifdef ETH_HAVE_SECP256K1
+	bytes o(65);
+	int pubkeylen;
+	if (!secp256k1_ecdsa_pubkey_create(o.data(), &pubkeylen, _secret.data(), false))
+		return Public();
+	return FixedHash<64>(o.data()+1, Public::ConstructFromPointer);
+#else
 	Public p;
-	s_secp256k1.toPublic(_secret, p);
+	s_secp256k1pp.toPublic(_secret, p);
 	return p;
+#endif
 }
 
 Address dev::toAddress(Public const& _public)
 {
-	return s_secp256k1.toAddress(_public);
+	return right160(sha3(_public.ref()));
 }
 
 Address dev::toAddress(Secret const& _secret)
 {
 	Public p;
-	s_secp256k1.toPublic(_secret, p);
-	return s_secp256k1.toAddress(p);
+	s_secp256k1pp.toPublic(_secret, p);
+	return toAddress(p);
 }
 
 void dev::encrypt(Public const& _k, bytesConstRef _plain, bytes& o_cipher)
 {
 	bytes io = _plain.toBytes();
-	s_secp256k1.encrypt(_k, io);
+	s_secp256k1pp.encrypt(_k, io);
 	o_cipher = std::move(io);
 }
 
 bool dev::decrypt(Secret const& _k, bytesConstRef _cipher, bytes& o_plaintext)
 {
 	bytes io = _cipher.toBytes();
-	s_secp256k1.decrypt(_k, io);
+	s_secp256k1pp.decrypt(_k, io);
 	if (io.empty())
 		return false;
 	o_plaintext = std::move(io);
@@ -90,14 +111,14 @@ bool dev::decrypt(Secret const& _k, bytesConstRef _cipher, bytes& o_plaintext)
 void dev::encryptECIES(Public const& _k, bytesConstRef _plain, bytes& o_cipher)
 {
 	bytes io = _plain.toBytes();
-	s_secp256k1.encryptECIES(_k, io);
+	s_secp256k1pp.encryptECIES(_k, io);
 	o_cipher = std::move(io);
 }
 
 bool dev::decryptECIES(Secret const& _k, bytesConstRef _cipher, bytes& o_plaintext)
 {
 	bytes io = _cipher.toBytes();
-	if (!s_secp256k1.decryptECIES(_k, io))
+	if (!s_secp256k1pp.decryptECIES(_k, io))
 		return false;
 	o_plaintext = std::move(io);
 	return true;
@@ -163,17 +184,17 @@ bytes dev::decryptAES128CTR(bytesConstRef _k, h128 const& _iv, bytesConstRef _ci
 
 Public dev::recover(Signature const& _sig, h256 const& _message)
 {
-	return s_secp256k1.recover(_sig, _message.ref());
+	return s_secp256k1pp.recover(_sig, _message.ref());
 }
 
 Signature dev::sign(Secret const& _k, h256 const& _hash)
 {
-	return s_secp256k1.sign(_k, _hash);
+	return s_secp256k1pp.sign(_k, _hash);
 }
 
 bool dev::verify(Public const& _p, Signature const& _s, h256 const& _hash)
 {
-	return s_secp256k1.verify(_p, _s, _hash.ref(), true);
+	return s_secp256k1pp.verify(_p, _s, _hash.ref(), true);
 }
 
 bytes dev::pbkdf2(string const& _pass, bytes const& _salt, unsigned _iterations, unsigned _dkLen)
@@ -232,8 +253,8 @@ KeyPair KeyPair::create()
 KeyPair::KeyPair(h256 _sec):
 	m_secret(_sec)
 {
-	if (s_secp256k1.verifySecret(m_secret, m_public))
-		m_address = s_secp256k1.toAddress(m_public);
+	if (s_secp256k1pp.verifySecret(m_secret, m_public))
+		m_address = toAddress(m_public);
 }
 
 KeyPair KeyPair::fromEncryptedSeed(bytesConstRef _seed, std::string const& _password)

libdevcrypto/Common.h

@@ -177,6 +177,8 @@ namespace crypto
 {
 struct InvalidState: public dev::Exception {};
 
+void secp256k1Init();
+
 /// Key derivation
 h256 kdf(Secret const& _priv, h256 const& _hash);
 

libdevcrypto/CryptoPP.cpp

@@ -33,7 +33,7 @@ static_assert(dev::Secret::size == 32, "Secret key must be 32 bytes.");
 static_assert(dev::Public::size == 64, "Public key must be 64 bytes.");
 static_assert(dev::Signature::size == 65, "Signature must be 65 bytes.");
 
-bytes Secp256k1::eciesKDF(Secret _z, bytes _s1, unsigned kdByteLen)
+bytes Secp256k1PP::eciesKDF(Secret _z, bytes _s1, unsigned kdByteLen)
 {
 	// interop w/go ecies implementation
 	
@@ -64,7 +64,7 @@ bytes Secp256k1::eciesKDF(Secret _z, bytes _s1, unsigned kdByteLen)
 	return k;
 }
 
-void Secp256k1::encryptECIES(Public const& _k, bytes& io_cipher)
+void Secp256k1PP::encryptECIES(Public const& _k, bytes& io_cipher)
 {
 	// interop w/go ecies implementation
 	auto r = KeyPair::create();
@@ -98,7 +98,7 @@ void Secp256k1::encryptECIES(Public const& _k, bytes& io_cipher)
 	io_cipher.swap(msg);
 }
 
-bool Secp256k1::decryptECIES(Secret const& _k, bytes& io_text)
+bool Secp256k1PP::decryptECIES(Secret const& _k, bytes& io_text)
 {
 	// interop w/go ecies implementation
 	
@@ -145,7 +145,7 @@ bool Secp256k1::decryptECIES(Secret const& _k, bytes& io_text)
 	return true;
 }
 
-void Secp256k1::encrypt(Public const& _k, bytes& io_cipher)
+void Secp256k1PP::encrypt(Public const& _k, bytes& io_cipher)
 {
 	ECIES<ECP>::Encryptor e;
 	initializeDLScheme(_k, e);
@@ -163,7 +163,7 @@ void Secp256k1::encrypt(Public const& _k, bytes& io_cipher)
 	io_cipher = std::move(ciphertext);
 }
 
-void Secp256k1::decrypt(Secret const& _k, bytes& io_text)
+void Secp256k1PP::decrypt(Secret const& _k, bytes& io_text)
 {
 	CryptoPP::ECIES<CryptoPP::ECP>::Decryptor d;
 	initializeDLScheme(_k, d);
@@ -194,12 +194,12 @@ void Secp256k1::decrypt(Secret const& _k, bytes& io_text)
 	io_text = std::move(plain);
 }
 
-Signature Secp256k1::sign(Secret const& _k, bytesConstRef _message)
+Signature Secp256k1PP::sign(Secret const& _k, bytesConstRef _message)
 {
 	return sign(_k, sha3(_message));
 }
 
-Signature Secp256k1::sign(Secret const& _key, h256 const& _hash)
+Signature Secp256k1PP::sign(Secret const& _key, h256 const& _hash)
 {
 	// assumption made by signing alogrithm
 	asserts(m_q == m_qs);
@@ -240,18 +240,18 @@ Signature Secp256k1::sign(Secret const& _key, h256 const& _hash)
 	return sig;
 }
 
-bool Secp256k1::verify(Signature const& _signature, bytesConstRef _message)
+bool Secp256k1PP::verify(Signature const& _signature, bytesConstRef _message)
 {
 	return !!recover(_signature, _message);
 }
 
-bool Secp256k1::verify(Public const& _p, Signature const& _sig, bytesConstRef _message, bool _hashed)
+bool Secp256k1PP::verify(Public const& _p, Signature const& _sig, bytesConstRef _message, bool _hashed)
 {
 	// todo: verify w/o recovery (if faster)
-	return (bool)_p == _hashed ? (bool)recover(_sig, _message) : (bool)recover(_sig, sha3(_message).ref());
+	return _p == (_hashed ? recover(_sig, _message) : recover(_sig, sha3(_message).ref()));
 }
 
-Public Secp256k1::recover(Signature _signature, bytesConstRef _message)
+Public Secp256k1PP::recover(Signature _signature, bytesConstRef _message)
 {
 	Public recovered;
 	
@@ -293,7 +293,7 @@ Public Secp256k1::recover(Signature _signature, bytesConstRef _message)
 	return recovered;
 }
 
-bool Secp256k1::verifySecret(Secret const& _s, Public& _p)
+bool Secp256k1PP::verifySecret(Secret const& _s, Public& _p)
 {
 	DL_PrivateKey_EC<ECP> k;
 	k.Initialize(m_params, secretToExponent(_s));
@@ -309,7 +309,7 @@ bool Secp256k1::verifySecret(Secret const& _s, Public& _p)
 	return true;
 }
 
-void Secp256k1::agree(Secret const& _s, Public const& _r, h256& o_s)
+void Secp256k1PP::agree(Secret const& _s, Public const& _r, h256& o_s)
 {
 	// TODO: mutex ASN1::secp256k1() singleton
 	// Creating Domain is non-const for m_oid and m_oid is not thread-safe
@@ -320,7 +320,7 @@ void Secp256k1::agree(Secret const& _s, Public const& _r, h256& o_s)
 	d.Agree(o_s.data(), _s.data(), remote);
 }
 
-void Secp256k1::exportPublicKey(CryptoPP::DL_PublicKey_EC<CryptoPP::ECP> const& _k, Public& o_p)
+void Secp256k1PP::exportPublicKey(CryptoPP::DL_PublicKey_EC<CryptoPP::ECP> const& _k, Public& o_p)
 {
 	bytes prefixedKey(_k.GetGroupParameters().GetEncodedElementSize(true));
 	
@@ -333,7 +333,7 @@ void Secp256k1::exportPublicKey(CryptoPP::DL_PublicKey_EC<CryptoPP::ECP> const&
 	memcpy(o_p.data(), &prefixedKey[1], Public::size);
 }
 
-void Secp256k1::exponentToPublic(Integer const& _e, Public& o_p)
+void Secp256k1PP::exponentToPublic(Integer const& _e, Public& o_p)
 {
 	CryptoPP::DL_PublicKey_EC<CryptoPP::ECP> pk;
 	

libdevcrypto/CryptoPP.h

@@ -67,13 +67,11 @@ inline Integer secretToExponent(Secret const& _s) { return std::move(Integer(_s.
  * CryptoPP secp256k1 algorithms.
  * @todo Collect ECIES methods into class.
  */
-class Secp256k1
+class Secp256k1PP
 {	
 public:
-	Secp256k1(): m_oid(ASN1::secp256k1()), m_params(m_oid), m_curve(m_params.GetCurve()), m_q(m_params.GetGroupOrder()), m_qs(m_params.GetSubgroupOrder()) {}
-	
-	Address toAddress(Public const& _p) { return right160(sha3(_p.ref())); }
-	
+	Secp256k1PP(): m_oid(ASN1::secp256k1()), m_params(m_oid), m_curve(m_params.GetCurve()), m_q(m_params.GetGroupOrder()), m_qs(m_params.GetSubgroupOrder()) {}
+
 	void toPublic(Secret const& _s, Public& o_public) { exponentToPublic(Integer(_s.data(), sizeof(_s)), o_public); }
 	
 	/// Encrypts text (replace input). (ECIES w/XOR-SHA1)

libdevcrypto/ECDHE.cpp

@@ -27,7 +27,7 @@ using namespace std;
 using namespace dev;
 using namespace dev::crypto;
 
-static Secp256k1 s_secp256k1;
+static Secp256k1PP s_secp256k1;
 
 void dev::crypto::ecdh::agree(Secret const& _s, Public const& _r, h256& o_s)
 {

libethash-cl/ethash_cl_miner.cpp

@@ -24,13 +24,13 @@
 
 #include <cstdio>
 #include <cstdlib>
+#include <chrono>
 #include <fstream>
 #include <iostream>
 #include <assert.h>
 #include <queue>
 #include <random>
 #include <vector>
-#include <boost/timer.hpp>
 #include <libethash/util.h>
 #include <libethash/ethash.h>
 #include <libethash/internal.h>
@@ -139,6 +139,7 @@ unsigned ethash_cl_miner::getNumDevices(unsigned _platformId)
 }
 
 bool ethash_cl_miner::configureGPU(
+	unsigned _platformId,
 	bool _allowCPU,
 	unsigned _extraGPUMemory,
 	boost::optional<uint64_t> _currentBlock
@@ -149,7 +150,7 @@ bool ethash_cl_miner::configureGPU(
 	// by default let's only consider the DAG of the first epoch
 	uint64_t dagSize = _currentBlock ? ethash_get_datasize(*_currentBlock) : 1073739904U;
 	uint64_t requiredSize =  dagSize + _extraGPUMemory;
-	return searchForAllDevices([&requiredSize](cl::Device const _device) -> bool
+	return searchForAllDevices(_platformId, [&requiredSize](cl::Device const _device) -> bool
 		{
 			cl_ulong result;
 			_device.getInfo(CL_DEVICE_GLOBAL_MEM_SIZE, &result);
@@ -416,6 +417,7 @@ bool ethash_cl_miner::init(
 
 void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook& hook, unsigned _msPerBatch)
 {
+	(void)_msPerBatch;
 	try
 	{
 		struct pending_batch
@@ -454,6 +456,8 @@ void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook
 		uint64_t start_nonce = uniform_int_distribution<uint64_t>()(engine);
 		for (;; start_nonce += m_batchSize)
 		{
+//			chrono::high_resolution_clock::time_point t = chrono::high_resolution_clock::now();
+
 			// supply output buffer to kernel
 			m_searchKernel.setArg(0, m_searchBuffer[buf]);
 			if (m_dagChunksCount == 1)
@@ -462,13 +466,7 @@ void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook
 				m_searchKernel.setArg(6, start_nonce);
 
 			// execute it!
-			boost::timer t;
 			m_queue.enqueueNDRangeKernel(m_searchKernel, cl::NullRange, m_batchSize, m_workgroupSize);
-			unsigned ms = t.elapsed() * 1000;
-			if (ms > _msPerBatch * 1.1)
-				m_batchSize = max<unsigned>(128, m_batchSize * 9 / 10);
-			else if (ms < _msPerBatch * 0.9)
-				m_batchSize = m_batchSize * 10 / 9;
 
 			pending.push({ start_nonce, buf });
 			buf = (buf + 1) % c_bufferCount;
@@ -498,6 +496,20 @@ void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook
 
 				pending.pop();
 			}
+
+/*			chrono::high_resolution_clock::duration d = chrono::high_resolution_clock::now() - t;
+			if (d > chrono::milliseconds(_msPerBatch * 10 / 9))
+			{
+				cerr << "Batch of" << m_batchSize << "took" << chrono::duration_cast<chrono::milliseconds>(d).count() << "ms, >>" << _msPerBatch << "ms.";
+				m_batchSize = max<unsigned>(128, m_batchSize * 9 / 10);
+				cerr << "New batch size" << m_batchSize;
+			}
+			else if (d < chrono::milliseconds(_msPerBatch * 9 / 10))
+			{
+				cerr << "Batch of" << m_batchSize << "took" << chrono::duration_cast<chrono::milliseconds>(d).count() << "ms, <<" << _msPerBatch << "ms.";
+				m_batchSize = m_batchSize * 10 / 9;
+				cerr << "New batch size" << m_batchSize;
+			}*/
 		}
 
 		// not safe to return until this is ready

libethash-cl/ethash_cl_miner.h

@@ -44,6 +44,7 @@ public:
 	static std::string platform_info(unsigned _platformId = 0, unsigned _deviceId = 0);
 	static void listDevices();
 	static bool configureGPU(
+		unsigned _platformId,
 		bool _allowCPU,
 		unsigned _extraGPUMemory,
 		boost::optional<uint64_t> _currentBlock

libethcore/Common.h

@@ -112,6 +112,7 @@ enum class ImportResult
 	AlreadyInChain,
 	AlreadyKnown,
 	Malformed,
+	OverbidGasPrice,
 	BadChain
 };
 

libethcore/Ethash.cpp

@@ -418,7 +418,7 @@ bool Ethash::GPUMiner::configureGPU(
 {
 	s_platformId = _platformId;
 	s_deviceId = _deviceId;
-	return ethash_cl_miner::configureGPU(_allowCPU, _extraGPUMemory, _currentBlock);
+	return ethash_cl_miner::configureGPU(_platformId, _allowCPU, _extraGPUMemory, _currentBlock);
 }
 
 #endif

libethereum/BlockChain.cpp

@@ -24,6 +24,7 @@
 #if ETH_PROFILING_GPERF
 #include <gperftools/profiler.h>
 #endif
+
 #include <boost/timer.hpp>
 #include <boost/filesystem.hpp>
 #include <test/JsonSpiritHeaders.h>
@@ -96,6 +97,15 @@ ldb::Slice dev::eth::toSlice(h256 const& _h, unsigned _sub)
 #endif
 }
 
+namespace dev
+{
+class WriteBatchNoter: public ldb::WriteBatch::Handler
+{
+	virtual void Put(ldb::Slice const& _key, ldb::Slice const& _value) { cnote << "Put" << toHex(bytesConstRef(_key)) << "=>" << toHex(bytesConstRef(_value)); }
+	virtual void Delete(ldb::Slice const& _key) { cnote << "Delete" << toHex(bytesConstRef(_key)); }
+};
+}
+
 #if ETH_DEBUG&&0
 static const chrono::system_clock::duration c_collectionDuration = chrono::seconds(15);
 static const unsigned c_collectionQueueSize = 2;
@@ -282,15 +292,6 @@ void BlockChain::rebuild(std::string const& _path, std::function<void(unsigned,
 	boost::filesystem::remove_all(path + "/details.old");
 }
 
-template <class T, class V>
-bool contains(T const& _t, V const& _v)
-{
-	for (auto const& i: _t)
-		if (i == _v)
-			return true;
-	return false;
-}
-
 LastHashes BlockChain::lastHashes(unsigned _n) const
 {
 	Guard l(x_lastLastHashes);
@@ -400,7 +401,6 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import
 //		clog(BlockChainNote) << "   Malformed block: " << diagnostic_information(ex);
 		ex << errinfo_phase(2);
 		ex << errinfo_now(time(0));
-		ex << errinfo_block(_block);
 		throw;
 	}
 #endif
@@ -533,7 +533,7 @@ ImportRoute BlockChain::import(VerifiedBlockRef const& _block, OverlayDB const&
 #endif
 	}
 #if ETH_CATCH
-	catch (BadRoot& ex)
+	catch (BadRoot&)
 	{
 		cwarn << "BadRoot error. Retrying import later.";
 		BOOST_THROW_EXCEPTION(FutureTime());
@@ -639,8 +639,25 @@ ImportRoute BlockChain::import(VerifiedBlockRef const& _block, OverlayDB const&
 		clog(BlockChainChat) << "   Imported but not best (oTD:" << details(last).totalDifficulty << " > TD:" << td << ")";
 	}
 
-	m_blocksDB->Write(m_writeOptions, &blocksBatch);
-	m_extrasDB->Write(m_writeOptions, &extrasBatch);
+	ldb::Status o = m_blocksDB->Write(m_writeOptions, &blocksBatch);
+	if (!o.ok())
+	{
+		cwarn << "Error writing to blockchain database: " << o.ToString();
+		WriteBatchNoter n;
+		blocksBatch.Iterate(&n);
+		cwarn << "Fail writing to blockchain database. Bombing out.";
+		exit(-1);
+	}
+	
+	o = m_extrasDB->Write(m_writeOptions, &extrasBatch);
+	if (!o.ok())
+	{
+		cwarn << "Error writing to extras database: " << o.ToString();
+		WriteBatchNoter n;
+		extrasBatch.Iterate(&n);
+		cwarn << "Fail writing to extras database. Bombing out.";
+		exit(-1);
+	}
 
 #if ETH_PARANOIA || !ETH_TRUE
 	if (isKnown(_block.info.hash()) && !details(_block.info.hash()))
@@ -668,7 +685,14 @@ ImportRoute BlockChain::import(VerifiedBlockRef const& _block, OverlayDB const&
 		{
 			m_lastBlockHash = newLastBlockHash;
 			m_lastBlockNumber = newLastBlockNumber;
-			m_extrasDB->Put(m_writeOptions, ldb::Slice("best"), ldb::Slice((char const*)&m_lastBlockHash, 32));
+			o = m_extrasDB->Put(m_writeOptions, ldb::Slice("best"), ldb::Slice((char const*)&m_lastBlockHash, 32));
+			if (!o.ok())
+			{
+				cwarn << "Error writing to extras database: " << o.ToString();
+				cout << "Put" << toHex(bytesConstRef(ldb::Slice("best"))) << "=>" << toHex(bytesConstRef(ldb::Slice((char const*)&m_lastBlockHash, 32)));
+				cwarn << "Fail writing to extras database. Bombing out.";
+				exit(-1);
+			}
 		}
 
 #if ETH_PARANOIA || !ETH_TRUE
@@ -1127,6 +1151,8 @@ VerifiedBlockRef BlockChain::verifyBlock(bytes const& _block, function<void(Exce
 			ex << errinfo_transactionIndex(i);
 			ex << errinfo_transaction(d.toBytes());
 			ex << errinfo_block(_block);
+			if (_onBad)
+				_onBad(ex);
 			throw;
 		}
 		++i;

libethereum/BlockChainSync.cpp

@@ -534,7 +534,7 @@ bool PV60Sync::shouldGrabBlocks(std::shared_ptr<EthereumPeer> _peer) const
 
 void PV60Sync::attemptSync(std::shared_ptr<EthereumPeer> _peer)
 {
-	if (m_state != SyncState::Idle)
+	if (m_state != SyncState::Idle || _peer->m_asking != Asking::Nothing)
 	{
 		clog(NetAllDetail) << "Can't sync with this peer - outstanding asks.";
 		return;
@@ -649,9 +649,8 @@ void PV60Sync::noteDoneBlocks(std::shared_ptr<EthereumPeer> _peer, bool _clemenc
 		}
 		resetSync();
 		downloadMan().reset();
-		transition(_peer, SyncState::Idle);
-	}
 	_peer->m_sub.doneFetch();
+	}
 }
 
 void PV60Sync::onPeerHashes(std::shared_ptr<EthereumPeer> _peer, h256s const& _hashes)

libethereum/BlockQueue.cpp

@@ -102,7 +102,7 @@ void BlockQueue::verifierBody()
 			BlockInfo bi;
 			bi.mixHash = work.hash;
 			bi.parentHash = work.parentHash;
-			m_verifying.push_back(VerifiedBlock { VerifiedBlockRef { bytesConstRef(), move(bi), Transactions() }, bytes() });
+			m_verifying.emplace_back(move(bi));
 		}
 
 		VerifiedBlock res;
@@ -148,7 +148,7 @@ void BlockQueue::verifierBody()
 					m_knownBad.insert(res.verified.info.hash());
 				}
 				else
-					m_verified.push_back(move(res));
+					m_verified.emplace_back(move(res));
 				while (m_verifying.size() && !m_verifying.front().blockData.empty())
 				{
 					if (m_knownBad.count(m_verifying.front().verified.info.parentHash))
@@ -157,7 +157,7 @@ void BlockQueue::verifierBody()
 						m_knownBad.insert(res.verified.info.hash());
 					}
 					else
-						m_verified.push_back(move(m_verifying.front()));
+						m_verified.emplace_back(move(m_verifying.front()));
 					m_verifying.pop_front();
 				}
 				ready = true;

libethereum/CMakeLists.txt

@@ -32,7 +32,6 @@ target_link_libraries(${EXECUTABLE} p2p)
 target_link_libraries(${EXECUTABLE} devcrypto)
 target_link_libraries(${EXECUTABLE} ethcore)
 target_link_libraries(${EXECUTABLE} ${Boost_REGEX_LIBRARIES})
-target_link_libraries(${EXECUTABLE} secp256k1)
 if (JSONRPC)
 	target_link_libraries(${EXECUTABLE} ${JSON_RPC_CPP_CLIENT_LIBRARIES})
 	target_link_libraries(${EXECUTABLE} ${CURL_LIBRARIES})

libethereum/Client.cpp

@@ -105,7 +105,8 @@ void Client::onBadBlock(Exception& _ex) const
 	bytes const* block = boost::get_error_info<errinfo_block>(_ex);
 	if (!block)
 	{
-		cwarn << "ODD: onBadBlock called but exception has no block in it.";
+		cwarn << "ODD: onBadBlock called but exception (" << _ex.what() << ") has no block in it.";
+		cwarn << boost::diagnostic_information(_ex, true);
 		return;
 	}
 

libethereum/EthereumPeer.cpp

@@ -152,6 +152,8 @@ void EthereumPeer::requestHashes(u256 _number, unsigned _count)
 
 void EthereumPeer::requestHashes(h256 const& _lastHash)
 {
+	if (m_asking != Asking::Nothing)
+		clog(NetWarn) << "Asking hashes while requesting " << (m_asking == Asking::Nothing ? "nothing" : m_asking == Asking::State ? "state" : m_asking == Asking::Hashes ? "hashes" : m_asking == Asking::Blocks ? "blocks" : "?");
 	assert(m_asking == Asking::Nothing);
 	setAsking(Asking::Hashes);
 	RLPStream s;

libethereum/State.cpp

@@ -25,7 +25,6 @@
 #include <random>
 #include <boost/filesystem.hpp>
 #include <boost/timer.hpp>
-#include <secp256k1/secp256k1.h>
 #include <libdevcore/CommonIO.h>
 #include <libdevcore/Assertions.h>
 #include <libdevcore/StructuredLogger.h>
@@ -117,7 +116,7 @@ State::State(OverlayDB const& _db, BaseState _bs, Address _coinbaseAddress):
 
 PopulationStatistics State::populateFromChain(BlockChain const& _bc, h256 const& _h, ImportRequirements::value _ir)
 {
-	PopulationStatistics ret;
+	PopulationStatistics ret { 0.0, 0.0 };
 
 	if (!_bc.isKnown(_h))
 	{

libethereum/TransactionQueue.cpp

@@ -113,6 +113,26 @@ ImportResult TransactionQueue::manageImport_WITH_LOCK(h256 const& _h, Transactio
 		// If it doesn't work, the signature is bad.
 		// The transaction's nonce may yet be invalid (or, it could be "valid" but we may be missing a marginally older transaction).
 
+		auto r = m_senders.equal_range(_transaction.from());
+		for (auto it = r.first; it != r.second; ++it)
+			if (m_current.count(it->second) && m_current[it->second].nonce() == _transaction.nonce())
+				if (_transaction.gasPrice() < m_current[it->second].gasPrice())
+					return ImportResult::OverbidGasPrice;
+				else
+				{
+					remove_WITH_LOCK(it->second);
+					break;
+				}
+			else if (m_future.count(it->second) && m_future[it->second].nonce() == _transaction.nonce())
+				if (_transaction.gasPrice() < m_future[it->second].gasPrice())
+					return ImportResult::OverbidGasPrice;
+				else
+				{
+					remove_WITH_LOCK(it->second);
+					break;
+				}
+			else {}
+
 		// If valid, append to blocks.
 		insertCurrent_WITH_LOCK(make_pair(_h, _transaction));
 		m_known.insert(_h);
@@ -200,6 +220,7 @@ bool TransactionQueue::remove_WITH_LOCK(h256 const& _txHash)
 
 unsigned TransactionQueue::waiting(Address const& _a) const
 {
+	ReadGuard l(m_lock);
 	auto it = m_senders.equal_range(_a);
 	unsigned ret = 0;
 	for (auto i = it.first; i != it.second; ++i, ++ret) {}

libethereum/VerifiedBlock.h

@@ -43,8 +43,32 @@ struct VerifiedBlockRef
 /// @brief Verified block info, combines block data and verified info/transactions
 struct VerifiedBlock
 {
+	VerifiedBlock() {};
+
+	VerifiedBlock(BlockInfo&& _bi)
+	{
+		verified.info = _bi;
+	}
+
+	VerifiedBlock(VerifiedBlock&& _other):
+		verified(std::move(_other.verified)),
+		blockData(std::move(_other.blockData))
+	{
+	}
+
+	VerifiedBlock& operator=(VerifiedBlock&& _other)
+	{
+		verified = (std::move(_other.verified));
+		blockData = (std::move(_other.blockData));
+		return *this;
+	}
+
 	VerifiedBlockRef verified;				///< Verified block structures
 	bytes blockData;						///< Block data
+
+private:
+	VerifiedBlock(VerifiedBlock const&) = delete;
+	VerifiedBlock operator=(VerifiedBlock const&) = delete;
 };
 
 using VerifiedBlocks = std::vector<VerifiedBlock>;

libethereumx/CMakeLists.txt

@@ -24,7 +24,6 @@ target_link_libraries(${EXECUTABLE} ethereum)
 target_link_libraries(${EXECUTABLE} evm)
 target_link_libraries(${EXECUTABLE} lll)
 target_link_libraries(${EXECUTABLE} ethcore)
-target_link_libraries(${EXECUTABLE} secp256k1)
 
 install( TARGETS ${EXECUTABLE} RUNTIME DESTINATION bin ARCHIVE DESTINATION lib LIBRARY DESTINATION lib )
 install( FILES ${HEADERS} DESTINATION include/${EXECUTABLE} )

libp2p/Common.h

@@ -149,14 +149,15 @@ using CapDescs = std::vector<CapDesc>;
  */
 struct PeerSessionInfo
 {
-	NodeId id;
-	std::string clientVersion;
-	std::string host;
-	unsigned short port;
+	NodeId const id;
+	std::string const clientVersion;
+	std::string const host;
+	unsigned short const port;
 	std::chrono::steady_clock::duration lastPing;
-	std::set<CapDesc> caps;
+	std::set<CapDesc> const caps;
 	unsigned socketId;
 	std::map<std::string, std::string> notes;
+	unsigned const protocolVersion;
 };
 
 using PeerSessionInfos = std::vector<PeerSessionInfo>;

libp2p/Host.cpp

@@ -254,7 +254,7 @@ void Host::startPeerSession(Public const& _id, RLP const& _rlp, RLPXFrameCoder*
 	clog(NetMessageSummary) << "Hello: " << clientVersion << "V[" << protocolVersion << "]" << _id << showbase << capslog.str() << dec << listenPort;
 	
 	// create session so disconnects are managed
-	auto ps = make_shared<Session>(this, _io, _s, p, PeerSessionInfo({_id, clientVersion, p->endpoint.address.to_string(), listenPort, chrono::steady_clock::duration(), _rlp[2].toSet<CapDesc>(), 0, map<string, string>()}));
+	auto ps = make_shared<Session>(this, _io, _s, p, PeerSessionInfo({_id, clientVersion, p->endpoint.address.to_string(), listenPort, chrono::steady_clock::duration(), _rlp[2].toSet<CapDesc>(), 0, map<string, string>(), protocolVersion}));
 	if (protocolVersion < dev::p2p::c_protocolVersion - 1)
 	{
 		ps->disconnect(IncompatibleProtocol);
@@ -725,8 +725,16 @@ void Host::startedWorking()
 
 void Host::doWork()
 {
-	if (m_run)
-		m_ioService.run();
+	try
+	{
+		if (m_run)
+			m_ioService.run();
+	}
+	catch (std::exception const& _e)
+	{
+		clog(NetP2PWarn) << "Exception in Network Thread:" << _e.what();
+		clog(NetP2PWarn) << "Network Restart is Recommended.";
+	}
 }
 
 void Host::keepAlivePeers()

libp2p/RLPXFrameCoder.cpp

@@ -29,6 +29,18 @@ using namespace dev;
 using namespace dev::p2p;
 using namespace CryptoPP;
 
+RLPXFrameInfo::RLPXFrameInfo(bytesConstRef _header)
+{
+	length = (_header[0] * 256 + _header[1]) * 256 + _header[2];
+	padding = ((16 - (length % 16)) % 16);
+	RLP header(_header.cropped(3), RLP::ThrowOnFail | RLP::FailIfTooSmall);
+	auto itemCount = header.itemCount();
+	protocolId = header[0].toInt<uint16_t>();
+	hasSequence = itemCount > 1;
+	sequenceId = hasSequence ? header[1].toInt<uint16_t>() : 0;
+	totalLength = itemCount == 3 ? header[2].toInt<uint32_t>() : 0;
+}
+
 RLPXFrameCoder::RLPXFrameCoder(RLPXHandshake const& _init)
 {
 	// we need:

libp2p/RLPXFrameCoder.h

@@ -32,7 +32,21 @@ namespace dev
 {
 namespace p2p
 {
+
+struct RLPXFrameInfo
+{
+	RLPXFrameInfo() = default;
+	/// Constructor. frame-size || protocol-type, [sequence-id[, total-packet-size]]
+	RLPXFrameInfo(bytesConstRef _frameHeader);
+	uint32_t length = 0;			///< Max: 2**24
+	uint8_t padding = 0;
 	
+	uint16_t protocolId = 0;
+	bool hasSequence = false;
+	uint16_t sequenceId = 0;
+	uint32_t totalLength = 0;
+};
+
 class RLPXHandshake;
 
 /**

libp2p/Session.cpp

@@ -27,7 +27,6 @@
 #include <libdevcore/CommonIO.h>
 #include <libdevcore/StructuredLogger.h>
 #include <libethcore/Exceptions.h>
-#include "RLPxHandshake.h"
 #include "Host.h"
 #include "Capability.h"
 using namespace std;
@@ -157,124 +156,76 @@ void Session::serviceNodesRequest()
 	addNote("peers", "done");
 }
 
-bool Session::interpret(PacketType _t, RLP const& _r)
+bool Session::readPacket(uint16_t _capId, PacketType _t, RLP const& _r)
 {
 	m_lastReceived = chrono::steady_clock::now();
-
 	clog(NetRight) << _t << _r;
 	try		// Generic try-catch block designed to capture RLP format errors - TODO: give decent diagnostics, make a bit more specific over what is caught.
 	{
-		switch (_t)
-		{
-		case DisconnectPacket:
-		{
-			string reason = "Unspecified";
-			auto r = (DisconnectReason)_r[0].toInt<int>();
-			if (!_r[0].isInt())
-				drop(BadProtocol);
-			else
-			{
-				reason = reasonOf(r);
-				clog(NetMessageSummary) << "Disconnect (reason: " << reason << ")";
-				drop(DisconnectRequested);
-			}
-			break;
-		}
-		case PingPacket:
-		{
-			clog(NetTriviaSummary) << "Ping";
-			RLPStream s;
-			sealAndSend(prep(s, PongPacket));
-			break;
-		}
-		case PongPacket:
-		{
-			DEV_GUARDED(x_info)
-				m_info.lastPing = std::chrono::steady_clock::now() - m_ping;
-			clog(NetTriviaSummary) << "Latency: " << chrono::duration_cast<chrono::milliseconds>(m_info.lastPing).count() << " ms";
-			break;
-		}
-		case GetPeersPacket:
-			// Disabled for interop testing.
-			// GetPeers/PeersPacket will be modified to only exchange new nodes which it's peers are interested in.
-			break;
-
-			clog(NetTriviaSummary) << "GetPeers";
-			m_theyRequestedNodes = true;
-			serviceNodesRequest();
-			break;
-		case PeersPacket:
-			// Disabled for interop testing.
-			// GetPeers/PeersPacket will be modified to only exchange new nodes which it's peers are interested in.
-			break;
-
-			clog(NetTriviaSummary) << "Peers (" << dec << (_r.itemCount() - 1) << " entries)";
-			m_weRequestedNodes = false;
-			for (unsigned i = 0; i < _r.itemCount(); ++i)
-			{
-				bi::address peerAddress;
-				if (_r[i][0].size() == 16)
-					peerAddress = bi::address_v6(_r[i][0].toHash<FixedHash<16>>().asArray());
-				else if (_r[i][0].size() == 4)
-					peerAddress = bi::address_v4(_r[i][0].toHash<FixedHash<4>>().asArray());
-				else
-				{
-					cwarn << "Received bad peer packet:" << _r;
-					disconnect(BadProtocol);
-					return true;
-				}
-				auto ep = bi::tcp::endpoint(peerAddress, _r[i][1].toInt<short>());
-				NodeId id = _r[i][2].toHash<NodeId>();
-
-				clog(NetAllDetail) << "Checking: " << ep << "(" << id << ")";
-
-				if (!isPublicAddress(peerAddress))
-					goto CONTINUE;	// Private address. Ignore.
-
-				if (!id)
-					goto LAMEPEER;	// Null identity. Ignore.
-
-				if (m_server->id() == id)
-					goto LAMEPEER;	// Just our info - we already have that.
-
-				if (id == this->id())
-					goto LAMEPEER;	// Just their info - we already have that.
-
-				if (!ep.port())
-					goto LAMEPEER;	// Zero port? Don't think so.
-
-				if (ep.port() >= /*49152*/32768)
-					goto LAMEPEER;	// Private port according to IANA.
-
-				// OK passed all our checks. Assume it's good.
-				addRating(1000);
-				m_server->addNode(id, NodeIPEndpoint(ep.address(), ep.port(), ep.port()));
-				clog(NetTriviaDetail) << "New peer: " << ep << "(" << id << ")";
-				CONTINUE:;
-				LAMEPEER:;
-			}
-			break;
-		default:
+		// v4 frame headers are useless, offset packet type used
+		// v5 protocol type is in header, packet type not offset
+		if (_capId == 0 && _t < UserPacket)
+			return interpret(_t, _r);
+		if (m_info.protocolVersion >= 5)
+			for (auto const& i: m_capabilities)
+				if (_capId == (uint16_t)i.first.second)
+					return i.second->m_enabled ? i.second->interpret(_t, _r) : true;
+		if (m_info.protocolVersion <= 4)
 			for (auto const& i: m_capabilities)
 				if (_t >= (int)i.second->m_idOffset && _t - i.second->m_idOffset < i.second->hostCapability()->messageCount())
-				{
-					if (i.second->m_enabled)
-						return i.second->interpret(_t - i.second->m_idOffset, _r);
-					else
-						return true;
-				}
-			return false;
-		}
+					return i.second->m_enabled ? i.second->interpret(_t - i.second->m_idOffset, _r) : true;
+		return false;
 	}
 	catch (std::exception const& _e)
 	{
-		clog(NetWarn) << "Peer causing an exception:" << _e.what() << _r;
+		clog(NetWarn) << "Exception caught in p2p::Session::interpret(): " << _e.what() << ". PacketType: " << _t << ". RLP: " << _r;
 		disconnect(BadProtocol);
 		return true;
 	}
 	return true;
 }
 
+bool Session::interpret(PacketType _t, RLP const& _r)
+{
+	switch (_t)
+	{
+	case DisconnectPacket:
+	{
+		string reason = "Unspecified";
+		auto r = (DisconnectReason)_r[0].toInt<int>();
+		if (!_r[0].isInt())
+			drop(BadProtocol);
+		else
+		{
+			reason = reasonOf(r);
+			clog(NetMessageSummary) << "Disconnect (reason: " << reason << ")";
+			drop(DisconnectRequested);
+		}
+		break;
+	}
+	case PingPacket:
+	{
+		clog(NetTriviaSummary) << "Ping";
+		RLPStream s;
+		sealAndSend(prep(s, PongPacket));
+		break;
+	}
+	case PongPacket:
+		DEV_GUARDED(x_info)
+		{
+			m_info.lastPing = std::chrono::steady_clock::now() - m_ping;
+			clog(NetTriviaSummary) << "Latency: " << chrono::duration_cast<chrono::milliseconds>(m_info.lastPing).count() << " ms";
+		}
+		break;
+	case GetPeersPacket:
+	case PeersPacket:
+		break;
+	default:
+		return false;
+	}
+	return true;
+}
+
 void Session::ping()
 {
 	RLPStream s;
@@ -296,12 +247,9 @@ void Session::sealAndSend(RLPStream& _s)
 
 bool Session::checkPacket(bytesConstRef _msg)
 {
-	if (_msg.size() < 2)
+	if (_msg[0] > 0x7f || _msg.size() < 2)
 		return false;
-	if (_msg[0] > 0x7f)
-		return false;
-	RLP r(_msg.cropped(1));
-	if (r.actualSize() + 1 != _msg.size())
+	if (RLP(_msg.cropped(1)).actualSize() + 1 != _msg.size())
 		return false;
 	return true;
 }
@@ -418,82 +366,78 @@ void Session::doRead()
 	{
 		ThreadContext tc(info().id.abridged());
 		ThreadContext tc2(info().clientVersion);
-		if (ec && ec.category() != boost::asio::error::get_misc_category() && ec.value() != boost::asio::error::eof)
+		if (!checkRead(h256::size, ec, length))
+			return;
+		else if (!m_io->authAndDecryptHeader(bytesRef(m_data.data(), length)))
 		{
-			clog(NetWarn) << "Error reading: " << ec.message();
-			drop(TCPError);
+			clog(NetWarn) << "header decrypt failed";
+			drop(BadProtocol); // todo: better error
+			return;
+		}
+
+		RLPXFrameInfo header;
+		try
+		{
+			header = RLPXFrameInfo(bytesConstRef(m_data.data(), length));
 		}
-		else if (ec && length == 0)
+		catch (std::exception const& _e)
+		{
+			clog(NetWarn) << "Exception decoding frame header RLP:" << bytesConstRef(m_data.data(), h128::size).cropped(3);
+			drop(BadProtocol);
 			return;
-		else
+		}
+		
+		/// read padded frame and mac
+		auto tlen = header.length + header.padding + h128::size;
+		ba::async_read(m_socket->ref(), boost::asio::buffer(m_data, tlen), [this, self, header, tlen](boost::system::error_code ec, std::size_t length)
 		{
-			/// authenticate and decrypt header
-			bytesRef header(m_data.data(), h256::size);
-			if (!m_io->authAndDecryptHeader(header))
+			ThreadContext tc(info().id.abridged());
+			ThreadContext tc2(info().clientVersion);
+			if (!checkRead(tlen, ec, length))
+				return;
+			else if (!m_io->authAndDecryptFrame(bytesRef(m_data.data(), tlen)))
 			{
-				clog(NetWarn) << "header decrypt failed";
+				clog(NetWarn) << "frame decrypt failed";
 				drop(BadProtocol); // todo: better error
 				return;
 			}
 
-			/// check frame size
-			uint32_t frameSize = (m_data[0] * 256 + m_data[1]) * 256 + m_data[2];
-			if (frameSize >= (uint32_t)1 << 24)
+			bytesConstRef frame(m_data.data(), header.length);
+			if (!checkPacket(frame))
 			{
-				clog(NetWarn) << "frame size too large";
-				drop(BadProtocol);
+				cerr << "Received " << frame.size() << ": " << toHex(frame) << endl;
+				clog(NetWarn) << "INVALID MESSAGE RECEIVED";
+				disconnect(BadProtocol);
 				return;
 			}
-			
-			/// rlp of header has protocol-type, sequence-id[, total-packet-size]
-			bytes headerRLP(13);
-			bytesConstRef(m_data.data(), h128::size).cropped(3).copyTo(&headerRLP);
-			
-			/// read padded frame and mac
-			auto tlen = frameSize + ((16 - (frameSize % 16)) % 16) + h128::size;
-			ba::async_read(m_socket->ref(), boost::asio::buffer(m_data, tlen), [this, self, headerRLP, frameSize, tlen](boost::system::error_code ec, std::size_t length)
+			else
 			{
-				ThreadContext tc(info().id.abridged());
-				ThreadContext tc2(info().clientVersion);
-				if (ec && ec.category() != boost::asio::error::get_misc_category() && ec.value() != boost::asio::error::eof)
-				{
-					clog(NetWarn) << "Error reading: " << ec.message();
-					drop(TCPError);
-				}
-				else if (ec && length < tlen)
-				{
-					clog(NetWarn) << "Error reading - Abrupt peer disconnect: " << ec.message();
-					repMan().noteRude(*this);
-					drop(TCPError);
-					return;
-				}
-				else
-				{
-					if (!m_io->authAndDecryptFrame(bytesRef(m_data.data(), tlen)))
-					{
-						clog(NetWarn) << "frame decrypt failed";
-						drop(BadProtocol); // todo: better error
-						return;
-					}
-					
-					bytesConstRef frame(m_data.data(), frameSize);
-					if (!checkPacket(frame))
-					{
-						cerr << "Received " << frame.size() << ": " << toHex(frame) << endl;
-						clog(NetWarn) << "INVALID MESSAGE RECEIVED";
-						disconnect(BadProtocol);
-						return;
-					}
-					else
-					{
-						auto packetType = (PacketType)RLP(frame.cropped(0, 1)).toInt<unsigned>();
-						RLP r(frame.cropped(1));
-						if (!interpret(packetType, r))
-							clog(NetWarn) << "Couldn't interpret packet." << RLP(r);
-					}
-					doRead();
-				}
-			});
-		}
+				auto packetType = (PacketType)RLP(frame.cropped(0, 1)).toInt<unsigned>();
+				RLP r(frame.cropped(1));
+				if (!readPacket(header.protocolId, packetType, r))
+					clog(NetWarn) << "Couldn't interpret packet." << RLP(r);
+			}
+			doRead();
+		});
 	});
 }
+
+bool Session::checkRead(std::size_t _expected, boost::system::error_code _ec, std::size_t _length)
+{
+	if (_ec && _ec.category() != boost::asio::error::get_misc_category() && _ec.value() != boost::asio::error::eof)
+	{
+		clog(NetConnect) << "Error reading: " << _ec.message();
+		drop(TCPError);
+		return false;
+	}
+	else if (_ec && _length < _expected)
+	{
+		clog(NetWarn) << "Error reading - Abrupt peer disconnect: " << _ec.message();
+		repMan().noteRude(*this);
+		drop(TCPError);
+		return false;
+	}
+	// If this fails then there's an unhandled asio error
+	assert(_expected == _length);
+	return true;
+}

libp2p/Session.h

@@ -96,13 +96,19 @@ private:
 
 	/// Perform a read on the socket.
 	void doRead();
+	
+	/// Check error code after reading and drop peer if error code.
+	bool checkRead(std::size_t _expected, boost::system::error_code _ec, std::size_t _length);
 
 	/// Perform a single round of the write operation. This could end up calling itself asynchronously.
 	void write();
 
-	/// Interpret an incoming message.
-	bool interpret(PacketType _t, RLP const& _r);
+	/// Deliver RLPX packet to Session or Capability for interpretation.
+	bool readPacket(uint16_t _capId, PacketType _t, RLP const& _r);
 
+	/// Interpret an incoming Session packet.
+	bool interpret(PacketType _t, RLP const& _r);
+	
 	/// @returns true iff the _msg forms a valid message for sending or receiving on the network.
 	static bool checkPacket(bytesConstRef _msg);
 

libsolidity/AST.cpp

@@ -175,24 +175,40 @@ void ContractDefinition::checkDuplicateFunctions() const
 
 void ContractDefinition::checkAbstractFunctions()
 {
-	map<string, bool> functions;
+	// Mapping from name to function definition (exactly one per argument type equality class) and
+	// flag to indicate whether it is fully implemented.
+	using FunTypeAndFlag = std::pair<FunctionTypePointer, bool>;
+	map<string, vector<FunTypeAndFlag>> functions;
 
 	// Search from base to derived
 	for (ContractDefinition const* contract: boost::adaptors::reverse(getLinearizedBaseContracts()))
 		for (ASTPointer<FunctionDefinition> const& function: contract->getDefinedFunctions())
 		{
-			string const& name = function->getName();
-			if (!function->isFullyImplemented() && functions.count(name) && functions[name])
-				BOOST_THROW_EXCEPTION(function->createTypeError("Redeclaring an already implemented function as abstract"));
-			functions[name] = function->isFullyImplemented();
+			auto& overloads = functions[function->getName()];
+			FunctionTypePointer funType = make_shared<FunctionType>(*function);
+			auto it = find_if(overloads.begin(), overloads.end(), [&](FunTypeAndFlag const& _funAndFlag)
+			{
+				return funType->hasEqualArgumentTypes(*_funAndFlag.first);
+			});
+			if (it == overloads.end())
+				overloads.push_back(make_pair(funType, function->isFullyImplemented()));
+			else if (it->second)
+			{
+				if (!function->isFullyImplemented())
+					BOOST_THROW_EXCEPTION(function->createTypeError("Redeclaring an already implemented function as abstract"));
+			}
+			else if (function->isFullyImplemented())
+				it->second = true;
 		}
 
+	// Set to not fully implemented if at least one flag is false.
 	for (auto const& it: functions)
-		if (!it.second)
-		{
-			setFullyImplemented(false);
-			break;
-		}
+		for (auto const& funAndFlag: it.second)
+			if (!funAndFlag.second)
+			{
+				setFullyImplemented(false);
+				return;
+			}
 }
 
 void ContractDefinition::checkAbstractConstructors()
@@ -340,8 +356,10 @@ vector<pair<FixedHash<4>, FunctionTypePointer>> const& ContractDefinition::getIn
 		{
 			for (ASTPointer<FunctionDefinition> const& f: contract->getDefinedFunctions())
 			{
+				if (!f->isPartOfExternalInterface())
+					continue;
 				string functionSignature = f->externalSignature();
-				if (f->isPartOfExternalInterface() && signaturesSeen.count(functionSignature) == 0)
+				if (signaturesSeen.count(functionSignature) == 0)
 				{
 					functionsSeen.insert(f->getName());
 					signaturesSeen.insert(functionSignature);
@@ -533,7 +551,16 @@ void VariableDeclaration::checkTypeRequirements()
 			BOOST_THROW_EXCEPTION(createTypeError("Variable cannot have void type."));
 		m_type = type->mobileType();
 	}
-	if (m_isStateVariable && getVisibility() >= Visibility::Public && !FunctionType(*this).externalType())
+	solAssert(!!m_type, "");
+	if (!m_isStateVariable)
+	{
+		if (m_type->dataStoredIn(DataLocation::Memory) || m_type->dataStoredIn(DataLocation::CallData))
+			if (!m_type->canLiveOutsideStorage())
+				BOOST_THROW_EXCEPTION(createTypeError(
+					"Type " + m_type->toString() + " is only valid in storage."
+				));
+	}
+	else if (getVisibility() >= Visibility::Public && !FunctionType(*this).externalType())
 		BOOST_THROW_EXCEPTION(createTypeError("Internal type is not allowed for public state variables."));
 }
 
@@ -925,8 +952,11 @@ void MemberAccess::checkTypeRequirements(TypePointers const* _argumentTypes)
 	else if (type.getCategory() == Type::Category::Array)
 	{
 		auto const& arrayType(dynamic_cast<ArrayType const&>(type));
-		m_isLValue = (*m_memberName == "length" &&
-			arrayType.location() != DataLocation::CallData && arrayType.isDynamicallySized());
+		m_isLValue = (
+			*m_memberName == "length" &&
+			arrayType.location() == DataLocation::Storage &&
+			arrayType.isDynamicallySized()
+		);
 	}
 	else
 		m_isLValue = false;

libsolidity/ArrayUtils.cpp

@@ -134,14 +134,14 @@ void ArrayUtils::copyArrayToStorage(ArrayType const& _targetType, ArrayType cons
 	if (sourceBaseType->getCategory() == Type::Category::Array)
 	{
 		solAssert(byteOffsetSize == 0, "Byte offset for array as base type.");
+		auto const& sourceBaseArrayType = dynamic_cast<ArrayType const&>(*sourceBaseType);
 		m_context << eth::Instruction::DUP3;
 		if (sourceIsStorage)
 			m_context << u256(0);
+		else if (sourceBaseArrayType.location() == DataLocation::Memory)
+			m_context << eth::Instruction::MLOAD;
 		m_context << eth::dupInstruction(sourceIsStorage ? 4 : 3) << u256(0);
-		copyArrayToStorage(
-			dynamic_cast<ArrayType const&>(*targetBaseType),
-			dynamic_cast<ArrayType const&>(*sourceBaseType)
-		);
+		copyArrayToStorage(dynamic_cast<ArrayType const&>(*targetBaseType), sourceBaseArrayType);
 		m_context << eth::Instruction::POP << eth::Instruction::POP;
 	}
 	else if (directCopy)
@@ -188,11 +188,18 @@ void ArrayUtils::copyArrayToStorage(ArrayType const& _targetType, ArrayType cons
 	if (haveByteOffsetSource)
 		incrementByteOffset(sourceBaseType->getStorageBytes(), 1, haveByteOffsetTarget ? 5 : 4);
 	else
+	{
+		m_context << eth::swapInstruction(2 + byteOffsetSize);
+		if (sourceIsStorage)
+			m_context << sourceBaseType->getStorageSize();
+		else if (_sourceType.location() == DataLocation::Memory)
+			m_context << sourceBaseType->memoryHeadSize();
+		else
+			m_context << sourceBaseType->getCalldataEncodedSize(true);
 		m_context
-			<< eth::swapInstruction(2 + byteOffsetSize)
-			<< (sourceIsStorage ? sourceBaseType->getStorageSize() : sourceBaseType->getCalldataEncodedSize())
 			<< eth::Instruction::ADD
 			<< eth::swapInstruction(2 + byteOffsetSize);
+	}
 	// increment target
 	if (haveByteOffsetTarget)
 		incrementByteOffset(targetBaseType->getStorageBytes(), byteOffsetSize, byteOffsetSize + 2);
@@ -235,8 +242,9 @@ void ArrayUtils::copyArrayToMemory(const ArrayType& _sourceType, bool _padToWord
 {
 	solAssert(
 		_sourceType.getBaseType()->getCalldataEncodedSize() > 0,
-		"Nested arrays not yet implemented here."
+		"Nested dynamic arrays not implemented here."
 	);
+	CompilerUtils utils(m_context);
 	unsigned baseSize = 1;
 	if (!_sourceType.isByteArray())
 		// We always pad the elements, regardless of _padToWordBoundaries.
@@ -246,7 +254,7 @@ void ArrayUtils::copyArrayToMemory(const ArrayType& _sourceType, bool _padToWord
 	{
 		if (!_sourceType.isDynamicallySized())
 			m_context << _sourceType.getLength();
-		if (_sourceType.getBaseType()->getCalldataEncodedSize() > 1)
+		if (baseSize > 1)
 			m_context << u256(baseSize) << eth::Instruction::MUL;
 		// stack: target source_offset source_len
 		m_context << eth::Instruction::DUP1 << eth::Instruction::DUP3 << eth::Instruction::DUP5;
@@ -257,8 +265,36 @@ void ArrayUtils::copyArrayToMemory(const ArrayType& _sourceType, bool _padToWord
 	}
 	else if (_sourceType.location() == DataLocation::Memory)
 	{
-		// memcpy using the built-in contract
 		retrieveLength(_sourceType);
+		// stack: target source length
+		if (!_sourceType.getBaseType()->isValueType())
+		{
+			// copy using a loop
+			m_context << u256(0) << eth::Instruction::SWAP3;
+			// stack: counter source length target
+			auto repeat = m_context.newTag();
+			m_context << repeat;
+			m_context << eth::Instruction::DUP2 << eth::Instruction::DUP5;
+			m_context << eth::Instruction::LT << eth::Instruction::ISZERO;
+			auto loopEnd = m_context.appendConditionalJump();
+			m_context << eth::Instruction::DUP3 << eth::Instruction::DUP5;
+			accessIndex(_sourceType, false);
+			MemoryItem(m_context, *_sourceType.getBaseType(), true).retrieveValue(SourceLocation(), true);
+			if (auto baseArray = dynamic_cast<ArrayType const*>(_sourceType.getBaseType().get()))
+				copyArrayToMemory(*baseArray, _padToWordBoundaries);
+			else
+				utils.storeInMemoryDynamic(*_sourceType.getBaseType());
+			m_context << eth::Instruction::SWAP3 << u256(1) << eth::Instruction::ADD;
+			m_context << eth::Instruction::SWAP3;
+			m_context.appendJumpTo(repeat);
+			m_context << loopEnd;
+			m_context << eth::Instruction::SWAP3;
+			utils.popStackSlots(3);
+			// stack: updated_target_pos
+			return;
+		}
+
+		// memcpy using the built-in contract
 		if (_sourceType.isDynamicallySized())
 		{
 			// change pointer to data part
@@ -271,7 +307,7 @@ void ArrayUtils::copyArrayToMemory(const ArrayType& _sourceType, bool _padToWord
 		// stack: <target> <source> <size>
 		//@TODO do not use ::CALL if less than 32 bytes?
 		m_context << eth::Instruction::DUP1 << eth::Instruction::DUP4 << eth::Instruction::DUP4;
-		CompilerUtils(m_context).memoryCopy();
+		utils.memoryCopy();
 
 		m_context << eth::Instruction::SWAP1 << eth::Instruction::POP;
 		// stack: <target> <size>
@@ -345,7 +381,7 @@ void ArrayUtils::copyArrayToMemory(const ArrayType& _sourceType, bool _padToWord
 		{
 			// actual array data is stored at SHA3(storage_offset)
 			m_context << eth::Instruction::SWAP1;
-			CompilerUtils(m_context).computeHashStatic();
+			utils.computeHashStatic();
 			m_context << eth::Instruction::SWAP1;
 		}
 
@@ -375,7 +411,10 @@ void ArrayUtils::copyArrayToMemory(const ArrayType& _sourceType, bool _padToWord
 			else
 				m_context << eth::Instruction::DUP2 << u256(0);
 			StorageItem(m_context, *_sourceType.getBaseType()).retrieveValue(SourceLocation(), true);
-			CompilerUtils(m_context).storeInMemoryDynamic(*_sourceType.getBaseType());
+			if (auto baseArray = dynamic_cast<ArrayType const*>(_sourceType.getBaseType().get()))
+				copyArrayToMemory(*baseArray, _padToWordBoundaries);
+			else
+				utils.storeInMemoryDynamic(*_sourceType.getBaseType());
 			// increment storage_data_offset and byte offset
 			if (haveByteOffset)
 				incrementByteOffset(storageBytes, 2, 3);
@@ -387,7 +426,8 @@ void ArrayUtils::copyArrayToMemory(const ArrayType& _sourceType, bool _padToWord
 			}
 		}
 		// check for loop condition
-		m_context << eth::Instruction::DUP1 << eth::dupInstruction(haveByteOffset ? 5 : 4) << eth::Instruction::GT;
+		m_context << eth::Instruction::DUP1 << eth::dupInstruction(haveByteOffset ? 5 : 4);
+		m_context << eth::Instruction::GT;
 		m_context.appendConditionalJumpTo(loopStart);
 		// stack here: memory_end_offset storage_data_offset [storage_byte_offset] memory_offset
 		if (haveByteOffset)
@@ -597,12 +637,14 @@ void ArrayUtils::convertLengthToSize(ArrayType const& _arrayType, bool _pad) con
 	}
 	else
 	{
-		solAssert(
-			_arrayType.getBaseType()->getCalldataEncodedSize() > 0,
-			"Copying nested dynamic arrays not yet implemented."
-		);
 		if (!_arrayType.isByteArray())
-			m_context << _arrayType.getBaseType()->getCalldataEncodedSize() << eth::Instruction::MUL;
+		{
+			if (_arrayType.location() == DataLocation::Memory)
+				m_context << _arrayType.getBaseType()->memoryHeadSize();
+			else
+				m_context << _arrayType.getBaseType()->getCalldataEncodedSize();
+			m_context << eth::Instruction::MUL;
+		}
 		else if (_pad)
 			m_context << u256(31) << eth::Instruction::ADD
 				<< u256(32) << eth::Instruction::DUP1
@@ -632,7 +674,7 @@ void ArrayUtils::retrieveLength(ArrayType const& _arrayType) const
 	}
 }
 
-void ArrayUtils::accessIndex(ArrayType const& _arrayType) const
+void ArrayUtils::accessIndex(ArrayType const& _arrayType, bool _doBoundsCheck) const
 {
 	DataLocation location = _arrayType.location();
 	eth::Instruction load =
@@ -640,19 +682,25 @@ void ArrayUtils::accessIndex(ArrayType const& _arrayType) const
 		location == DataLocation::Memory ? eth::Instruction::MLOAD :
 		eth::Instruction::CALLDATALOAD;
 
-	// retrieve length
-	if (!_arrayType.isDynamicallySized())
-		m_context << _arrayType.getLength();
-	else if (location == DataLocation::CallData)
-		// length is stored on the stack
-		m_context << eth::Instruction::SWAP1;
-	else
-		m_context << eth::Instruction::DUP2 << load;
-	// stack: <base_ref> <index> <length>
-	// check out-of-bounds access
-	m_context << eth::Instruction::DUP2 << eth::Instruction::LT << eth::Instruction::ISZERO;
-	// out-of-bounds access throws exception
-	m_context.appendConditionalJumpTo(m_context.errorTag());
+	if (_doBoundsCheck)
+	{
+		// retrieve length
+		if (!_arrayType.isDynamicallySized())
+			m_context << _arrayType.getLength();
+		else if (location == DataLocation::CallData)
+			// length is stored on the stack
+			m_context << eth::Instruction::SWAP1;
+		else
+			m_context << eth::Instruction::DUP2 << load;
+		// stack: <base_ref> <index> <length>
+		// check out-of-bounds access
+		m_context << eth::Instruction::DUP2 << eth::Instruction::LT << eth::Instruction::ISZERO;
+		// out-of-bounds access throws exception
+		m_context.appendConditionalJumpTo(m_context.errorTag());
+	}
+	else if (location == DataLocation::CallData && _arrayType.isDynamicallySized())
+		// remove length if present
+		m_context << eth::Instruction::SWAP1 << eth::Instruction::POP;
 
 	// stack: <base_ref> <index>
 	m_context << eth::Instruction::SWAP1;
@@ -671,18 +719,13 @@ void ArrayUtils::accessIndex(ArrayType const& _arrayType) const
 		if (!_arrayType.isByteArray())
 		{
 			m_context << eth::Instruction::SWAP1;
-			m_context << _arrayType.getBaseType()->getCalldataEncodedSize() << eth::Instruction::MUL;
+			if (location == DataLocation::CallData)
+				m_context << _arrayType.getBaseType()->getCalldataEncodedSize();
+			else
+				m_context << u256(_arrayType.memoryHeadSize());
+			m_context << eth::Instruction::MUL;
 		}
 		m_context << eth::Instruction::ADD;
-		//@todo we should also load if it is a reference type of dynamic length
-		// but we should apply special logic if we load from calldata.
-		if (_arrayType.getBaseType()->isValueType())
-			CompilerUtils(m_context).loadFromMemoryDynamic(
-				*_arrayType.getBaseType(),
-				location == DataLocation::CallData,
-				!_arrayType.isByteArray(),
-				false
-			);
 		break;
 	case DataLocation::Storage:
 		m_context << eth::Instruction::SWAP1;

libsolidity/ArrayUtils.h

@@ -44,7 +44,11 @@ public:
 	/// Stack pre: source_reference [source_byte_offset/source_length] target_reference target_byte_offset
 	/// Stack post: target_reference target_byte_offset
 	void copyArrayToStorage(ArrayType const& _targetType, ArrayType const& _sourceType) const;
-	/// Copies an array (which cannot be dynamically nested) from anywhere to memory.
+	/// Copies the data part of an array (which cannot be dynamically nested) from anywhere
+	/// to a given position in memory.
+	/// This always copies contained data as is (i.e. structs and fixed-size arrays are copied in
+	/// place as required by the ABI encoding). Use CompilerUtils::convertType if you want real
+	/// memory copies of nested arrays.
 	/// Stack pre: memory_offset source_item
 	/// Stack post: memory_offest + length(padded)
 	void copyArrayToMemory(ArrayType const& _sourceType, bool _padToWordBoundaries = true) const;
@@ -74,12 +78,11 @@ public:
 	/// Stack pre: reference (excludes byte offset for dynamic storage arrays)
 	/// Stack post: reference length
 	void retrieveLength(ArrayType const& _arrayType) const;
-	/// Retrieves the value at a specific index. If the location is storage, only retrieves the
-	/// position.
+	/// Performs bounds checking and returns a reference on the stack.
 	/// Stack pre: reference [length] index
-	/// Stack post for storage: slot byte_offset
-	/// Stack post for calldata: value
-	void accessIndex(ArrayType const& _arrayType) const;
+	/// Stack post (storage): storage_slot byte_offset
+	/// Stack post: memory/calldata_offset
+	void accessIndex(ArrayType const& _arrayType, bool _doBoundsCheck = true) const;
 
 private:
 	/// Adds the given number of bytes to a storage byte offset counter and also increments

libsolidity/Compiler.cpp

@@ -261,7 +261,7 @@ void Compiler::appendCalldataUnpacker(
 {
 	// We do not check the calldata size, everything is zero-paddedd
 
-	//@todo this does not yet support nested arrays
+	//@todo this does not yet support nested dynamic arrays
 
 	if (_startOffset == u256(-1))
 		_startOffset = u256(CompilerUtils::dataStartOffset);
@@ -279,6 +279,12 @@ void Compiler::appendCalldataUnpacker(
 			solAssert(!arrayType.getBaseType()->isDynamicallySized(), "Nested arrays not yet implemented.");
 			if (_fromMemory)
 			{
+				solAssert(
+					arrayType.getBaseType()->isValueType(),
+					"Nested memory arrays not yet implemented here."
+				);
+				// @todo If base type is an array or struct, it is still calldata-style encoded, so
+				// we would have to convert it like below.
 				solAssert(arrayType.location() == DataLocation::Memory, "");
 				// compute data pointer
 				m_context << eth::Instruction::DUP1 << eth::Instruction::MLOAD;
@@ -311,6 +317,7 @@ void Compiler::appendCalldataUnpacker(
 				}
 				if (arrayType.location() == DataLocation::Memory)
 				{
+					// stack: calldata_ref [length] next_calldata
 					// copy to memory
 					// move calldata type up again
 					CompilerUtils(m_context).moveIntoStack(calldataType->getSizeOnStack());
@@ -657,7 +664,7 @@ void Compiler::appendStackVariableInitialisation(VariableDeclaration const& _var
 {
 	CompilerContext::LocationSetter location(m_context, _variable);
 	m_context.addVariable(_variable);
-	ExpressionCompiler(m_context).appendStackVariableInitialisation(*_variable.getType());
+	CompilerUtils(m_context).pushZeroValue(*_variable.getType());
 }
 
 void Compiler::compileExpression(Expression const& _expression, TypePointer const& _targetType)

libsolidity/CompilerUtils.cpp

@@ -54,6 +54,13 @@ void CompilerUtils::storeFreeMemoryPointer()
 	m_context << u256(freeMemoryPointer) << eth::Instruction::MSTORE;
 }
 
+void CompilerUtils::allocateMemory()
+{
+	fetchFreeMemoryPointer();
+	m_context << eth::Instruction::SWAP1 << eth::Instruction::DUP2 << eth::Instruction::ADD;
+	storeFreeMemoryPointer();
+}
+
 void CompilerUtils::toSizeAfterFreeMemoryPointer()
 {
 	fetchFreeMemoryPointer();
@@ -101,17 +108,20 @@ void CompilerUtils::storeInMemory(unsigned _offset)
 
 void CompilerUtils::storeInMemoryDynamic(Type const& _type, bool _padToWordBoundaries)
 {
-	if (_type.getCategory() == Type::Category::Array)
-		ArrayUtils(m_context).copyArrayToMemory(
-			dynamic_cast<ArrayType const&>(_type),
-			_padToWordBoundaries
-		);
+	if (auto ref = dynamic_cast<ReferenceType const*>(&_type))
+	{
+		solAssert(ref->location() == DataLocation::Memory, "");
+		storeInMemoryDynamic(IntegerType(256), _padToWordBoundaries);
+	}
 	else
 	{
 		unsigned numBytes = prepareMemoryStore(_type, _padToWordBoundaries);
 		if (numBytes > 0)
 		{
-			solAssert(_type.getSizeOnStack() == 1, "Memory store of types with stack size != 1 not implemented.");
+			solAssert(
+				_type.getSizeOnStack() == 1,
+				"Memory store of types with stack size != 1 not implemented."
+			);
 			m_context << eth::Instruction::DUP2 << eth::Instruction::MSTORE;
 			m_context << u256(numBytes) << eth::Instruction::ADD;
 		}
@@ -164,7 +174,10 @@ void CompilerUtils::encodeToMemory(
 				type = _givenTypes[i]; // delay conversion
 			else
 				convertType(*_givenTypes[i], *targetType, true);
-			storeInMemoryDynamic(*type, _padToWordBoundaries);
+			if (auto arrayType = dynamic_cast<ArrayType const*>(type.get()))
+				ArrayUtils(m_context).copyArrayToMemory(*arrayType, _padToWordBoundaries);
+			else
+				storeInMemoryDynamic(*type, _padToWordBoundaries);
 		}
 		stackPos += _givenTypes[i]->getSizeOnStack();
 	}
@@ -207,7 +220,7 @@ void CompilerUtils::encodeToMemory(
 			m_context << eth::swapInstruction(arrayType.getSizeOnStack() + 1) << eth::Instruction::POP;
 			// stack: ... <end_of_mem''> <value...>
 			// copy data part
-			storeInMemoryDynamic(arrayType, true);
+			ArrayUtils(m_context).copyArrayToMemory(arrayType, _padToWordBoundaries);
 			// stack: ... <end_of_mem'''>
 
 			thisDynPointer++;
@@ -349,63 +362,64 @@ void CompilerUtils::convertType(Type const& _typeOnStack, Type const& _targetTyp
 			{
 				// stack: <source ref> (variably sized)
 				unsigned stackSize = typeOnStack.getSizeOnStack();
-				fetchFreeMemoryPointer();
-				moveIntoStack(stackSize);
-				// stack: <mem start> <source ref> (variably sized)
-				if (targetType.isDynamicallySized())
+				bool fromStorage = (typeOnStack.location() == DataLocation::Storage);
+				if (fromStorage)
 				{
-					bool fromStorage = (typeOnStack.location() == DataLocation::Storage);
-					// store length
-					if (fromStorage)
-					{
-						stackSize--;
-						// remove storage offset, as requested by ArrayUtils::retrieveLength
-						m_context << eth::Instruction::POP;
-					}
-					ArrayUtils(m_context).retrieveLength(typeOnStack);
-					// Stack: <mem start> <source ref> <length>
-					m_context << eth::dupInstruction(2 + stackSize) << eth::Instruction::MSTORE;
-					m_context << eth::dupInstruction(1 + stackSize) << u256(0x20);
-					m_context << eth::Instruction::ADD;
-					moveIntoStack(stackSize);
-					if (fromStorage)
-					{
-						m_context << u256(0);
-						stackSize++;
-					}
+					stackSize--;
+					// remove storage offset, as requested by ArrayUtils::retrieveLength
+					m_context << eth::Instruction::POP;
 				}
-				else
+				ArrayUtils(m_context).retrieveLength(typeOnStack);
+
+				// allocate memory
+				// stack: <source ref> (variably sized) <length>
+				m_context << eth::Instruction::DUP1;
+				ArrayUtils(m_context).convertLengthToSize(targetType, true);
+				// stack: <source ref> (variably sized) <length> <size>
+				if (targetType.isDynamicallySized())
+					m_context << u256(0x20) << eth::Instruction::ADD;
+				allocateMemory();
+				// stack: <source ref> (variably sized) <length> <mem start>
+				m_context << eth::Instruction::DUP1;
+				moveIntoStack(2 + stackSize);
+				if (targetType.isDynamicallySized())
 				{
-					m_context << eth::dupInstruction(1 + stackSize);
-					moveIntoStack(stackSize);
+					m_context << eth::Instruction::DUP2;
+					storeInMemoryDynamic(IntegerType(256));
 				}
-				// Stack: <mem start> <mem data start> <value>
-				// Store data part.
-				storeInMemoryDynamic(typeOnStack);
-				// Stack <mem start> <mem end>
-				storeFreeMemoryPointer();
-			}
-			else if (typeOnStack.location() == DataLocation::CallData)
-			{
-				// Stack: <offset> [<length>]
-				// length is present if dynamically sized
-				fetchFreeMemoryPointer();
-				moveIntoStack(typeOnStack.getSizeOnStack());
-				// stack: memptr calldataoffset [<length>]
-				if (typeOnStack.isDynamicallySized())
+				// stack: <mem start> <source ref> (variably sized) <length> <mem data pos>
+				if (targetType.getBaseType()->isValueType())
 				{
-					solAssert(targetType.isDynamicallySized(), "");
-					m_context << eth::Instruction::DUP3 << eth::Instruction::DUP2;
-					storeInMemoryDynamic(IntegerType(256));
-					moveIntoStack(typeOnStack.getSizeOnStack());
+					solAssert(typeOnStack.getBaseType()->isValueType(), "");
+					copyToStackTop(2 + stackSize, stackSize);
+					if (fromStorage)
+						m_context << u256(0); // add byte offset again
+					ArrayUtils(m_context).copyArrayToMemory(typeOnStack);
 				}
 				else
-					m_context << eth::Instruction::DUP2 << eth::Instruction::SWAP1;
-				// stack: mem_ptr mem_data_ptr calldataoffset [<length>]
-				storeInMemoryDynamic(typeOnStack);
-				storeFreeMemoryPointer();
+				{
+					m_context << u256(0) << eth::Instruction::SWAP1;
+					// stack: <mem start> <source ref> (variably sized) <length> <counter> <mem data pos>
+					auto repeat = m_context.newTag();
+					m_context << repeat;
+					m_context << eth::Instruction::DUP3 << eth::Instruction::DUP3;
+					m_context << eth::Instruction::LT << eth::Instruction::ISZERO;
+					auto loopEnd = m_context.appendConditionalJump();
+					copyToStackTop(3 + stackSize, stackSize);
+					copyToStackTop(2 + stackSize, 1);
+					ArrayUtils(m_context).accessIndex(typeOnStack, false);
+					convertType(*typeOnStack.getBaseType(), *targetType.getBaseType(), _cleanupNeeded);
+					storeInMemoryDynamic(*targetType.getBaseType(), true);
+					m_context << eth::Instruction::SWAP1 << u256(1) << eth::Instruction::ADD;
+					m_context << eth::Instruction::SWAP1;
+					m_context.appendJumpTo(repeat);
+					m_context << loopEnd;
+					m_context << eth::Instruction::POP;
+				}
+				// stack: <mem start> <source ref> (variably sized) <length> <mem data pos updated>
+				popStackSlots(2 + stackSize);
+				// Stack: <mem start>
 			}
-			// nothing to do for memory to memory
 			break;
 		}
 		default:
@@ -444,6 +458,57 @@ void CompilerUtils::convertType(Type const& _typeOnStack, Type const& _targetTyp
 	}
 }
 
+void CompilerUtils::pushZeroValue(const Type& _type)
+{
+	auto const* referenceType = dynamic_cast<ReferenceType const*>(&_type);
+	if (!referenceType || referenceType->location() == DataLocation::Storage)
+	{
+		for (size_t i = 0; i < _type.getSizeOnStack(); ++i)
+			m_context << u256(0);
+		return;
+	}
+	solAssert(referenceType->location() == DataLocation::Memory, "");
+
+	m_context << u256(max(32u, _type.getCalldataEncodedSize()));
+	allocateMemory();
+	m_context << eth::Instruction::DUP1;
+
+	if (auto structType = dynamic_cast<StructType const*>(&_type))
+		for (auto const& member: structType->getMembers())
+		{
+			pushZeroValue(*member.type);
+			storeInMemoryDynamic(*member.type);
+		}
+	else if (auto arrayType = dynamic_cast<ArrayType const*>(&_type))
+	{
+		if (arrayType->isDynamicallySized())
+		{
+			// zero length
+			m_context << u256(0);
+			storeInMemoryDynamic(IntegerType(256));
+		}
+		else if (arrayType->getLength() > 0)
+		{
+			m_context << arrayType->getLength() << eth::Instruction::SWAP1;
+			// stack: items_to_do memory_pos
+			auto repeat = m_context.newTag();
+			m_context << repeat;
+			pushZeroValue(*arrayType->getBaseType());
+			storeInMemoryDynamic(*arrayType->getBaseType());
+			m_context << eth::Instruction::SWAP1 << u256(1) << eth::Instruction::SWAP1;
+			m_context << eth::Instruction::SUB << eth::Instruction::SWAP1;
+			m_context << eth::Instruction::DUP2;
+			m_context.appendConditionalJumpTo(repeat);
+			m_context << eth::Instruction::SWAP1 << eth::Instruction::POP;
+		}
+	}
+	else
+		solAssert(false, "Requested initialisation for unknown type: " + _type.toString());
+
+	// remove the updated memory pointer
+	m_context << eth::Instruction::POP;
+}
+
 void CompilerUtils::moveToStackVariable(VariableDeclaration const& _variable)
 {
 	unsigned const stackPosition = m_context.baseToCurrentStackOffset(m_context.getBaseStackOffsetOfVariable(_variable));

libsolidity/CompilerUtils.h

@@ -41,6 +41,10 @@ public:
 	void fetchFreeMemoryPointer();
 	/// Stores the free memory pointer from the stack.
 	void storeFreeMemoryPointer();
+	/// Allocates a number of bytes in memory as given on the stack.
+	/// Stack pre: <size>
+	/// Stack post: <mem_start>
+	void allocateMemory();
 	/// Appends code that transforms memptr to (memptr - free_memptr) memptr
 	void toSizeAfterFreeMemoryPointer();
 
@@ -70,7 +74,8 @@ public:
 	/// @param _type type of the data on the stack
 	void storeInMemory(unsigned _offset);
 	/// Dynamic version of @see storeInMemory, expects the memory offset below the value on the stack
-	/// and also updates that. For arrays, only copies the data part.
+	/// and also updates that. For reference types, only copies the data pointer. Fails for
+	/// non-memory-references.
 	/// @param _padToWordBoundaries if true, adds zeros to pad to multiple of 32 bytes. Array elements
 	/// are always padded (except for byte arrays), regardless of this parameter.
 	/// Stack pre: memory_offset value...
@@ -107,6 +112,10 @@ public:
 	/// necessary.
 	void convertType(Type const& _typeOnStack, Type const& _targetType, bool _cleanupNeeded = false);
 
+	/// Creates a zero-value for the given type and puts it onto the stack. This might allocate
+	/// memory for memory references.
+	void pushZeroValue(Type const& _type);
+
 	/// Moves the value that is at the top of the stack to a stack variable.
 	void moveToStackVariable(VariableDeclaration const& _variable);
 	/// Copies an item that occupies @a _itemSize stack slots from a stack depth of @a _stackDepth

libsolidity/ExpressionCompiler.cpp

@@ -56,62 +56,6 @@ void ExpressionCompiler::appendStateVariableInitialization(VariableDeclaration c
 	StorageItem(m_context, _varDecl).storeValue(*_varDecl.getType(), _varDecl.getLocation(), true);
 }
 
-void ExpressionCompiler::appendStackVariableInitialisation(Type const& _type, bool _toMemory)
-{
-	CompilerUtils utils(m_context);
-	auto const* referenceType = dynamic_cast<ReferenceType const*>(&_type);
-	if (!referenceType || referenceType->location() == DataLocation::Storage)
-	{
-		for (size_t i = 0; i < _type.getSizeOnStack(); ++i)
-			m_context << u256(0);
-		if (_toMemory)
-			utils.storeInMemoryDynamic(_type);
-		return;
-	}
-	solAssert(referenceType->location() == DataLocation::Memory, "");
-	if (!_toMemory)
-	{
-		// allocate memory
-		utils.fetchFreeMemoryPointer();
-		m_context << eth::Instruction::DUP1 << u256(max(32u, _type.getCalldataEncodedSize()));
-		m_context << eth::Instruction::ADD;
-		utils.storeFreeMemoryPointer();
-		m_context << eth::Instruction::DUP1;
-	}
-
-	if (auto structType = dynamic_cast<StructType const*>(&_type))
-		for (auto const& member: structType->getMembers())
-			appendStackVariableInitialisation(*member.type, true);
-	else if (auto arrayType = dynamic_cast<ArrayType const*>(&_type))
-	{
-		if (arrayType->isDynamicallySized())
-		{
-			// zero length
-			m_context << u256(0);
-			CompilerUtils(m_context).storeInMemoryDynamic(IntegerType(256));
-		}
-		else if (arrayType->getLength() > 0)
-		{
-			m_context << arrayType->getLength() << eth::Instruction::SWAP1;
-			// stack: items_to_do memory_pos
-			auto repeat = m_context.newTag();
-			m_context << repeat;
-			appendStackVariableInitialisation(*arrayType->getBaseType(), true);
-			m_context << eth::Instruction::SWAP1 << u256(1) << eth::Instruction::SWAP1;
-			m_context << eth::Instruction::SUB << eth::Instruction::SWAP1;
-			m_context << eth::Instruction::DUP2;
-			m_context.appendConditionalJumpTo(repeat);
-			m_context << eth::Instruction::SWAP1 << eth::Instruction::POP;
-		}
-	}
-	else
-		solAssert(false, "Requested initialisation for unknown type: " + _type.toString());
-
-	if (!_toMemory)
-		// remove the updated memory pointer
-		m_context << eth::Instruction::POP;
-}
-
 void ExpressionCompiler::appendStateVariableAccessor(VariableDeclaration const& _varDecl)
 {
 	CompilerContext::LocationSetter locationSetter(m_context, _varDecl);
@@ -211,6 +155,8 @@ bool ExpressionCompiler::visit(Assignment const& _assignment)
 	TypePointer type = _assignment.getRightHandSide().getType();
 	if (!_assignment.getType()->dataStoredIn(DataLocation::Storage))
 	{
+		//@todo we should delay conversion here if RHS is not in memory, LHS is a MemoryItem
+		// and not dynamically-sized.
 		utils().convertType(*type, *_assignment.getType());
 		type = _assignment.getType();
 	}
@@ -733,10 +679,24 @@ void ExpressionCompiler::endVisit(MemberAccess const& _memberAccess)
 	case Type::Category::Struct:
 	{
 		StructType const& type = dynamic_cast<StructType const&>(*_memberAccess.getExpression().getType());
-		m_context << eth::Instruction::POP; // structs always align to new slot
-		pair<u256, unsigned> const& offsets = type.getStorageOffsetsOfMember(member);
-		m_context << offsets.first << eth::Instruction::ADD << u256(offsets.second);
-		setLValueToStorageItem(_memberAccess);
+		switch (type.location())
+		{
+		case DataLocation::Storage:
+		{
+			m_context << eth::Instruction::POP; // structs always align to new slot
+			pair<u256, unsigned> const& offsets = type.getStorageOffsetsOfMember(member);
+			m_context << offsets.first << eth::Instruction::ADD << u256(offsets.second);
+			setLValueToStorageItem(_memberAccess);
+			break;
+		}
+		case DataLocation::Memory:
+		{
+			solAssert(false, "Member access for memory structs not yet implemented.");
+			break;
+		}
+		default:
+			solAssert(false, "Illegal data location for struct.");
+		}
 		break;
 	}
 	case Type::Category::Enum:
@@ -827,8 +787,9 @@ bool ExpressionCompiler::visit(IndexAccess const& _indexAccess)
 		_indexAccess.getIndexExpression()->accept(*this);
 		// stack layout: <base_ref> [<length>] <index>
 		ArrayUtils(m_context).accessIndex(arrayType);
-		if (arrayType.location() == DataLocation::Storage)
+		switch (arrayType.location())
 		{
+		case DataLocation::Storage:
 			if (arrayType.isByteArray())
 			{
 				solAssert(!arrayType.isString(), "Index access to string is not allowed.");
@@ -836,6 +797,21 @@ bool ExpressionCompiler::visit(IndexAccess const& _indexAccess)
 			}
 			else
 				setLValueToStorageItem(_indexAccess);
+			break;
+		case DataLocation::Memory:
+			setLValue<MemoryItem>(_indexAccess, *_indexAccess.getType(), !arrayType.isByteArray());
+			break;
+		case DataLocation::CallData:
+			//@todo if we implement this, the value in calldata has to be added to the base offset
+			solAssert(!arrayType.getBaseType()->isDynamicallySized(), "Nested arrays not yet implemented.");
+			if (arrayType.getBaseType()->isValueType())
+				CompilerUtils(m_context).loadFromMemoryDynamic(
+					*arrayType.getBaseType(),
+					true,
+					!arrayType.isByteArray(),
+					false
+				);
+			break;
 		}
 	}
 	else

libsolidity/ExpressionCompiler.h

@@ -64,13 +64,6 @@ public:
 	/// Appends code to set a state variable to its initial value/expression.
 	void appendStateVariableInitialization(VariableDeclaration const& _varDecl);
 
-	/// Appends code to initialise a local variable.
-	/// If @a _toMemory is false, leaves the value on the stack. For memory references, this
-	/// allocates new memory.
-	/// If @a _toMemory is true, directly stores the data in the memory pos on the stack and
-	/// updates it.
-	void appendStackVariableInitialisation(Type const& _type, bool _toMemory = false);
-
 	/// Appends code for a State Variable accessor function
 	void appendStateVariableAccessor(VariableDeclaration const& _varDecl);
 

libsolidity/LValue.cpp

@@ -69,19 +69,66 @@ void StackVariable::storeValue(Type const&, SourceLocation const& _location, boo
 
 void StackVariable::setToZero(SourceLocation const& _location, bool) const
 {
-	unsigned stackDiff = m_context.baseToCurrentStackOffset(m_baseStackOffset);
-	if (stackDiff > 16)
-		BOOST_THROW_EXCEPTION(
-			CompilerError() <<
-			errinfo_sourceLocation(_location) <<
-			errinfo_comment("Stack too deep, try removing local variables.")
-		);
-	solAssert(stackDiff >= m_size - 1, "");
-	for (unsigned i = 0; i < m_size; ++i)
-		m_context << u256(0) << eth::swapInstruction(stackDiff + 1 - i)
-			<< eth::Instruction::POP;
+	CompilerUtils(m_context).pushZeroValue(m_dataType);
+	storeValue(m_dataType, _location, true);
+}
+
+MemoryItem::MemoryItem(CompilerContext& _compilerContext, Type const& _type, bool _padded):
+	LValue(_compilerContext, _type),
+	m_padded(_padded)
+{
 }
 
+void MemoryItem::retrieveValue(SourceLocation const&, bool _remove) const
+{
+	if (m_dataType.isValueType())
+	{
+		if (!_remove)
+			m_context << eth::Instruction::DUP1;
+		CompilerUtils(m_context).loadFromMemoryDynamic(m_dataType, false, m_padded, false);
+	}
+	else
+		m_context << eth::Instruction::MLOAD;
+}
+
+void MemoryItem::storeValue(Type const& _sourceType, SourceLocation const&, bool _move) const
+{
+	CompilerUtils utils(m_context);
+	if (m_dataType.isValueType())
+	{
+		solAssert(_sourceType.isValueType(), "");
+		utils.moveIntoStack(_sourceType.getSizeOnStack());
+		utils.convertType(_sourceType, m_dataType, true);
+		if (!_move)
+		{
+			utils.moveToStackTop(m_dataType.getSizeOnStack());
+			utils.copyToStackTop(2, m_dataType.getSizeOnStack());
+		}
+		utils.storeInMemoryDynamic(m_dataType, m_padded);
+		m_context << eth::Instruction::POP;
+	}
+	else
+	{
+		solAssert(_sourceType == m_dataType, "Conversion not implemented for assignment to memory.");
+
+		solAssert(m_dataType.getSizeOnStack() == 1, "");
+		if (!_move)
+			m_context << eth::Instruction::DUP2 << eth::Instruction::SWAP1;
+		// stack: [value] value lvalue
+		// only store the reference
+		m_context << eth::Instruction::MSTORE;
+	}
+}
+
+void MemoryItem::setToZero(SourceLocation const&, bool _removeReference) const
+{
+	CompilerUtils utils(m_context);
+	if (!_removeReference)
+		m_context << eth::Instruction::DUP1;
+	utils.pushZeroValue(m_dataType);
+	utils.storeInMemoryDynamic(m_dataType, m_padded);
+	m_context << eth::Instruction::POP;
+}
 
 StorageItem::StorageItem(CompilerContext& _compilerContext, Declaration const& _declaration):
 	StorageItem(_compilerContext, *_declaration.getType())

libsolidity/LValue.h

@@ -97,6 +97,29 @@ private:
 	unsigned m_size;
 };
 
+/**
+ * Reference to some item in memory.
+ */
+class MemoryItem: public LValue
+{
+public:
+	MemoryItem(CompilerContext& _compilerContext, Type const& _type, bool _padded);
+	virtual unsigned sizeOnStack() const override { return 1; }
+	virtual void retrieveValue(SourceLocation const& _location, bool _remove = false) const override;
+	virtual void storeValue(
+		Type const& _sourceType,
+		SourceLocation const& _location = SourceLocation(),
+		bool _move = false
+	) const override;
+	virtual void setToZero(
+		SourceLocation const& _location = SourceLocation(),
+		bool _removeReference = true
+	) const override;
+private:
+	/// Special flag to deal with byte array elements.
+	bool m_padded = false;
+};
+
 /**
  * Reference to some item in storage. On the stack this is <storage key> <offset_inside_value>,
  * where 0 <= offset_inside_value < 32 and an offset of i means that the value is multiplied

libsolidity/Types.cpp

@@ -671,6 +671,26 @@ TypePointer ContractType::unaryOperatorResult(Token::Value _operator) const
 	return _operator == Token::Delete ? make_shared<VoidType>() : TypePointer();
 }
 
+TypePointer ReferenceType::unaryOperatorResult(Token::Value _operator) const
+{
+	if (_operator != Token::Delete)
+		return TypePointer();
+	// delete can be used on everything except calldata references or storage pointers
+	// (storage references are ok)
+	switch (location())
+	{
+	case DataLocation::CallData:
+		return TypePointer();
+	case DataLocation::Memory:
+		return make_shared<VoidType>();
+	case DataLocation::Storage:
+		return m_isPointer ? TypePointer() : make_shared<VoidType>();
+	default:
+		solAssert(false, "");
+	}
+	return TypePointer();
+}
+
 TypePointer ReferenceType::copyForLocationIfReference(DataLocation _location, TypePointer const& _type)
 {
 	if (auto type = dynamic_cast<ReferenceType const*>(_type.get()))
@@ -738,13 +758,6 @@ bool ArrayType::isImplicitlyConvertibleTo(const Type& _convertTo) const
 	}
 }
 
-TypePointer ArrayType::unaryOperatorResult(Token::Value _operator) const
-{
-	if (_operator == Token::Delete)
-		return make_shared<VoidType>();
-	return TypePointer();
-}
-
 bool ArrayType::operator==(Type const& _other) const
 {
 	if (_other.getCategory() != getCategory())
@@ -827,15 +840,16 @@ TypePointer ArrayType::externalType() const
 {
 	if (m_arrayKind != ArrayKind::Ordinary)
 		return this->copyForLocation(DataLocation::Memory, true);
-	if (!m_baseType->externalType())
+	TypePointer baseExt = m_baseType->externalType();
+	if (!baseExt)
 		return TypePointer();
 	if (m_baseType->getCategory() == Category::Array && m_baseType->isDynamicallySized())
 		return TypePointer();
 
 	if (isDynamicallySized())
-		return std::make_shared<ArrayType>(DataLocation::Memory, m_baseType->externalType());
+		return std::make_shared<ArrayType>(DataLocation::Memory, baseExt);
 	else
-		return std::make_shared<ArrayType>(DataLocation::Memory, m_baseType->externalType(), m_length);
+		return std::make_shared<ArrayType>(DataLocation::Memory, baseExt, m_length);
 }
 
 TypePointer ArrayType::copyForLocation(DataLocation _location, bool _isPointer) const
@@ -961,11 +975,6 @@ bool StructType::isImplicitlyConvertibleTo(const Type& _convertTo) const
 	return this->m_struct == convertTo.m_struct;
 }
 
-TypePointer StructType::unaryOperatorResult(Token::Value _operator) const
-{
-	return _operator == Token::Delete ? make_shared<VoidType>() : TypePointer();
-}
-
 bool StructType::operator==(Type const& _other) const
 {
 	if (_other.getCategory() != getCategory())
@@ -1268,15 +1277,17 @@ FunctionTypePointer FunctionType::externalFunctionType() const
 
 	for (auto type: m_parameterTypes)
 	{
-		if (!type->externalType())
+		if (auto ext = type->externalType())
+			paramTypes.push_back(ext);
+		else
 			return FunctionTypePointer();
-		paramTypes.push_back(type->externalType());
 	}
 	for (auto type: m_returnParameterTypes)
 	{
-		if (!type->externalType())
+		if (auto ext = type->externalType())
+			retParamTypes.push_back(ext);
+		else
 			return FunctionTypePointer();
-		retParamTypes.push_back(type->externalType());
 	}
 	return make_shared<FunctionType>(paramTypes, retParamTypes, m_parameterNames, m_returnParameterNames, m_location, m_arbitraryParameters);
 }

libsolidity/Types.h

@@ -179,6 +179,9 @@ public:
 	/// is not a simple big-endian encoding or the type cannot be stored in calldata.
 	/// If @a _padded then it is assumed that each element is padded to a multiple of 32 bytes.
 	virtual unsigned getCalldataEncodedSize(bool _padded) const { (void)_padded; return 0; }
+	/// @returns the size of this data type in bytes when stored in memory. For memory-reference
+	/// types, this is the size of the memory pointer.
+	virtual unsigned memoryHeadSize() const { return getCalldataEncodedSize(); }
 	/// Convenience version of @see getCalldataEncodedSize(bool)
 	unsigned getCalldataEncodedSize() const { return getCalldataEncodedSize(true); }
 	/// @returns true if the type is dynamically encoded in calldata
@@ -373,6 +376,9 @@ public:
 	explicit ReferenceType(DataLocation _location): m_location(_location) {}
 	DataLocation location() const { return m_location; }
 
+	virtual TypePointer unaryOperatorResult(Token::Value _operator) const override;
+	virtual unsigned memoryHeadSize() const override { return 32; }
+
 	/// @returns a copy of this type with location (recursively) changed to @a _location,
 	/// whereas isPointer is only shallowly changed - the deep copy is always a bound reference.
 	virtual TypePointer copyForLocation(DataLocation _location, bool _isPointer) const = 0;
@@ -439,11 +445,11 @@ public:
 	{}
 
 	virtual bool isImplicitlyConvertibleTo(Type const& _convertTo) const override;
-	virtual TypePointer unaryOperatorResult(Token::Value _operator) const override;
 	virtual bool operator==(const Type& _other) const override;
 	virtual unsigned getCalldataEncodedSize(bool _padded) const override;
 	virtual bool isDynamicallySized() const override { return m_hasDynamicLength; }
 	virtual u256 getStorageSize() const override;
+	virtual bool canLiveOutsideStorage() const override { return m_baseType->canLiveOutsideStorage(); }
 	virtual unsigned getSizeOnStack() const override;
 	virtual std::string toString(bool _short) const override;
 	virtual MemberList const& getMembers() const override
@@ -540,7 +546,6 @@ public:
 		//@todo only storage until we have non-storage structs
 		ReferenceType(DataLocation::Storage), m_struct(_struct) {}
 	virtual bool isImplicitlyConvertibleTo(const Type& _convertTo) const override;
-	virtual TypePointer unaryOperatorResult(Token::Value _operator) const override;
 	virtual bool operator==(Type const& _other) const override;
 	virtual unsigned getCalldataEncodedSize(bool _padded) const override;
 	virtual u256 getStorageSize() const override;

libweb3jsonrpc/CMakeLists.txt

@@ -27,7 +27,6 @@ target_link_libraries(${EXECUTABLE} ${JSON_RPC_CPP_SERVER_LIBRARIES})
 target_link_libraries(${EXECUTABLE} ${MHD_LIBRARIES})
 
 target_link_libraries(${EXECUTABLE} webthree)
-target_link_libraries(${EXECUTABLE} secp256k1)
 
 if (SOLIDITY)
 	target_link_libraries(${EXECUTABLE} solidity)

libwebthree/CMakeLists.txt

@@ -28,7 +28,6 @@ target_link_libraries(${EXECUTABLE} whisper)
 target_link_libraries(${EXECUTABLE} p2p)
 target_link_libraries(${EXECUTABLE} ethcore)
 target_link_libraries(${EXECUTABLE} devcrypto)
-target_link_libraries(${EXECUTABLE} secp256k1)
 
 install( TARGETS ${EXECUTABLE} RUNTIME DESTINATION bin ARCHIVE DESTINATION lib LIBRARY DESTINATION lib )
 install( FILES ${HEADERS} DESTINATION include/${EXECUTABLE} )

libwhisper/BloomFilter.h

@@ -91,7 +91,7 @@ bool TopicBloomFilterBase<N>::isBitSet(FixedHash<N> const& _h, unsigned _index)
 	return (_h[iByte] & c_powerOfTwoBitMmask[iBit]) != 0;
 }
 
-using TopicBloomFilter = TopicBloomFilterBase<TopicBloomFilterSize>;
+using TopicBloomFilter = TopicBloomFilterBase<c_topicBloomFilterSize>;
 
 }
 }

libwhisper/CMakeLists.txt

@@ -25,7 +25,6 @@ target_link_libraries(${EXECUTABLE} ethcore)
 target_link_libraries(${EXECUTABLE} devcrypto)
 target_link_libraries(${EXECUTABLE} devcore)
 target_link_libraries(${EXECUTABLE} p2p)
-target_link_libraries(${EXECUTABLE} secp256k1)
 
 install( TARGETS ${EXECUTABLE} RUNTIME DESTINATION bin ARCHIVE DESTINATION lib LIBRARY DESTINATION lib )
 install( FILES ${HEADERS} DESTINATION include/${EXECUTABLE} )

libwhisper/Common.cpp

@@ -95,12 +95,12 @@ TopicFilter::TopicFilter(RLP const& _r)
 	}
 }
 
-FixedHash<TopicBloomFilter::size> TopicFilter::exportBloom() const
+TopicBloomFilterHash TopicFilter::exportBloom() const
 {
-	FixedHash<TopicBloomFilter::size> ret;
+	TopicBloomFilterHash ret;
 	for (TopicMask const& t: m_topicMasks)
 		for (auto const& i: t)
-			ret |= i.first.template bloomPart<TopicBloomFilter::BitsPerBloom, TopicBloomFilter::size>();		
+			ret |= i.first.template bloomPart<TopicBloomFilter::BitsPerBloom, c_topicBloomFilterSize>();
 		
 	return ret;
 }

libwhisper/Common.h

@@ -54,12 +54,12 @@ enum WhisperPacket
 {
 	StatusPacket = 0,
 	MessagesPacket,
-	AddFilterPacket,
-	RemoveFilterPacket,
+	TopicFilterPacket,
 	PacketCount
 };
 
-enum { TopicBloomFilterSize = 8 };
+static const int c_topicBloomFilterSize = 64;
+static const int c_whisperProtocolVersion = 3;
 
 using AbridgedTopic = FixedHash<4>;
 using Topic = h256;
@@ -67,6 +67,8 @@ using Topic = h256;
 using AbridgedTopics = std::vector<AbridgedTopic>;
 using Topics = h256s;
 
+using TopicBloomFilterHash = FixedHash<c_topicBloomFilterSize>;
+
 AbridgedTopic abridge(Topic const& _topic);
 AbridgedTopics abridge(Topics const& _topics);
 
@@ -107,7 +109,7 @@ public:
 	void streamRLP(RLPStream& _s) const { _s << m_topicMasks; }
 	h256 sha3() const;
 	bool matches(Envelope const& _m) const;
-	FixedHash<TopicBloomFilterSize> exportBloom() const;
+	TopicBloomFilterHash exportBloom() const;
 
 private:
 	TopicMasks m_topicMasks;

libwhisper/Interface.h

@@ -67,7 +67,6 @@ public:
 
 	virtual Topics const& fullTopics(unsigned _id) const = 0;
 	virtual unsigned installWatch(Topics const& _filter) = 0;
-	virtual unsigned installWatchOnId(h256 _filterId) = 0;
 	virtual void uninstallWatch(unsigned _watchId) = 0;
 	virtual h256s peekWatch(unsigned _watchId) const = 0;
 	virtual h256s checkWatch(unsigned _watchId) = 0;

libwhisper/WhisperHost.cpp

@@ -66,12 +66,13 @@ void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p)
 		m_expiryQueue.insert(make_pair(_m.expiry(), h));
 	}
 
-//	if (_p)
+	DEV_GUARDED(m_filterLock)
 	{
-		Guard l(m_filterLock);
 		for (auto const& f: m_filters)
 			if (f.second.filter.matches(_m))
-				noteChanged(h, f.first);
+				for (auto& i: m_watches)
+					if (i.second.id == f.first)
+						i.second.changes.push_back(h);
 	}
 
 	// TODO p2p: capability-based rating
@@ -85,36 +86,28 @@ void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p)
 	}
 }
 
-void WhisperHost::noteChanged(h256 _messageHash, h256 _filter)
-{
-	for (auto& i: m_watches)
-		if (i.second.id == _filter)
-		{
-			cwatshh << "!!!" << i.first << i.second.id;
-			i.second.changes.push_back(_messageHash);
-		}
-}
-
-unsigned WhisperHost::installWatchOnId(h256 _h)
-{
-	auto ret = m_watches.size() ? m_watches.rbegin()->first + 1 : 0;
-	m_watches[ret] = ClientWatch(_h);
-	cwatshh << "+++" << ret << _h;
-	return ret;
-}
-
 unsigned WhisperHost::installWatch(shh::Topics const& _t)
 {
-	Guard l(m_filterLock);
-
 	InstalledFilter f(_t);
 	h256 h = f.filter.sha3();
+	unsigned ret = 0;
 
-	if (!m_filters.count(h))
-		m_filters.insert(make_pair(h, f));
+	DEV_GUARDED(m_filterLock)
+	{
+		auto it = m_filters.find(h);
+		if (m_filters.end() == it)
+			m_filters.insert(make_pair(h, f));
+		else
+			it->second.refCount++;
 
-	m_bloom.addRaw(f.filter.exportBloom());
-	return installWatchOnId(h);
+		m_bloom.addRaw(f.filter.exportBloom());
+		ret = m_watches.size() ? m_watches.rbegin()->first + 1 : 0;
+		m_watches[ret] = ClientWatch(h);
+		cwatshh << "+++" << ret << h;
+	}
+
+	noteAdvertiseTopicsOfInterest();
+	return ret;
 }
 
 h256s WhisperHost::watchMessages(unsigned _watchId)
@@ -142,21 +135,25 @@ void WhisperHost::uninstallWatch(unsigned _i)
 {
 	cwatshh << "XXX" << _i;
 
-	Guard l(m_filterLock);
+	DEV_GUARDED(m_filterLock)
+	{
+		auto it = m_watches.find(_i);
+		if (it == m_watches.end())
+			return;
 
-	auto it = m_watches.find(_i);
-	if (it == m_watches.end())
-		return;
-	auto id = it->second.id;
-	m_watches.erase(it);
+		auto id = it->second.id;
+		m_watches.erase(it);
 
-	auto fit = m_filters.find(id);
-	if (fit != m_filters.end())
-	{
-		m_bloom.removeRaw(fit->second.filter.exportBloom());
-		if (!--fit->second.refCount)
-			m_filters.erase(fit);
+		auto fit = m_filters.find(id);
+		if (fit != m_filters.end())
+		{
+			m_bloom.removeRaw(fit->second.filter.exportBloom());
+			if (!--fit->second.refCount)
+				m_filters.erase(fit);
+		}
 	}
+
+	noteAdvertiseTopicsOfInterest();
 }
 
 void WhisperHost::doWork()
@@ -175,3 +172,9 @@ void WhisperHost::cleanup()
 	for (auto it = m_expiryQueue.begin(); it != m_expiryQueue.end() && it->first <= now; it = m_expiryQueue.erase(it))
 		m_messages.erase(it->second);
 }
+
+void WhisperHost::noteAdvertiseTopicsOfInterest()
+{
+	for (auto i: peerSessions())
+		i.first->cap<WhisperPeer>().get()->noteAdvertiseTopicsOfInterest();
+}

libwhisper/WhisperHost.h

@@ -50,36 +50,31 @@ class WhisperHost: public HostCapability<WhisperPeer>, public Interface, public
 public:
 	WhisperHost();
 	virtual ~WhisperHost();
-
-	unsigned protocolVersion() const { return 2; }
+	unsigned protocolVersion() const { return c_whisperProtocolVersion; }
+	/// remove old messages
+	void cleanup(); 
+	std::map<h256, Envelope> all() const { dev::ReadGuard l(x_messages); return m_messages; }
+	TopicBloomFilterHash bloom() const { dev::Guard l(m_filterLock); return m_bloom; }
 
 	virtual void inject(Envelope const& _e, WhisperPeer* _from = nullptr) override;
-
 	virtual Topics const& fullTopics(unsigned _id) const override { try { return m_filters.at(m_watches.at(_id).id).full; } catch (...) { return EmptyTopics; } }
 	virtual unsigned installWatch(Topics const& _filter) override;
-	virtual unsigned installWatchOnId(h256 _filterId) override;
 	virtual void uninstallWatch(unsigned _watchId) override;
 	virtual h256s peekWatch(unsigned _watchId) const override { dev::Guard l(m_filterLock); try { return m_watches.at(_watchId).changes; } catch (...) { return h256s(); } }
 	virtual h256s checkWatch(unsigned _watchId) override { cleanup(); dev::Guard l(m_filterLock); h256s ret; try { ret = m_watches.at(_watchId).changes; m_watches.at(_watchId).changes.clear(); } catch (...) {} return ret; }
-	virtual h256s watchMessages(unsigned _watchId) override; /// returns IDs of messages, which match specific watch criteria
-
+	/// returns IDs of messages, which match specific watch criteria
+	virtual h256s watchMessages(unsigned _watchId) override; 
 	virtual Envelope envelope(h256 _m) const override { try { dev::ReadGuard l(x_messages); return m_messages.at(_m); } catch (...) { return Envelope(); } }
 
-	std::map<h256, Envelope> all() const { ReadGuard l(x_messages); return m_messages; }
-
-	void cleanup();
-
 protected:
 	virtual void doWork() override;
+	void noteAdvertiseTopicsOfInterest();
 
 private:
 	virtual void onStarting() override { startWorking(); }
 	virtual void onStopping() override { stopWorking(); }
-
 	void streamMessage(h256 _m, RLPStream& _s) const;
 
-	void noteChanged(h256 _messageHash, h256 _filter);
-
 	mutable dev::SharedMutex x_messages;
 	std::map<h256, Envelope> m_messages;
 	std::multimap<unsigned, h256> m_expiryQueue;

libwhisper/WhisperPeer.cpp

@@ -19,11 +19,10 @@
  * @date 2014
  */
 
-#include "WhisperPeer.h"
-
 #include <libdevcore/Log.h>
 #include <libp2p/All.h>
 #include "WhisperHost.h"
+
 using namespace std;
 using namespace dev;
 using namespace dev::p2p;
@@ -33,6 +32,7 @@ WhisperPeer::WhisperPeer(std::shared_ptr<Session> _s, HostCapabilityFace* _h, un
 {
 	RLPStream s;
 	sealAndSend(prep(s, StatusPacket, 1) << version());
+	noteAdvertiseTopicsOfInterest();
 }
 
 WhisperPeer::~WhisperPeer()
@@ -58,10 +58,15 @@ bool WhisperPeer::interpret(unsigned _id, RLP const& _r)
 			disable("Invalid protocol version.");
 
 		for (auto const& m: host()->all())
+		{
+			Guard l(x_unseen);
 			m_unseen.insert(make_pair(0, m.first));
+		}
 
 		if (session()->id() < host()->host()->id())
 			sendMessages();
+
+		noteAdvertiseTopicsOfInterest();
 		break;
 	}
 	case MessagesPacket:
@@ -70,6 +75,11 @@ bool WhisperPeer::interpret(unsigned _id, RLP const& _r)
 			host()->inject(Envelope(i), this);
 		break;
 	}
+	case TopicFilterPacket:
+	{
+		setBloom((TopicBloomFilterHash)_r[0]);
+		break;
+	}
 	default:
 		return false;
 	}
@@ -78,6 +88,9 @@ bool WhisperPeer::interpret(unsigned _id, RLP const& _r)
 
 void WhisperPeer::sendMessages()
 {
+	if (m_advertiseTopicsOfInterest)
+		sendTopicsOfInterest(host()->bloom());
+
 	RLPStream amalg;
 	unsigned msgCount = 0;
 	{
@@ -104,3 +117,15 @@ void WhisperPeer::noteNewMessage(h256 _h, Envelope const& _m)
 	Guard l(x_unseen);
 	m_unseen.insert(make_pair(rating(_m), _h));
 }
+
+void WhisperPeer::sendTopicsOfInterest(TopicBloomFilterHash const& _bloom)
+{
+	DEV_GUARDED(x_advertiseTopicsOfInterest)
+		m_advertiseTopicsOfInterest = false;
+
+	RLPStream s;
+	prep(s, TopicFilterPacket, 1);
+	s << _bloom;
+	sealAndSend(s);
+}
+

libwhisper/WhisperPeer.h

@@ -44,8 +44,6 @@ using p2p::HostCapability;
 using p2p::Capability;
 using p2p::CapDesc;
 
-/**
- */
 class WhisperPeer: public Capability
 {
 	friend class WhisperHost;
@@ -53,25 +51,30 @@ class WhisperPeer: public Capability
 public:
 	WhisperPeer(std::shared_ptr<Session> _s, HostCapabilityFace* _h, unsigned _i, CapDesc const& _cap);
 	virtual ~WhisperPeer();
-
+	WhisperHost* host() const;
 	static std::string name() { return "shh"; }
-	static u256 version() { return 2; }
+	static u256 version() { return c_whisperProtocolVersion; }
 	static unsigned messageCount() { return PacketCount; }
-
-	WhisperHost* host() const;
+	TopicBloomFilterHash bloom() const { dev::Guard g(x_bloom); return m_bloom; }
+	void sendTopicsOfInterest(TopicBloomFilterHash const& _bloom); ///< sends our bloom filter to remote peer
+	void noteAdvertiseTopicsOfInterest() { dev::Guard g(x_advertiseTopicsOfInterest); m_advertiseTopicsOfInterest = true; }
 
 private:
 	virtual bool interpret(unsigned _id, RLP const&) override;
-
 	void sendMessages();
-
 	unsigned rating(Envelope const&) const { return 0; }	// TODO
 	void noteNewMessage(h256 _h, Envelope const& _m);
+	void setBloom(TopicBloomFilterHash const& _b) { dev::Guard g(x_bloom); m_bloom = _b; }
 
 	mutable dev::Mutex x_unseen;
 	std::multimap<unsigned, h256> m_unseen;	///< Rated according to what they want.
-
 	std::chrono::system_clock::time_point m_timer = std::chrono::system_clock::now();
+
+	mutable dev::Mutex x_bloom;
+	TopicBloomFilterHash m_bloom; ///< Peer's topics of interest
+
+	mutable dev::Mutex x_advertiseTopicsOfInterest;
+	bool m_advertiseTopicsOfInterest;
 };
 
 }

mix/DebuggingStateWrapper.cpp

@@ -38,13 +38,17 @@ using namespace dev::mix;
 
 namespace
 {
-	static QVariantList memDumpToList(bytes const& _bytes, unsigned _width)
+	static QVariantList memDumpToList(bytes const& _bytes, unsigned _width, bool _includeAddress = false)
 	{
 		QVariantList dumpList;
 		for (unsigned i = 0; i < _bytes.size(); i += _width)
 		{
 			std::stringstream ret;
-
+			if (_includeAddress)
+			{
+				ret << std::setfill('0') << std::setw(6) << std::hex << i << " ";
+				ret << "   ";
+			}
 			for (unsigned j = i; j < i + _width; ++j)
 				if (j < _bytes.size())
 					if (_bytes[j] >= 32 && _bytes[j] < 127)
@@ -137,7 +141,7 @@ QStringList QMachineState::debugStorage()
 
 QVariantList QMachineState::debugMemory()
 {
-	return memDumpToList(m_state.memory, 16);
+	return memDumpToList(m_state.memory, 16, true);
 }
 
 QCallData* QMachineState::getDebugCallData(QObject* _owner, bytes const& _data)

mix/FileIo.cpp

@@ -180,7 +180,7 @@ QStringList FileIo::makePackage(QString const& _deploymentFolder)
 	dev::h256 dappHash = dev::sha3(dapp);
 	//encrypt
 	KeyPair key(dappHash);
-	Secp256k1 enc;
+	Secp256k1PP enc;
 	enc.encrypt(key.pub(), dapp);
 
 	QUrl url(_deploymentFolder + "package.dapp");

neth/CMakeLists.txt

@@ -19,7 +19,6 @@ endif()
 
 target_link_libraries(${EXECUTABLE} webthree)
 target_link_libraries(${EXECUTABLE} ethereum)
-target_link_libraries(${EXECUTABLE} secp256k1)
 target_link_libraries(${EXECUTABLE} ncurses)
 target_link_libraries(${EXECUTABLE} form)
 

test/CMakeLists.txt

@@ -74,7 +74,9 @@ target_link_libraries(testeth ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
 target_link_libraries(testeth ${CURL_LIBRARIES})
 target_link_libraries(testeth ethereum)
 target_link_libraries(testeth ethcore)
-target_link_libraries(testeth secp256k1)
+if (NOT WIN32)
+	target_link_libraries(testeth secp256k1)
+endif()
 
 if (JSCONSOLE)
 	target_link_libraries(testeth jsengine)

test/libdevcrypto/crypto.cpp

@@ -22,7 +22,9 @@
  */
 
 #include <random>
+#if ETH_HAVE_SECP256K1
 #include <secp256k1/secp256k1.h>
+#endif
 #include <libdevcore/Common.h>
 #include <libdevcore/RLP.h>
 #include <libdevcore/Log.h>
@@ -43,7 +45,7 @@ BOOST_GLOBAL_FIXTURE( MoveNonceToTempDir )
 
 BOOST_AUTO_TEST_SUITE(devcrypto)
 
-static Secp256k1 s_secp256k1;
+static Secp256k1PP s_secp256k1;
 static CryptoPP::AutoSeededRandomPool s_rng;
 static CryptoPP::OID s_curveOID(CryptoPP::ASN1::secp256k1());
 static CryptoPP::DL_GroupParameters_EC<CryptoPP::ECP> s_params(s_curveOID);
@@ -64,6 +66,18 @@ BOOST_AUTO_TEST_CASE(emptySHA3Types)
 	BOOST_REQUIRE_EQUAL(emptyListSHA3, EmptyListSHA3);
 }
 
+#if ETH_HAVE_SECP256K1
+BOOST_AUTO_TEST_CASE(secp256k1lib)
+{
+	secp256k1Init();
+	KeyPair k = KeyPair::create();
+	BOOST_REQUIRE(!!k.sec());
+	BOOST_REQUIRE(!!k.pub());
+	Public test = toPublic(k.sec());
+	BOOST_REQUIRE(k.pub() == test);
+}
+#endif
+
 BOOST_AUTO_TEST_CASE(cryptopp_patch)
 {
 	KeyPair k = KeyPair::create();
@@ -101,7 +115,9 @@ BOOST_AUTO_TEST_CASE(common_encrypt_decrypt)
 
 BOOST_AUTO_TEST_CASE(cryptopp_cryptopp_secp256k1libport)
 {
+#if ETH_HAVE_SECP256K1
 	secp256k1_start();
+#endif
 	
 	// base secret
 	Secret secret(sha3("privacy"));
@@ -150,12 +166,15 @@ BOOST_AUTO_TEST_CASE(cryptopp_cryptopp_secp256k1libport)
 		byte dersig[72];
 		size_t cssz = DSAConvertSignatureFormat(dersig, 72, DSA_DER, sig.data(), 64, DSA_P1363);
 		BOOST_CHECK(cssz <= 72);
+#if ETH_HAVE_SECP256K1
 		BOOST_REQUIRE(1 == secp256k1_ecdsa_verify(he.data(), sizeof(he), dersig, cssz, encpub, 65));
+#endif
 	}
 }
 
 BOOST_AUTO_TEST_CASE(cryptopp_ecdsa_sipaseckp256k1)
 {
+#ifdef CRYPTOPPNOTBROKEN
 	secp256k1_start();
 	
 	// cryptopp integer encoding
@@ -237,6 +256,7 @@ BOOST_AUTO_TEST_CASE(cryptopp_ecdsa_sipaseckp256k1)
 		BOOST_CHECK(cssz <= 72);
 		BOOST_REQUIRE(1 == secp256k1_ecdsa_verify(hm.data(), sizeof(hm), dersig, cssz, encpub, 65));
 	}
+#endif
 }
 
 BOOST_AUTO_TEST_CASE(sha3_norestart)
@@ -747,7 +767,6 @@ BOOST_AUTO_TEST_CASE(cryptopp_aes128_cbc)
 BOOST_AUTO_TEST_CASE(eth_keypairs)
 {
 	cnote << "Testing Crypto...";
-	secp256k1_start();
 
 	KeyPair p(Secret(fromHex("3ecb44df2159c26e0f995712d4f39b6f6e499b40749b1cf1246c37f9516cb6a4")));
 	BOOST_REQUIRE(p.pub() == Public(fromHex("97466f2b32bc3bb76d4741ae51cd1d8578b48d3f1e68da206d47321aec267ce78549b514e4453d74ef11b0cd5e4e4c364effddac8b51bcfc8de80682f952896f")));
@@ -771,7 +790,6 @@ BOOST_AUTO_TEST_CASE(eth_keypairs)
 int cryptoTest()
 {
 	cnote << "Testing Crypto...";
-	secp256k1_start();
 
 	KeyPair p(Secret(fromHex("3ecb44df2159c26e0f995712d4f39b6f6e499b40749b1cf1246c37f9516cb6a4")));
 	BOOST_REQUIRE(p.pub() == Public(fromHex("97466f2b32bc3bb76d4741ae51cd1d8578b48d3f1e68da206d47321aec267ce78549b514e4453d74ef11b0cd5e4e4c364effddac8b51bcfc8de80682f952896f")));

test/libethereum/stateOriginal.cpp

@@ -22,7 +22,6 @@
 
 #include <boost/test/unit_test.hpp>
 #include <boost/filesystem/operations.hpp>
-#include <secp256k1/secp256k1.h>
 #include <libethereum/CanonBlockChain.h>
 #include <libethereum/State.h>
 #include <libethcore/Farm.h>

test/libp2p/net.cpp

@@ -314,23 +314,23 @@ BOOST_AUTO_TEST_CASE(kademlia)
 	node.nodeTable->discover(); // ideally, joining with empty node table logs warning we can check for
 	node.setup();
 	node.populate();
-	clog << "NodeTable:\n" << *node.nodeTable.get() << endl;
+//	clog << "NodeTable:\n" << *node.nodeTable.get() << endl;
 
 	node.populateAll();
-	clog << "NodeTable:\n" << *node.nodeTable.get() << endl;
+//	clog << "NodeTable:\n" << *node.nodeTable.get() << endl;
 
 	auto nodes = node.nodeTable->nodes();
 	nodes.sort();
 
 	node.nodeTable->reset();
-	clog << "NodeTable:\n" << *node.nodeTable.get() << endl;
+//	clog << "NodeTable:\n" << *node.nodeTable.get() << endl;
 
 	node.populate(1);
-	clog << "NodeTable:\n" << *node.nodeTable.get() << endl;
+//	clog << "NodeTable:\n" << *node.nodeTable.get() << endl;
 
 	node.nodeTable->discover();
 	this_thread::sleep_for(chrono::milliseconds(2000));
-	clog << "NodeTable:\n" << *node.nodeTable.get() << endl;
+//	clog << "NodeTable:\n" << *node.nodeTable.get() << endl;
 
 	BOOST_REQUIRE_EQUAL(node.nodeTable->count(), 8);
 

test/libp2p/rlpx.cpp

@@ -39,7 +39,7 @@ using namespace CryptoPP;
 
 BOOST_AUTO_TEST_SUITE(rlpx)
 
-static Secp256k1 s_secp256k1;
+static Secp256k1PP s_secp256k1;
 static CryptoPP::AutoSeededRandomPool s_rng;
 static CryptoPP::OID s_curveOID(CryptoPP::ASN1::secp256k1());
 static CryptoPP::DL_GroupParameters_EC<CryptoPP::ECP> s_params(s_curveOID);

test/libsolidity/SolidityEndToEndTest.cpp

@@ -4493,7 +4493,7 @@ BOOST_AUTO_TEST_CASE(bytes_in_constructors_packer)
 			}
 		}
 		contract Main is Base {
-			function Main(bytes s, uint x) Base(x, s){}//f(s)) {}
+			function Main(bytes s, uint x) Base(x, f(s)) {}
 			function f(bytes s) returns (bytes) {
 				return s;
 			}
@@ -4517,6 +4517,45 @@ BOOST_AUTO_TEST_CASE(bytes_in_constructors_packer)
 	);
 }
 
+BOOST_AUTO_TEST_CASE(arrays_in_constructors)
+{
+	char const* sourceCode = R"(
+		contract Base {
+			uint public m_x;
+			address[] m_s;
+			function Base(uint x, address[] s) {
+				m_x = x;
+				m_s = s;
+			}
+			function part(uint i) returns (address) {
+				return m_s[i];
+			}
+		}
+		contract Main is Base {
+			function Main(address[] s, uint x) Base(x, f(s)) {}
+			function f(address[] s) returns (address[]) {
+				return s;
+			}
+		}
+		contract Creator {
+			function f(uint x, address[] s) returns (uint r, address ch) {
+				var c = new Main(s, x);
+				r = c.m_x();
+				ch = c.part(x);
+			}
+		}
+	)";
+	compileAndRun(sourceCode, 0, "Creator");
+	vector<u256> s1{1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
+	bytes dyn1 = encodeArgs(u256(s1.size()), s1);
+	u256 x = 7;
+	bytes args1 = encodeArgs(x, u256(0x40)) + dyn1;
+	BOOST_REQUIRE(
+		callContractFunction("f(uint256,address[])", asString(args1)) ==
+		encodeArgs(x, s1[unsigned(x)])
+	);
+}
+
 BOOST_AUTO_TEST_CASE(arrays_from_and_to_storage)
 {
 	char const* sourceCode = R"(
@@ -4691,6 +4730,82 @@ BOOST_AUTO_TEST_CASE(memory_types_initialisation)
 	BOOST_CHECK(callContractFunction("nestedStat()") == encodeArgs(vector<u256>(3 * 7)));
 }
 
+BOOST_AUTO_TEST_CASE(memory_arrays_delete)
+{
+	char const* sourceCode = R"(
+		contract Test {
+			function del() returns (uint24[3][4]) {
+				uint24[3][4] memory x;
+				for (uint24 i = 0; i < x.length; i ++)
+					for (uint24 j = 0; j < x[i].length; j ++)
+						x[i][j] = i * 0x10 + j;
+				delete x[1];
+				delete x[3][2];
+				return x;
+			}
+		}
+	)";
+	compileAndRun(sourceCode, 0, "Test");
+
+	vector<u256> data(3 * 4);
+	for (unsigned i = 0; i < 4; i++)
+		for (unsigned j = 0; j < 3; j++)
+		{
+			u256 v = 0;
+			if (!(i == 1 || (i == 3 && j == 2)))
+				v = i * 0x10 + j;
+			data[i * 3 + j] = v;
+		}
+	BOOST_CHECK(callContractFunction("del()") == encodeArgs(data));
+}
+
+BOOST_AUTO_TEST_CASE(memory_arrays_index_access_write)
+{
+	char const* sourceCode = R"(
+		contract Test {
+			function set(uint24[3][4] x) {
+				x[2][2] = 1;
+				x[3][2] = 7;
+			}
+			function f() returns (uint24[3][4]){
+				uint24[3][4] memory data;
+				set(data);
+				return data;
+			}
+		}
+	)";
+	compileAndRun(sourceCode, 0, "Test");
+
+	vector<u256> data(3 * 4);
+	data[3 * 2 + 2] = 1;
+	data[3 * 3 + 2] = 7;
+	BOOST_CHECK(callContractFunction("f()") == encodeArgs(data));
+}
+
+BOOST_AUTO_TEST_CASE(memory_arrays_dynamic_index_access_write)
+{
+	char const* sourceCode = R"(
+		contract Test {
+			uint24[3][][4] data;
+			function set(uint24[3][][4] x) internal returns (uint24[3][][4]) {
+				x[1][2][2] = 1;
+				x[1][3][2] = 7;
+				return x;
+			}
+			function f() returns (uint24[3][]) {
+				data[1].length = 4;
+				return set(data)[1];
+			}
+		}
+	)";
+	compileAndRun(sourceCode, 0, "Test");
+
+	vector<u256> data(3 * 4);
+	data[3 * 2 + 2] = 1;
+	data[3 * 3 + 2] = 7;
+	BOOST_CHECK(callContractFunction("f()") == encodeArgs(u256(0x20), u256(4), data));
+}
+
 BOOST_AUTO_TEST_SUITE_END()
 
 }

test/libsolidity/SolidityNameAndTypeResolution.cpp

@@ -403,6 +403,23 @@ BOOST_AUTO_TEST_CASE(abstract_contract)
 	BOOST_CHECK(derived->getDefinedFunctions()[0]->isFullyImplemented());
 }
 
+BOOST_AUTO_TEST_CASE(abstract_contract_with_overload)
+{
+	ASTPointer<SourceUnit> sourceUnit;
+	char const* text = R"(
+		contract base { function foo(bool); }
+		contract derived is base { function foo(uint) {} }
+		)";
+	ETH_TEST_REQUIRE_NO_THROW(sourceUnit = parseTextAndResolveNames(text), "Parsing and name Resolving failed");
+	std::vector<ASTPointer<ASTNode>> nodes = sourceUnit->getNodes();
+	ContractDefinition* base = dynamic_cast<ContractDefinition*>(nodes[0].get());
+	ContractDefinition* derived = dynamic_cast<ContractDefinition*>(nodes[1].get());
+	BOOST_REQUIRE(base);
+	BOOST_CHECK(!base->isFullyImplemented());
+	BOOST_REQUIRE(derived);
+	BOOST_CHECK(!derived->isFullyImplemented());
+}
+
 BOOST_AUTO_TEST_CASE(create_abstract_contract)
 {
 	ASTPointer<SourceUnit> sourceUnit;
@@ -1900,6 +1917,18 @@ BOOST_AUTO_TEST_CASE(storage_location_local_variables)
 	BOOST_CHECK_NO_THROW(parseTextAndResolveNames(sourceCode));
 }
 
+BOOST_AUTO_TEST_CASE(no_mappings_in_memory_array)
+{
+	char const* sourceCode = R"(
+		contract C {
+			function f() {
+				mapping(uint=>uint)[] memory x;
+			}
+		}
+	)";
+	BOOST_CHECK_THROW(parseTextAndResolveNames(sourceCode), TypeError);
+}
+
 BOOST_AUTO_TEST_CASE(assignment_mem_to_local_storage_variable)
 {
 	char const* sourceCode = R"(
@@ -1931,6 +1960,20 @@ BOOST_AUTO_TEST_CASE(storage_assign_to_different_local_variable)
 	BOOST_CHECK_THROW(parseTextAndResolveNames(sourceCode), TypeError);
 }
 
+BOOST_AUTO_TEST_CASE(no_delete_on_storage_pointers)
+{
+	char const* sourceCode = R"(
+		contract C {
+			uint[] data;
+			function f() {
+				var x = data;
+				delete x;
+			}
+		}
+	)";
+	BOOST_CHECK_THROW(parseTextAndResolveNames(sourceCode), TypeError);
+}
+
 BOOST_AUTO_TEST_CASE(assignment_mem_storage_variable_directly)
 {
 	char const* sourceCode = R"(
@@ -2000,6 +2043,19 @@ BOOST_AUTO_TEST_CASE(dynamic_return_types_not_possible)
 	BOOST_CHECK_THROW(parseTextAndResolveNames(sourceCode), TypeError);
 }
 
+BOOST_AUTO_TEST_CASE(memory_arrays_not_resizeable)
+{
+	char const* sourceCode = R"(
+		contract C {
+			function f() {
+				uint[] memory x;
+				x.length = 2;
+			}
+		}
+	)";
+	BOOST_CHECK_THROW(parseTextAndResolveNames(sourceCode), TypeError);
+}
+
 BOOST_AUTO_TEST_SUITE_END()
 
 }

test/libsolidity/SolidityWallet.cpp

@@ -200,17 +200,17 @@ contract multiowned {
 	}
 
 	// the number of owners that must confirm the same operation before it is run.
-	uint m_required;
+	uint public m_required;
 	// pointer used to find a free slot in m_owners
-	uint m_numOwners;
+	uint public m_numOwners;
 	// list of owners
-	uint[256] m_owners;
+	uint[256] public m_owners;
 	uint constant c_maxOwners = 250;
 	// index on the list of owners to allow reverse lookup
-	mapping(uint => uint) m_ownerIndex;
+	mapping(uint => uint) public m_ownerIndex;
 	// the ongoing operations.
-	mapping(bytes32 => PendingState) m_pending;
-	bytes32[] m_pendingIndex;
+	mapping(bytes32 => PendingState) public m_pending;
+	bytes32[] public m_pendingIndex;
 }
 
 // inheritable "property" contract that enables methods to be protected by placing a linear limit (specifiable)
@@ -251,9 +251,9 @@ contract daylimit is multiowned {
 	}
 	// determines today's index.
 	function today() private constant returns (uint) { return now / 1 days; }
-	uint m_spentToday;
-	uint m_dailyLimit;
-	uint m_lastDay;
+	uint public m_spentToday;
+	uint public m_dailyLimit;
+	uint public m_lastDay;
 }
 // interface contract for multisig proxy contracts; see below for docs.
 contract multisig {
@@ -275,11 +275,14 @@ contract Wallet is multisig, multiowned, daylimit {
 		uint value;
 		bytes data;
 	}
+	/*
 	// logged events:
 	// Funds has arrived into the wallet (record how much).
 	event Deposit(address from, uint value);
 	// Single transaction going out of the wallet (record who signed for it, how much, and to whom it's going).
 	event SingleTransact(address owner, uint value, address to, bytes data);
+	// Multi-sig transaction going out of the wallet (record who signed for it last, the operation hash, how much, and to whom it's going).
+	event MultiTransact(address owner, bytes32 operation, uint value, address to, bytes data);*/
 	// constructor - just pass on the owner arra to the multiowned.
 	event Created();
 	function Wallet() {
@@ -299,7 +302,7 @@ contract Wallet is multisig, multiowned, daylimit {
 	// If not, goes into multisig process. We provide a hash on return to allow the sender to provide
 	// shortcuts for the other confirmations (allowing them to avoid replicating the _to, _value
 	// and _data arguments). They still get the option of using them if they want, anyways.
-	function execute(address _to, uint _value, bytes _data) onlyowner external returns (bytes32 _r) {
+	function execute(address _to, uint _value, bytes _data) external onlyowner returns (bytes32 _r) {
 		// first, take the opportunity to check that we're under the daily limit.
 		if (underLimit(_value)) {
 			SingleTransact(msg.sender, _value, _to, _data);
@@ -332,8 +335,14 @@ contract Wallet is multisig, multiowned, daylimit {
 			delete m_txs[m_pendingIndex[i]];
 		super.clearPending();
 	}
+	// // internally confirm transaction with all of the info. returns true iff confirmed good and executed.
+	// function confirmVerbose(bytes32 _h, address _to, uint _value, bytes _data) private onlymanyowners(_h) returns (bool) {
+	//     _to.call.value(_value)(_data);
+	//     MultiTransact("out", msg.sender, _h, _value, _to);
+	//     return true;
+	// }
 	// pending transactions we have at present.
-	mapping (bytes32 => Transaction) m_txs;
+	mapping (bytes32 => Transaction) public m_txs;
 }
 )DELIMITER";
 

test/libwhisper/bloomFilter.cpp

@@ -28,8 +28,7 @@ using namespace dev;
 using namespace dev::shh;
 
 using TopicBloomFilterShort = TopicBloomFilterBase<4>;
-using TopicBloomFilterLong = TopicBloomFilterBase<8>;
-using TopicBloomFilterTest = TopicBloomFilterLong;
+using TopicBloomFilterTest = TopicBloomFilterBase<c_topicBloomFilterSize>;
 
 void testAddNonExisting(TopicBloomFilterShort& _f, AbridgedTopic const& _h)
 {
@@ -59,7 +58,7 @@ void testRemoveExistingBloom(TopicBloomFilterShort& _f, AbridgedTopic const& _h)
 	BOOST_REQUIRE(!_f.containsBloom(_h));
 }
 
-int calculateExpected(TopicBloomFilterTest const& f, int const n)
+double calculateExpected(TopicBloomFilterTest const& f, int n)
 {
 	int const m = f.size * 8; // number of bits in the bloom
 	int const k = f.BitsPerBloom; // number of hash functions (e.g. bits set to 1 in every bloom)
@@ -77,10 +76,10 @@ int calculateExpected(TopicBloomFilterTest const& f, int const n)
 	for (int i = 0; i < k; ++i)
 		kBitsSet *= single;
 
-	return static_cast<int>(kBitsSet * 100 + 0.5); // in percents, rounded up
+	return kBitsSet;
 }
 
-void testFalsePositiveRate(TopicBloomFilterTest const& f, int const inserted, Topic& x)
+double testFalsePositiveRate(TopicBloomFilterTest const& f, int inserted, Topic& x)
 {
 	int const c_sampleSize = 1000;
 	int falsePositive = 0;
@@ -93,12 +92,14 @@ void testFalsePositiveRate(TopicBloomFilterTest const& f, int const inserted, To
 			++falsePositive;
 	}
 
-	falsePositive /= (c_sampleSize / 100); // in percents
-	int expected = calculateExpected(f, inserted);
-	int allowed = expected + (expected / 5); // allow deviations ~20%
+	double res = double(falsePositive) / double(c_sampleSize);
 
-	//cnote << "Inserted: " << inserted << ", False Positive Rate: " << falsePositive << ", Expected: " << expected;
-	BOOST_REQUIRE(falsePositive <= allowed);
+	double expected = calculateExpected(f, inserted);
+	double allowed = expected * 1.2 + 0.05; // allow deviations ~25%
+
+	//cnote << "Inserted: " << inserted << ", False Positive Rate: " << res << ", Expected: " << expected;
+	BOOST_REQUIRE(res <= allowed);
+	return expected;
 }
 
 BOOST_AUTO_TEST_SUITE(bloomFilter)
@@ -111,11 +112,13 @@ BOOST_AUTO_TEST_CASE(falsePositiveRate)
 	TopicBloomFilterTest f;
 	Topic x(0xC0DEFEED); // deterministic pseudorandom value
 
-	for (int i = 1; i < 21; ++i)
+	double expectedRate = 0;
+
+	for (int i = 1; i < 50 && isless(expectedRate, 0.5); ++i)
 	{
 		x = sha3(x);
 		f.addBloom(AbridgedTopic(x));
-		testFalsePositiveRate(f, i, x);
+		expectedRate = testFalsePositiveRate(f, i, x);
 	}
 }
 
@@ -241,4 +244,4 @@ BOOST_AUTO_TEST_CASE(bloomFilterRaw)
 	BOOST_REQUIRE(!f.contains(b00110111));
 }
 
-BOOST_AUTO_TEST_SUITE_END()
+BOOST_AUTO_TEST_SUITE_END()

test/libwhisper/whisperTopic.cpp

@@ -23,6 +23,7 @@
 #include <boost/test/unit_test.hpp>
 
 #include <libp2p/Host.h>
+#include <libp2p/Session.h>
 #include <libwhisper/WhisperPeer.h>
 #include <libwhisper/WhisperHost.h>
 #include <test/TestHelper.h>
@@ -301,4 +302,86 @@ BOOST_AUTO_TEST_CASE(asyncforwarding)
 	BOOST_REQUIRE_EQUAL(result, 1);
 }
 
+BOOST_AUTO_TEST_CASE(topicAdvertising)
+{
+	if (test::Options::get().nonetwork)
+		return;
+
+	cnote << "Testing Topic Advertising...";
+	VerbosityHolder setTemporaryLevel(2);
+
+	Host host1("first", NetworkPreferences("127.0.0.1", 30303, false));
+	host1.setIdealPeerCount(1);
+	auto whost1 = host1.registerCapability(new WhisperHost());
+	host1.start();
+	while (!host1.haveNetwork())
+		this_thread::sleep_for(chrono::milliseconds(10));
+
+	Host host2("second", NetworkPreferences("127.0.0.1", 30305, false));
+	host2.setIdealPeerCount(1);
+	auto whost2 = host2.registerCapability(new WhisperHost());
+	unsigned w2 = whost2->installWatch(BuildTopicMask("test2"));
+
+	host2.start();
+	while (!host2.haveNetwork())
+		this_thread::sleep_for(chrono::milliseconds(10));
+
+	host1.addNode(host2.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), 30305, 30305));
+	while (!host1.haveNetwork())
+		this_thread::sleep_for(chrono::milliseconds(10));
+
+	while (!host1.peerCount())
+		this_thread::sleep_for(chrono::milliseconds(10));
+
+	while (!host2.peerCount())
+		this_thread::sleep_for(chrono::milliseconds(10));
+
+	std::vector<std::pair<std::shared_ptr<Session>, std::shared_ptr<Peer>>> sessions;
+
+	for (int i = 0; i < 600; ++i)
+	{
+		sessions = whost1->peerSessions();
+		if (!sessions.empty() && sessions.back().first->cap<WhisperPeer>()->bloom())
+			break;
+		else
+			this_thread::sleep_for(chrono::milliseconds(10));
+	}
+
+	BOOST_REQUIRE(sessions.size());
+	TopicBloomFilterHash bf1 = sessions.back().first->cap<WhisperPeer>()->bloom();
+	TopicBloomFilterHash bf2 = whost2->bloom();
+	BOOST_REQUIRE_EQUAL(bf1, bf2);
+	BOOST_REQUIRE(bf1);
+	BOOST_REQUIRE(!whost1->bloom());
+
+	unsigned w1 = whost1->installWatch(BuildTopicMask("test1"));
+
+	for (int i = 0; i < 600; ++i)
+	{
+		sessions = whost2->peerSessions();
+		if (!sessions.empty() && sessions.back().first->cap<WhisperPeer>()->bloom())
+			break;
+		else
+			this_thread::sleep_for(chrono::milliseconds(10));
+	}
+
+	BOOST_REQUIRE(sessions.size());
+	BOOST_REQUIRE_EQUAL(sessions.back().second->id, host1.id());
+
+	bf2 = sessions.back().first->cap<WhisperPeer>()->bloom();
+	bf1 = whost1->bloom();
+	BOOST_REQUIRE_EQUAL(bf1, bf2);
+	BOOST_REQUIRE(bf1);
+
+	unsigned random = 0xC0FFEE;
+	whost1->uninstallWatch(w1);
+	whost1->uninstallWatch(random);
+	whost1->uninstallWatch(w1);
+	whost1->uninstallWatch(random);
+	whost2->uninstallWatch(random);
+	whost2->uninstallWatch(w2);
+	whost2->uninstallWatch(random);
+	whost2->uninstallWatch(w2);
+}
+
 BOOST_AUTO_TEST_SUITE_END()

third/CMakeLists.txt

@@ -41,7 +41,6 @@ target_link_libraries(${EXECUTABLE} webthree)
 target_link_libraries(${EXECUTABLE} ethereum)
 target_link_libraries(${EXECUTABLE} evm)
 target_link_libraries(${EXECUTABLE} ethcore) 
-target_link_libraries(${EXECUTABLE} secp256k1) 
 if (NOT ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC"))
 target_link_libraries(${EXECUTABLE} serpent)
 endif()