Merge branch 'develop' into sha3xor

alethzero/MainWin.cpp

@@ -1097,6 +1097,7 @@ void Main::refreshBlockChain()
 			blockItem->setSelected(true);
 
 		int n = 0;
+		try {
 			auto b = bc.block(h);
 			for (auto const& i: RLP(b)[1])
 			{
@@ -1121,6 +1122,8 @@ void Main::refreshBlockChain()
 					txItem->setSelected(true);
 				n++;
 			}
+		}
+		catch (...) {}
 	};
 
 	if (filters.empty())

libdevcore/Common.cpp

@@ -27,7 +27,7 @@ using namespace dev;
 namespace dev
 {
 
-char const* Version = "0.9.13";
+char const* Version = "0.9.14";
 
 }
 

libdevcrypto/MemoryDB.cpp

@@ -32,51 +32,55 @@ const char* DBWarn::name() { return "TDB"; }
 
 std::map<h256, std::string> MemoryDB::get() const
 {
-	if (!m_enforceRefs)
-		return m_over;
 	std::map<h256, std::string> ret;
-	for (auto const& i: m_refCount)
+	for (auto const& i: m_main)
-		if (i.second)
+		if (!m_enforceRefs || i.second.second > 0)
-			ret.insert(*m_over.find(i.first));
+			ret.insert(make_pair(i.first, i.second.first));
 	return ret;
 }
 
-std::string MemoryDB::lookup(h256 _h) const
+std::string MemoryDB::lookup(h256 const& _h) const
 {
-	auto it = m_over.find(_h);
+	auto it = m_main.find(_h);
-	if (it != m_over.end())
+	if (it != m_main.end())
 	{
-		if (!m_enforceRefs || (m_refCount.count(it->first) && m_refCount.at(it->first)))
+		if (!m_enforceRefs || it->second.second > 0)
-			return it->second;
+			return it->second.first;
 //		else if (m_enforceRefs && m_refCount.count(it->first) && !m_refCount.at(it->first))
 //			cnote << "Lookup required for value with no refs. Let's hope it's in the DB." << _h;
 	}
 	return std::string();
 }
 
-bool MemoryDB::exists(h256 _h) const
+bool MemoryDB::exists(h256 const& _h) const
 {
-	auto it = m_over.find(_h);
+	auto it = m_main.find(_h);
-	if (it != m_over.end() && (!m_enforceRefs || (m_refCount.count(it->first) && m_refCount.at(it->first))))
+	if (it != m_main.end() && (!m_enforceRefs || it->second.second > 0))
 		return true;
 	return false;
 }
 
-void MemoryDB::insert(h256 _h, bytesConstRef _v)
+void MemoryDB::insert(h256 const& _h, bytesConstRef _v)
 {
-	m_over[_h] = _v.toString();
+	auto it = m_main.find(_h);
-	m_refCount[_h]++;
+	if (it != m_main.end())
+	{
+		it->second.first = _v.toString();
+		it->second.second++;
+	}
+	else
+		m_main[_h] = make_pair(_v.toString(), 1);
 #if ETH_PARANOIA
-	dbdebug << "INST" << _h << "=>" << m_refCount[_h];
+	dbdebug << "INST" << _h << "=>" << m_main[_h].second;
 #endif
 }
 
-bool MemoryDB::kill(h256 _h)
+bool MemoryDB::kill(h256 const& _h)
 {
-	if (m_refCount.count(_h))
+	if (m_main.count(_h))
 	{
-		if (m_refCount[_h] > 0)
+		if (m_main[_h].second > 0)
-			--m_refCount[_h];
+			m_main[_h].second--;
 #if ETH_PARANOIA
 		else
 		{
@@ -85,7 +89,7 @@ bool MemoryDB::kill(h256 _h)
 			dbdebug << "NOKILL-WAS" << _h;
 			return false;
 		}
-		dbdebug << "KILL" << _h << "=>" << m_refCount[_h];
+		dbdebug << "KILL" << _h << "=>" << m_main[_h].second;
 		return true;
 	}
 	else
@@ -101,16 +105,18 @@ bool MemoryDB::kill(h256 _h)
 
 void MemoryDB::purge()
 {
-	for (auto const& i: m_refCount)
+	for (auto it = m_main.begin(); it != m_main.end(); )
-		if (!i.second)
+		if (it->second.second)
-			m_over.erase(i.first);
+			++it;
+		else
+			it = m_main.erase(it);
 }
 
 set<h256> MemoryDB::keys() const
 {
 	set<h256> ret;
-	for (auto const& i: m_refCount)
+	for (auto const& i: m_main)
-		if (i.second && h128(i.first.ref().cropped(0, 16)))
+		if (i.second.second)
 			ret.insert(i.first);
 	return ret;
 }

libdevcrypto/MemoryDB.h

@@ -44,28 +44,24 @@ class MemoryDB
 public:
 	MemoryDB() {}
 
-	void clear() { m_over.clear(); }
+	void clear() { m_main.clear(); }	// WARNING !!!! didn't originally clear m_refCount!!!
 	std::map<h256, std::string> get() const;
 
-	std::string lookup(h256 _h) const;
+	std::string lookup(h256 const& _h) const;
-	bool exists(h256 _h) const;
+	bool exists(h256 const& _h) const;
-	void insert(h256 _h, bytesConstRef _v);
+	void insert(h256 const& _h, bytesConstRef _v);
-	bool kill(h256 _h);
+	bool kill(h256 const& _h);
 	void purge();
 
-	bytes lookupAux(h256 _h) const { auto h = aux(_h); return m_aux.count(h) ? m_aux.at(h) : bytes(); }
+	bytes lookupAux(h256 const& _h) const { try { return m_aux.at(_h).first; } catch (...) { return bytes(); } }
-	void removeAux(h256 _h) { m_auxActive.erase(aux(_h)); }
+	void removeAux(h256 const& _h) { m_aux[_h].second = false; }
-	void insertAux(h256 _h, bytesConstRef _v) { auto h = aux(_h); m_auxActive.insert(h); m_aux[h] = _v.toBytes(); }
+	void insertAux(h256 const& _h, bytesConstRef _v) { m_aux[_h] = make_pair(_v.toBytes(), true); }
 
 	std::set<h256> keys() const;
 
 protected:
-	static h256 aux(h256 _k) { return h256(sha3(_k).ref().cropped(0, 24), h256::AlignLeft); }
+	std::map<h256, std::pair<std::string, unsigned>> m_main;
-
+	std::map<h256, std::pair<bytes, bool>> m_aux;
-	std::map<h256, std::string> m_over;
-	std::map<h256, unsigned> m_refCount;
-	std::set<h256> m_auxActive;
-	std::map<h256, bytes> m_aux;
 
 	mutable bool m_enforceRefs = false;
 };
@@ -83,7 +79,7 @@ private:
 
 inline std::ostream& operator<<(std::ostream& _out, MemoryDB const& _m)
 {
-	for (auto i: _m.get())
+	for (auto const& i: _m.get())
 	{
 		_out << i.first << ": ";
 		_out << RLP(i.second);

libdevcrypto/OverlayDB.cpp

@@ -41,23 +41,22 @@ void OverlayDB::commit()
 	{
 		ldb::WriteBatch batch;
 //		cnote << "Committing nodes to disk DB:";
-		for (auto const& i: m_over)
+		for (auto const& i: m_main)
 		{
-//			cnote << i.first << "#" << m_refCount[i.first];
+//			cnote << i.first << "#" << m_main[i.first].second;
-			if (m_refCount[i.first])
+			if (i.second.second)
-				batch.Put(ldb::Slice((char const*)i.first.data(), i.first.size), ldb::Slice(i.second.data(), i.second.size()));
+				batch.Put(ldb::Slice((char const*)i.first.data(), i.first.size), ldb::Slice(i.second.first.data(), i.second.first.size()));
 		}
-		for (auto const& i: m_auxActive)
+		for (auto const& i: m_aux)
-			if (m_aux.count(i))
+			if (i.second.second)
 			{
-				batch.Put(i.ref(), bytesConstRef(&m_aux[i]));
+				bytes b = i.first.asBytes();
-				m_aux.erase(i);
+				b.push_back(255);	// for aux
+				batch.Put(bytesConstRef(&b), bytesConstRef(&i.second.first));
 			}
 		m_db->Write(m_writeOptions, &batch);
-		m_auxActive.clear();
 		m_aux.clear();
-		m_over.clear();
+		m_main.clear();
-		m_refCount.clear();
 	}
 }
 
@@ -67,7 +66,9 @@ bytes OverlayDB::lookupAux(h256 _h) const
 	if (!ret.empty())
 		return ret;
 	std::string v;
-	m_db->Get(m_readOptions, aux(_h).ref(), &v);
+	bytes b = _h.asBytes();
+	b.push_back(255);	// for aux
+	m_db->Get(m_readOptions, bytesConstRef(&b), &v);
 	if (v.empty())
 		cwarn << "Aux not found: " << _h;
 	return asBytes(v);
@@ -75,8 +76,7 @@ bytes OverlayDB::lookupAux(h256 _h) const
 
 void OverlayDB::rollback()
 {
-	m_over.clear();
+	m_main.clear();
-	m_refCount.clear();
 }
 
 std::string OverlayDB::lookup(h256 _h) const

libdevcrypto/TrieDB.h

@@ -47,6 +47,11 @@ struct InvalidTrie: virtual dev::Exception {};
 extern const h256 c_shaNull;
 extern const h256 EmptyTrie;
 
+enum class Verification {
+	Skip,
+	Normal
+};
+
 /**
  * @brief Merkle Patricia Tree "Trie": a modifed base-16 Radix tree.
  * This version uses a database backend.
@@ -69,17 +74,19 @@ public:
 	using DB = _DB;
 
 	GenericTrieDB(DB* _db = nullptr): m_db(_db) {}
-	GenericTrieDB(DB* _db, h256 _root) { open(_db, _root); }
+	GenericTrieDB(DB* _db, h256 const& _root, Verification _v = Verification::Normal) { open(_db, _root, _v); }
 	~GenericTrieDB() {}
 
 	void open(DB* _db) { m_db = _db; }
-	void open(DB* _db, h256 _root) { m_db = _db; setRoot(_root); }
+	void open(DB* _db, h256 const& _root, Verification _v = Verification::Normal) { m_db = _db; setRoot(_root, _v); }
 
 	void init() { setRoot(insertNode(&RLPNull)); assert(node(m_root).size()); }
 
-	void setRoot(h256 _root)
+	void setRoot(h256 const& _root, Verification _v = Verification::Normal)
 	{
 		m_root = _root;
+		if (_v == Verification::Normal)
+		{
 			if (m_root == c_shaNull && !m_db->exists(m_root))
 				init();
 
@@ -87,13 +94,14 @@ public:
 			if (!node(m_root).size())
 				BOOST_THROW_EXCEPTION(RootNotFound());
 		}
+	}
 
 	/// True if the trie is uninitialised (i.e. that the DB doesn't contain the root node).
 	bool isNull() const { return !node(m_root).size(); }
 	/// True if the trie is initialised but empty (i.e. that the DB contains the root node which is empty).
 	bool isEmpty() const { return m_root == c_shaNull && node(m_root).size(); }
 
-	h256 root() const { if (!node(m_root).size()) BOOST_THROW_EXCEPTION(BadRoot()); /*std::cout << "Returning root as " << ret << " (really " << m_root << ")" << std::endl;*/ return m_root; }	// patch the root in the case of the empty trie. TODO: handle this properly.
+	h256 const& root() const { if (!node(m_root).size()) BOOST_THROW_EXCEPTION(BadRoot()); /*std::cout << "Returning root as " << ret << " (really " << m_root << ")" << std::endl;*/ return m_root; }	// patch the root in the case of the empty trie. TODO: handle this properly.
 
 	void debugPrint() {}
 
@@ -301,7 +309,7 @@ public:
 	using KeyType = _KeyType;
 
 	SpecificTrieDB(DB* _db = nullptr): Generic(_db) {}
-	SpecificTrieDB(DB* _db, h256 _root): Generic(_db, _root) {}
+	SpecificTrieDB(DB* _db, h256 _root, Verification _v = Verification::Normal): Generic(_db, _root, _v) {}
 
 	std::string operator[](KeyType _k) const { return at(_k); }
 
@@ -349,7 +357,7 @@ public:
 	using DB = _DB;
 
 	HashedGenericTrieDB(DB* _db = nullptr): Super(_db) {}
-	HashedGenericTrieDB(DB* _db, h256 _root): Super(_db, _root) {}
+	HashedGenericTrieDB(DB* _db, h256 _root, Verification _v = Verification::Normal): Super(_db, _root, _v) {}
 
 	using Super::open;
 	using Super::init;
@@ -402,20 +410,20 @@ class FatGenericTrieDB: public GenericTrieDB<DB>
 
 public:
 	FatGenericTrieDB(DB* _db): Super(_db), m_secure(_db) {}
-	FatGenericTrieDB(DB* _db, h256 _root) { open(_db, _root); }
+	FatGenericTrieDB(DB* _db, h256 _root, Verification _v = Verification::Normal) { open(_db, _root, _v); }
 
-	void open(DB* _db, h256 _root) { Super::open(_db); m_secure.open(_db); setRoot(_root); }
+	void open(DB* _db, h256 _root, Verification _v = Verification::Normal) { Super::open(_db); m_secure.open(_db); setRoot(_root, _v); }
 
 	void init() { Super::init(); m_secure.init(); syncRoot(); }
 
-	void setRoot(h256 _root)
+	void setRoot(h256 _root, Verification _v = Verification::Normal)
 	{
 		if (!m_secure.isNull())
 			Super::db()->removeAux(m_secure.root());
-		m_secure.setRoot(_root);
+		m_secure.setRoot(_root, _v);
 		auto rb = Super::db()->lookupAux(m_secure.root());
 		auto r = h256(rb);
-		Super::setRoot(r);
+		Super::setRoot(r, _v);
 	}
 
 	h256 root() const { return m_secure.root(); }
@@ -853,8 +861,8 @@ template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSli
 
 template <class DB> void GenericTrieDB<DB>::mergeAtAux(RLPStream& _out, RLP const& _orig, NibbleSlice _k, bytesConstRef _v)
 {
-#if ETH_PARANOIA
+#if ETH_PARANOIA || !ETH_TRUE
-	tdebug << "mergeAtAux " << _orig << _k << sha3(_orig.data()) << ((_orig.isData() && _orig.size() <= 32) ? _orig.toHash<h256>() : std::string());
+	tdebug << "mergeAtAux " << _orig << _k << sha3(_orig.data()) << ((_orig.isData() && _orig.size() <= 32) ? _orig.toHash<h256>().abridged() : std::string());
 #endif
 
 	RLP r = _orig;
@@ -1008,8 +1016,8 @@ template <class DB> bytes GenericTrieDB<DB>::deleteAt(RLP const& _orig, NibbleSl
 
 template <class DB> bool GenericTrieDB<DB>::deleteAtAux(RLPStream& _out, RLP const& _orig, NibbleSlice _k)
 {
-#if ETH_PARANOIA
+#if ETH_PARANOIA || !ETH_TRUE
-	tdebug << "deleteAtAux " << _orig << _k << sha3(_orig.data()) << ((_orig.isData() && _orig.size() <= 32) ? _orig.toHash<h256>() : std::string());
+	tdebug << "deleteAtAux " << _orig << _k << sha3(_orig.data()) << ((_orig.isData() && _orig.size() <= 32) ? _orig.toHash<h256>().abridged() : std::string());
 #endif
 
 	bytes b = _orig.isEmpty() ? bytes() : deleteAt(_orig.isList() ? _orig : RLP(node(_orig.toHash<h256>())), _k);

libethcore/Common.cpp

@@ -36,7 +36,7 @@ namespace eth
 {
 
 const unsigned c_protocolVersion = 60;
-const unsigned c_minorProtocolVersion = 1;
+const unsigned c_minorProtocolVersion = 2;
 const unsigned c_databaseBaseVersion = 9;
 #if ETH_FATDB
 const unsigned c_databaseVersionModifier = 1;

libethereum/BlockChain.cpp

@@ -47,7 +47,7 @@ using namespace dev::eth;
 namespace js = json_spirit;
 
 #define ETH_CATCH 1
-#define ETH_TIMED_IMPORTS 0
+#define ETH_TIMED_IMPORTS 1
 
 #ifdef _WIN32
 const char* BlockChainDebug::name() { return EthBlue "8" EthWhite " <>"; }
@@ -303,7 +303,7 @@ LastHashes BlockChain::lastHashes(unsigned _n) const
 
 tuple<h256s, h256s, bool> BlockChain::sync(BlockQueue& _bq, OverlayDB const& _stateDB, unsigned _max)
 {
-	_bq.tick(*this);
+//	_bq.tick(*this);
 
 	vector<bytes> blocks;
 	_bq.drain(blocks, _max);
@@ -445,8 +445,8 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import
 
 	ldb::WriteBatch blocksBatch;
 	ldb::WriteBatch extrasBatch;
-	h256 newLastBlockHash;
+	h256 newLastBlockHash = currentHash();
-	unsigned newLastBlockNumber = 0;
+	unsigned newLastBlockNumber = number();
 
 	u256 td;
 #if ETH_CATCH
@@ -478,30 +478,27 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import
 #endif
 
 		// All ok - insert into DB
-		{
+
 		// ensure parent is cached for later addition.
 		// TODO: this is a bit horrible would be better refactored into an enveloping UpgradableGuard
 		// together with an "ensureCachedWithUpdatableLock(l)" method.
 		// This is safe in practice since the caches don't get flushed nearly often enough to be
 		// done here.
 		details(bi.parentHash);
-			WriteGuard l(x_details);
+		ETH_WRITE_GUARDED(x_details)
 			m_details[bi.parentHash].children.push_back(bi.hash());
-		}
 
 #if ETH_TIMED_IMPORTS
 		collation = t.elapsed();
 		t.restart();
 #endif
 
-		{
+		blocksBatch.Put(toSlice(bi.hash()), (ldb::Slice)ref(_block));
-			ReadGuard l2(x_details);
+		ETH_READ_GUARDED(x_details)
-			extrasBatch.Put(toSlice(bi.hash(), ExtraDetails), (ldb::Slice)dev::ref(BlockDetails((unsigned)pd.number + 1, td, bi.parentHash, {}).rlp()));
 			extrasBatch.Put(toSlice(bi.parentHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp()));
+		extrasBatch.Put(toSlice(bi.hash(), ExtraDetails), (ldb::Slice)dev::ref(BlockDetails((unsigned)pd.number + 1, td, bi.parentHash, {}).rlp()));
 		extrasBatch.Put(toSlice(bi.hash(), ExtraLogBlooms), (ldb::Slice)dev::ref(blb.rlp()));
 		extrasBatch.Put(toSlice(bi.hash(), ExtraReceipts), (ldb::Slice)dev::ref(br.rlp()));
-			blocksBatch.Put(toSlice(bi.hash()), (ldb::Slice)ref(_block));
-		}
 
 #if ETH_TIMED_IMPORTS
 		writing = t.elapsed();

libethereum/BlockQueue.cpp

@@ -74,17 +74,19 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo
 	}
 
 	UpgradeGuard ul(l);
+	invariants_WITH_LOCK();
 
 	// Check it's not in the future
 	(void)_isOurs;
 	if (bi.timestamp > (u256)time(0)/* && !_isOurs*/)
 	{
-		m_future.insert(make_pair((unsigned)bi.timestamp, _block.toBytes()));
+		m_future.insert(make_pair((unsigned)bi.timestamp, make_pair(h, _block.toBytes())));
 		char buf[24];
 		time_t bit = (unsigned)bi.timestamp;
 		if (strftime(buf, 24, "%X", localtime(&bit)) == 0)
 			buf[0] = '\0'; // empty if case strftime fails
 		cblockq << "OK - queued for future [" << bi.timestamp << "vs" << time(0) << "] - will wait until" << buf;
+		invariants_WITH_LOCK();
 		return ImportResult::FutureTime;
 	}
 	else
@@ -94,6 +96,7 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo
 		{
 			m_knownBad.insert(bi.hash());
 			// bad parent; this is bad too, note it as such
+			invariants_WITH_LOCK();
 			return ImportResult::BadChain;
 		}
 		else if (!m_readySet.count(bi.parentHash) && !m_drainingSet.count(bi.parentHash) && !_bc.isKnown(bi.parentHash))
@@ -103,16 +106,18 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo
 			m_unknown.insert(make_pair(bi.parentHash, make_pair(h, _block.toBytes())));
 			m_unknownSet.insert(h);
 
+			invariants_WITH_LOCK();
 			return ImportResult::UnknownParent;
 		}
 		else
 		{
 			// If valid, append to blocks.
 			cblockq << "OK - ready for chain insertion.";
-			m_ready.push_back(_block.toBytes());
+			m_ready.push_back(make_pair(h, _block.toBytes()));
 			m_readySet.insert(h);
+			invariants_WITH_LOCK();
 
-			noteReadyWithoutWriteGuard(h);
+			noteReady_WITH_LOCK(h);
 			m_onReady();
 			return ImportResult::Success;
 		}
@@ -122,27 +127,33 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo
 bool BlockQueue::doneDrain(h256s const& _bad)
 {
 	WriteGuard l(m_lock);
+	invariants_WITH_LOCK();
 	m_drainingSet.clear();
 	if (_bad.size())
 	{
-		vector<bytes> old;
+		vector<pair<h256, bytes>> old;
 		swap(m_ready, old);
 		for (auto& b: old)
 		{
-			BlockInfo bi(b);
+			BlockInfo bi(b.second);
 			if (m_knownBad.count(bi.parentHash))
-				m_knownBad.insert(bi.hash());
+			{
+				m_knownBad.insert(b.first);
+				m_readySet.erase(b.first);
+			}
 			else
 				m_ready.push_back(std::move(b));
 		}
 	}
 	m_knownBad += _bad;
+	// GAA!!!! NEVER EMPTY?!?!?! TODO: remove items from readySet!
+	invariants_WITH_LOCK();
 	return !m_readySet.empty();
 }
 
 void BlockQueue::tick(BlockChain const& _bc)
 {
-	vector<bytes> todo;
+	vector<pair<h256, bytes>> todo;
 	{
 		UpgradableGuard l(m_lock);
 		if (m_future.empty())
@@ -158,16 +169,18 @@ void BlockQueue::tick(BlockChain const& _bc)
 
 		{
 			UpgradeGuard l2(l);
+			invariants_WITH_LOCK();
 			auto end = m_future.lower_bound(t);
 			for (auto i = m_future.begin(); i != end; ++i)
 				todo.push_back(move(i->second));
 			m_future.erase(m_future.begin(), end);
+			invariants_WITH_LOCK();
 		}
 	}
 	cblockq << "Importing" << todo.size() << "past-future blocks.";
 
 	for (auto const& b: todo)
-		import(&b, _bc);
+		import(&b.second, _bc);
 }
 
 template <class T> T advanced(T _t, unsigned _n)
@@ -194,25 +207,34 @@ QueueStatus BlockQueue::blockStatus(h256 const& _h) const
 void BlockQueue::drain(std::vector<bytes>& o_out, unsigned _max)
 {
 	WriteGuard l(m_lock);
+	invariants_WITH_LOCK();
 	if (m_drainingSet.empty())
 	{
 		o_out.resize(min<unsigned>(_max, m_ready.size()));
 		for (unsigned i = 0; i < o_out.size(); ++i)
-			swap(o_out[i], m_ready[i]);
+			swap(o_out[i], m_ready[i].second);
 		m_ready.erase(m_ready.begin(), advanced(m_ready.begin(), o_out.size()));
 		for (auto const& bs: o_out)
 		{
-			auto h = sha3(bs);
+			// TODO: @optimise use map<h256, bytes> rather than vector<bytes> & set<h256>.
+			auto h = BlockInfo::headerHash(bs);
 			m_drainingSet.insert(h);
 			m_readySet.erase(h);
 		}
 //		swap(o_out, m_ready);
 //		swap(m_drainingSet, m_readySet);
 	}
+	invariants_WITH_LOCK();
+}
+
+void BlockQueue::invariants_WITH_LOCK() const
+{
+	assert(m_readySet.size() == m_ready.size());
 }
 
-void BlockQueue::noteReadyWithoutWriteGuard(h256 _good)
+void BlockQueue::noteReady_WITH_LOCK(h256 const& _good)
 {
+	invariants_WITH_LOCK();
 	list<h256> goodQueue(1, _good);
 	while (!goodQueue.empty())
 	{
@@ -220,7 +242,7 @@ void BlockQueue::noteReadyWithoutWriteGuard(h256 _good)
 		goodQueue.pop_front();
 		for (auto it = r.first; it != r.second; ++it)
 		{
-			m_ready.push_back(it->second.second);
+			m_ready.push_back(it->second);
 			auto newReady = it->second.first;
 			m_unknownSet.erase(newReady);
 			m_readySet.insert(newReady);
@@ -228,16 +250,19 @@ void BlockQueue::noteReadyWithoutWriteGuard(h256 _good)
 		}
 		m_unknown.erase(r.first, r.second);
 	}
+	invariants_WITH_LOCK();
 }
 
 void BlockQueue::retryAllUnknown()
 {
+	invariants_WITH_LOCK();
 	for (auto it = m_unknown.begin(); it != m_unknown.end(); ++it)
 	{
-		m_ready.push_back(it->second.second);
+		m_ready.push_back(it->second);
 		auto newReady = it->second.first;
 		m_unknownSet.erase(newReady);
 		m_readySet.insert(newReady);
 	}
 	m_unknown.clear();
+	invariants_WITH_LOCK();
 }

libethereum/BlockQueue.h

@@ -78,7 +78,7 @@ public:
 	bool doneDrain(h256s const& _knownBad = h256s());
 
 	/// Notify the queue that the chain has changed and a new block has attained 'ready' status (i.e. is in the chain).
-	void noteReady(h256 _b) { WriteGuard l(m_lock); noteReadyWithoutWriteGuard(_b); }
+	void noteReady(h256 const& _b) { WriteGuard l(m_lock); noteReady_WITH_LOCK(_b); }
 
 	/// Force a retry of all the blocks with unknown parents.
 	void retryAllUnknown();
@@ -87,7 +87,7 @@ public:
 	std::pair<unsigned, unsigned> items() const { ReadGuard l(m_lock); return std::make_pair(m_ready.size(), m_unknown.size()); }
 
 	/// Clear everything.
-	void clear() { WriteGuard l(m_lock); m_readySet.clear(); m_drainingSet.clear(); m_ready.clear(); m_unknownSet.clear(); m_unknown.clear(); m_future.clear(); }
+	void clear() { WriteGuard l(m_lock); invariants_WITH_LOCK(); m_readySet.clear(); m_drainingSet.clear(); m_ready.clear(); m_unknownSet.clear(); m_unknown.clear(); m_future.clear(); invariants_WITH_LOCK(); }
 
 	/// Return first block with an unknown parent.
 	h256 firstUnknown() const { ReadGuard l(m_lock); return m_unknownSet.size() ? *m_unknownSet.begin() : h256(); }
@@ -101,18 +101,18 @@ public:
 	template <class T> Handler onReady(T const& _t) { return m_onReady.add(_t); }
 
 private:
-	void noteReadyWithoutWriteGuard(h256 _b);
+	void noteReady_WITH_LOCK(h256 const& _b);
-	void notePresentWithoutWriteGuard(bytesConstRef _block);
+	void invariants_WITH_LOCK() const;
 
 	mutable boost::shared_mutex m_lock;						///< General lock.
-	std::set<h256> m_readySet;								///< All blocks ready for chain-import.
 	std::set<h256> m_drainingSet;							///< All blocks being imported.
-	std::vector<bytes> m_ready;								///< List of blocks, in correct order, ready for chain-import.
+	std::set<h256> m_readySet;								///< All blocks ready for chain-import.
+	std::vector<std::pair<h256, bytes>> m_ready;			///< List of blocks, in correct order, ready for chain-import.
 	std::set<h256> m_unknownSet;							///< Set of all blocks whose parents are not ready/in-chain.
-	std::multimap<h256, std::pair<h256, bytes>> m_unknown;	///< For transactions that have an unknown parent; we map their parent hash to the block stuff, and insert once the block appears.
+	std::multimap<h256, std::pair<h256, bytes>> m_unknown;	///< For blocks that have an unknown parent; we map their parent hash to the block stuff, and insert once the block appears.
-	std::multimap<unsigned, bytes> m_future;				///< Set of blocks that are not yet valid.
 	std::set<h256> m_knownBad;								///< Set of blocks that we know will never be valid.
-	Signal m_onReady;										///< Called when a subsequent call to import transactions will return a non-empty container. Be nice and exit fast.
+	std::multimap<unsigned, std::pair<h256, bytes>> m_future;///< Set of blocks that are not yet valid.
+	Signal m_onReady;										///< Called when a subsequent call to import blocks will return a non-empty container. Be nice and exit fast.
 };
 
 }

libethereum/Client.cpp

@@ -582,11 +582,12 @@ void Client::onChainChanged(ImportRoute const& _ir)
 			m_preMine = newPreMine;
 		DEV_TIMED(working) ETH_WRITE_GUARDED(x_working)
 			m_working = newPreMine;
+//		Transactions ts = m_postMine.pending();
 		ETH_READ_GUARDED(x_postMine)
 			for (auto const& t: m_postMine.pending())
 			{
 				clog(ClientNote) << "Resubmitting post-mine transaction " << t;
-				m_tq.import(t.rlp(), TransactionQueue::ImportCallback(), IfDropped::Retry);
+				m_tq.import(t, TransactionQueue::ImportCallback(), IfDropped::Retry);
 			}
 		ETH_READ_GUARDED(x_working) DEV_TIMED(post) ETH_WRITE_GUARDED(x_postMine)
 			m_postMine = m_working;
@@ -662,7 +663,7 @@ void Client::doWork()
 
 	bool t = true;
 	if (m_syncBlockQueue.compare_exchange_strong(t, false))
-		syncBlockQueue();
+		syncBlockQueue();	// GAAA!!!!! CALLED TOO OFTEN!!!
 
 	t = true;
 	if (m_syncTransactionQueue.compare_exchange_strong(t, false) && !m_remoteWorking)
@@ -735,6 +736,7 @@ eth::State Client::state(unsigned _txi) const
 	ETH_READ_GUARDED(x_postMine)
 		return m_postMine.fromPending(_txi);
 	assert(false);
+	return State();
 }
 
 void Client::flushTransactions()

libethereum/State.cpp

@@ -152,7 +152,7 @@ State::State(OverlayDB const& _db, BlockChain const& _bc, h256 _h):
 
 State::State(State const& _s):
 	m_db(_s.m_db),
-	m_state(&m_db, _s.m_state.root()),
+	m_state(&m_db, _s.m_state.root(), Verification::Skip),
 	m_transactions(_s.m_transactions),
 	m_receipts(_s.m_receipts),
 	m_transactionSet(_s.m_transactionSet),
@@ -184,7 +184,7 @@ void State::paranoia(std::string const& _when, bool _enforceRefs) const
 State& State::operator=(State const& _s)
 {
 	m_db = _s.m_db;
-	m_state.open(&m_db, _s.m_state.root());
+	m_state.open(&m_db, _s.m_state.root(), Verification::Skip);
 	m_transactions = _s.m_transactions;
 	m_receipts = _s.m_receipts;
 	m_transactionSet = _s.m_transactionSet;

libethereum/Transaction.h

@@ -149,7 +149,7 @@ public:
 	bytes rlp(IncludeSignature _sig = WithSignature) const { RLPStream s; streamRLP(s, _sig); return s.out(); }
 
 	/// @returns the SHA3 hash of the RLP serialisation of this transaction.
-	h256 sha3(IncludeSignature _sig = WithSignature) const { RLPStream s; streamRLP(s, _sig); return dev::sha3(s.out()); }
+	h256 sha3(IncludeSignature _sig = WithSignature) const { if (_sig == WithSignature && m_hashWith) return m_hashWith; RLPStream s; streamRLP(s, _sig); auto ret = dev::sha3(s.out()); if (_sig == WithSignature) m_hashWith = ret; return ret; }
 
 	/// @returns the amount of ETH to be transferred by this (message-call) transaction, in Wei. Synonym for endowment().
 	u256 value() const { return m_value; }
@@ -211,6 +211,7 @@ private:
 	bytes m_data;						///< The data associated with the transaction, or the initialiser if it's a creation transaction.
 	SignatureStruct m_vrs;				///< The signature of the transaction. Encodes the sender.
 
+	mutable h256 m_hashWith;			///< Cached hash of transaction with signature.
 	mutable Address m_sender;			///< Cached sender, determined from signature.
 	mutable bigint m_gasRequired = 0;	///< Memoised amount required for the transaction to run.
 };

libethereum/TransactionQueue.cpp

@@ -38,26 +38,56 @@ ImportResult TransactionQueue::import(bytesConstRef _transactionRLP, ImportCallb
 	UpgradableGuard l(m_lock);
 	// TODO: keep old transactions around and check in State for nonce validity
 
-	if (m_known.count(h))
+	auto ir = check_WITH_LOCK(h, _ik);
+	if (ir != ImportResult::Success)
+		return ir;
+
+	Transaction t(_transactionRLP, CheckTransaction::Everything);
+	UpgradeGuard ul(l);
+	return manageImport_WITH_LOCK(h, t, _cb);
+}
+
+ImportResult TransactionQueue::check_WITH_LOCK(h256 const& _h, IfDropped _ik)
+{
+	if (m_known.count(_h))
 		return ImportResult::AlreadyKnown;
 
-	if (m_dropped.count(h) && _ik == IfDropped::Ignore)
+	if (m_dropped.count(_h) && _ik == IfDropped::Ignore)
 		return ImportResult::AlreadyInChain;
 
+	return ImportResult::Success;
+}
+
+ImportResult TransactionQueue::import(Transaction const& _transaction, ImportCallback const& _cb, IfDropped _ik)
+{
+	// Check if we already know this transaction.
+	h256 h = _transaction.sha3(WithSignature);
+
+	UpgradableGuard l(m_lock);
+	// TODO: keep old transactions around and check in State for nonce validity
+
+	auto ir = check_WITH_LOCK(h, _ik);
+	if (ir != ImportResult::Success)
+		return ir;
+
+	UpgradeGuard ul(l);
+	return manageImport_WITH_LOCK(h, _transaction, _cb);
+}
+
+ImportResult TransactionQueue::manageImport_WITH_LOCK(h256 const& _h, Transaction const& _transaction, ImportCallback const& _cb)
+{
 	try
 	{
 		// Check validity of _transactionRLP as a transaction. To do this we just deserialise and attempt to determine the sender.
 		// If it doesn't work, the signature is bad.
 		// The transaction's nonce may yet be invalid (or, it could be "valid" but we may be missing a marginally older transaction).
-		Transaction t(_transactionRLP, CheckTransaction::Everything);
 
-		UpgradeGuard ul(l);
 		// If valid, append to blocks.
-		insertCurrent_WITH_LOCK(make_pair(h, t));
+		insertCurrent_WITH_LOCK(make_pair(_h, _transaction));
-		m_known.insert(h);
+		m_known.insert(_h);
 		if (_cb)
-			m_callbacks[h] = _cb;
+			m_callbacks[_h] = _cb;
-		ctxq << "Queued vaguely legit-looking transaction" << h;
+		ctxq << "Queued vaguely legit-looking transaction" << _h;
 		m_onReady();
 	}
 	catch (Exception const& _e)

libethereum/TransactionQueue.h

@@ -49,6 +49,7 @@ class TransactionQueue
 public:
 	using ImportCallback = std::function<void(ImportResult)>;
 
+	ImportResult import(Transaction const& _tx, ImportCallback const& _cb = ImportCallback(), IfDropped _ik = IfDropped::Ignore);
 	ImportResult import(bytes const& _tx, ImportCallback const& _cb = ImportCallback(), IfDropped _ik = IfDropped::Ignore) { return import(&_tx, _cb, _ik); }
 	ImportResult import(bytesConstRef _tx, ImportCallback const& _cb = ImportCallback(), IfDropped _ik = IfDropped::Ignore);
 
@@ -65,6 +66,9 @@ public:
 	template <class T> Handler onReady(T const& _t) { return m_onReady.add(_t); }
 
 private:
+	ImportResult check_WITH_LOCK(h256 const& _h, IfDropped _ik);
+	ImportResult manageImport_WITH_LOCK(h256 const& _h, Transaction const& _transaction, ImportCallback const& _cb);
+
 	void insertCurrent_WITH_LOCK(std::pair<h256, Transaction> const& _p);
 	bool removeCurrent_WITH_LOCK(h256 const& _txHash);