performance optimizations

libethereum/BlockChain.cpp

@@ -84,17 +84,22 @@ std::ostream& dev::eth::operator<<(std::ostream& _out, BlockChain const& _bc)
 
 ldb::Slice dev::eth::toSlice(h256 const& _h, unsigned _sub)
 {
-#if ALL_COMPILERS_ARE_CPP11_COMPLIANT
-	static thread_local h256 h = _h ^ sha3(h256(u256(_sub)));
-	return ldb::Slice((char const*)&h, 32);
-#else
 	static boost::thread_specific_ptr<FixedHash<33>> t_h;
 	if (!t_h.get())
 		t_h.reset(new FixedHash<33>);
 	*t_h = FixedHash<33>(_h);
 	(*t_h)[32] = (uint8_t)_sub;
 	return (ldb::Slice)t_h->ref();//(char const*)t_h.get(), 32);
-#endif
+}
+
+ldb::Slice dev::eth::toSlice(uint64_t _n, unsigned _sub)
+{
+	static boost::thread_specific_ptr<FixedHash<33>> t_h;
+	if (!t_h.get())
+		t_h.reset(new FixedHash<33>);
+	toBigEndian(_n, bytesRef(t_h->data() + 24, 8));
+	(*t_h)[32] = (uint8_t)_sub;
+	return (ldb::Slice)t_h->ref();
 }
 
 namespace dev
@@ -289,7 +294,7 @@ void BlockChain::rebuild(std::string const& _path, std::function<void(unsigned,
 		}
 		try
 		{
-			bytes b = block(queryExtras<BlockHash, ExtraBlockHash>(h256(u256(d)), m_blockHashes, x_blockHashes, NullBlockHash, oldExtrasDB).value);
+			bytes b = block(queryExtras<BlockHash, uint64_t, ExtraBlockHash>(d, m_blockHashes, x_blockHashes, NullBlockHash, oldExtrasDB).value);
 
 			BlockInfo bi(&b);
 			if (_prepPoW)
@@ -359,7 +364,8 @@ tuple<ImportRoute, bool, unsigned> BlockChain::sync(BlockQueue& _bq, OverlayDB c
 					r = import(block.verified, _stateDB, ImportRequirements::Everything & ~ImportRequirements::ValidSeal & ~ImportRequirements::CheckUncles);
 				fresh += r.liveBlocks;
 				dead += r.deadBlocks;
-				goodTransactions += r.goodTranactions;
+				goodTransactions.reserve(goodTransactions.size() + r.goodTranactions.size());
+				std::move(std::begin(r.goodTranactions), std::end(r.goodTranactions), std::back_inserter(goodTransactions));
 				++count;
 			}
 			catch (dev::eth::UnknownParent)
@@ -947,7 +953,7 @@ void BlockChain::noteUsed(h256 const& _h, unsigned _extra) const
 		m_inUse.insert(id);
 }
 
-template <class T> static unsigned getHashSize(unordered_map<h256, T> const& _map)
+template <class K, class T> static unsigned getHashSize(unordered_map<K, T> const& _map)
 {
 	unsigned ret = 0;
 	for (auto const& i: _map)
@@ -1005,9 +1011,6 @@ void BlockChain::garbageCollect(bool _force)
 		case ExtraDetails:
 			m_details.erase(id.first);
 			break;
-		case ExtraBlockHash:
-			m_blockHashes.erase(id.first);
-			break;
 		case ExtraReceipts:
 			m_receipts.erase(id.first);
 			break;

libethereum/BlockChain.h

@@ -72,6 +72,7 @@ struct BlockChainDebug: public LogChannel { static const char* name(); static co
 std::unordered_map<Address, Account> const& genesisState();
 
 ldb::Slice toSlice(h256 const& _h, unsigned _sub = 0);
+ldb::Slice toSlice(uint64_t _n, unsigned _sub = 0);
 
 using BlocksHash = std::unordered_map<h256, bytes>;
 using TransactionHashes = h256s;
@@ -168,7 +169,7 @@ public:
 	UncleHashes uncleHashes() const { return uncleHashes(currentHash()); }
 	
 	/// Get the hash for a given block's number.
-	h256 numberHash(unsigned _i) const { if (!_i) return genesisHash(); return queryExtras<BlockHash, ExtraBlockHash>(h256(_i), m_blockHashes, x_blockHashes, NullBlockHash).value; }
+	h256 numberHash(unsigned _i) const { if (!_i) return genesisHash(); return queryExtras<BlockHash, uint64_t, ExtraBlockHash>(_i, m_blockHashes, x_blockHashes, NullBlockHash).value; }
 
 	/// Get the last N hashes for a given block. (N is determined by the LastHashes type.)
 	LastHashes lastHashes() const { return lastHashes(number()); }
@@ -293,7 +294,7 @@ protected:
 	unsigned open(std::string const& _path, WithExisting _we = WithExisting::Trust);
 	void close();
 
-	template<class T, unsigned N> T queryExtras(h256 const& _h, std::unordered_map<h256, T>& _m, boost::shared_mutex& _x, T const& _n, ldb::DB* _extrasDB = nullptr) const
+	template<class T, class K, unsigned N> T queryExtras(K const& _h, std::unordered_map<K, T>& _m, boost::shared_mutex& _x, T const& _n, ldb::DB* _extrasDB = nullptr) const
 	{
 		{
 			ReadGuard l(_x);
@@ -305,10 +306,7 @@ protected:
 		std::string s;
 		(_extrasDB ? _extrasDB : m_extrasDB)->Get(m_readOptions, toSlice(_h, N), &s);
 		if (s.empty())
-		{
-//			cout << "Not found in DB: " << _h << endl;
 			return _n;
-		}
 
 		noteUsed(_h, N);
 
@@ -317,6 +315,11 @@ protected:
 		return ret.first->second;
 	}
 
+	template<class T, unsigned N> T queryExtras(h256 const& _h, std::unordered_map<h256, T>& _m, boost::shared_mutex& _x, T const& _n, ldb::DB* _extrasDB = nullptr) const
+	{
+		return queryExtras<T, h256, N>(_h, _m, _x, _n, _extrasDB);
+	}
+
 	void checkConsistency();
 
 	/// The caches of the disk DB and their locks.
@@ -340,6 +343,7 @@ protected:
 	mutable std::deque<std::unordered_set<CacheID>> m_cacheUsage;
 	mutable std::unordered_set<CacheID> m_inUse;
 	void noteUsed(h256 const& _h, unsigned _extra = (unsigned)-1) const;
+	void noteUsed(uint64_t const& _h, unsigned _extra = (unsigned)-1) const { (void)_h; (void)_extra; } // don't note non-hash types
 	std::chrono::system_clock::time_point m_lastCollection;
 
 	void noteCanonChanged() const { Guard l(x_lastLastHashes); m_lastLastHashes.clear(); }

libethereum/BlockDetails.h

@@ -114,7 +114,7 @@ using BlockDetailsHash = std::unordered_map<h256, BlockDetails>;
 using BlockLogBloomsHash = std::unordered_map<h256, BlockLogBlooms>;
 using BlockReceiptsHash = std::unordered_map<h256, BlockReceipts>;
 using TransactionAddressHash = std::unordered_map<h256, TransactionAddress>;
-using BlockHashHash = std::unordered_map<h256, BlockHash>;
+using BlockHashHash = std::unordered_map<uint64_t, BlockHash>;
 using BlocksBloomsHash = std::unordered_map<h256, BlocksBlooms>;
 
 static const BlockDetails NullBlockDetails;

libethereum/BlockQueue.cpp

@@ -292,7 +292,7 @@ void BlockQueue::updateBad_WITH_LOCK(h256 const& _bad)
 		while (moreBad)
 		{
 			moreBad = false;
-			std::vector<VerifiedBlock> oldVerified;
+			std::deque<VerifiedBlock> oldVerified;
 			swap(m_verified, oldVerified);
 			for (auto& b: oldVerified)
 				if (m_knownBad.count(b.verified.info.parentHash()) || m_knownBad.count(b.verified.info.hash()))

libethereum/BlockQueue.h

@@ -154,7 +154,7 @@ private:
 
 	mutable Mutex m_verification;										///< Mutex that allows writing to m_verified, m_verifying and m_unverified.
 	std::condition_variable m_moreToVerify;								///< Signaled when m_unverified has a new entry.
-	std::vector<VerifiedBlock> m_verified;								///< List of blocks, in correct order, verified and ready for chain-import.
+	std::deque<VerifiedBlock> m_verified;								///< List of blocks, in correct order, verified and ready for chain-import.
 	std::deque<VerifiedBlock> m_verifying;								///< List of blocks being verified; as long as the block component (bytes) is empty, it's not finished.
 	std::deque<UnverifiedBlock> m_unverified;							///< List of <block hash, parent hash, block data> in correct order, ready for verification.
 

libethereum/DownloadMan.cpp

@@ -24,6 +24,8 @@ using namespace std;
 using namespace dev;
 using namespace dev::eth;
 
+size_t const c_maxDownloadAhead = 50000; // Must not be higher than BlockQueue::c_maxUnknownCount
+
 DownloadMan::Overview DownloadMan::overview() const
 {
 	ReadGuard l(m_lock);
@@ -63,9 +65,10 @@ h256Hash DownloadSub::nextFetch(unsigned _n)
 	if (!m_man || m_man->chainEmpty())
 		return h256Hash();
 
-	m_asked = (~(m_man->taken() + m_attempted)).lowest(_n);
-	if (m_asked.empty())
-		m_asked = (~(m_man->taken(true) + m_attempted)).lowest(_n);
+	RangeMask<unsigned> downloaded = m_man->taken(true);
+	m_asked = (~(m_man->taken(false) + m_attempted)).lowest(_n);
+	if (m_asked.empty() || m_asked.lastIn() - downloaded.firstOut() >= c_maxDownloadAhead)
+		m_asked = (~(downloaded + m_attempted)).lowest(_n);
 	m_attempted += m_asked;
 	for (auto i: m_asked)
 	{
@@ -85,50 +88,3 @@ bool DownloadSub::noteBlock(h256 _hash)
 	m_remaining.erase(_hash);
 	return ret;
 }
-
-HashDownloadSub::HashDownloadSub(HashDownloadMan& _man): m_man(&_man)
-{
-	WriteGuard l(m_man->x_subs);
-	m_asked = RangeMask<unsigned>(m_man->m_chainStart, m_man->m_chainStart + m_man->m_chainCount);
-	m_man->m_subs.insert(this);
-}
-
-HashDownloadSub::~HashDownloadSub()
-{
-	if (m_man)
-	{
-		WriteGuard l(m_man->x_subs);
-		m_man->m_subs.erase(this);
-	}
-}
-
-void HashDownloadSub::resetFetch()
-{
-	Guard l(m_fetch);
-	m_remaining = 0;
-	m_asked = RangeMask<unsigned>(m_man->m_chainStart, m_man->m_chainStart + m_man->m_chainCount);
-}
-
-unsigned HashDownloadSub::nextFetch(unsigned _n)
-{
-	Guard l(m_fetch);
-
-	m_asked = RangeMask<unsigned>(m_man->m_chainStart, m_man->m_chainStart + m_man->m_chainCount);
-
-	if (!m_man || m_man->chainEmpty())
-		return 0;
-
-	m_asked = (~(m_man->taken())).lowest(_n);
-	if (m_asked.empty())
-		m_asked = (~(m_man->taken(true))).lowest(_n);
-	return *m_asked.begin();
-}
-
-void HashDownloadSub::noteHash(unsigned _index, unsigned _size)
-{
-	Guard l(m_fetch);
-	if (m_man)
-		for(unsigned i = _index; i < _index + _size; ++i)
-			if (i >= m_man->m_got.all().first && i < m_man->m_got.all().second)
-				m_man->m_got += i;
-}

libethereum/DownloadMan.h

@@ -170,110 +170,6 @@ private:
 	std::unordered_set<DownloadSub*> m_subs;
 };
 
-
-class HashDownloadMan;
-
-class HashDownloadSub
-{
-	friend class HashDownloadMan;
-
-public:
-	HashDownloadSub(HashDownloadMan& _man);
-	~HashDownloadSub();
-
-	/// Finished last fetch - grab the next hash index to download
-	unsigned nextFetch(unsigned _n);
-
-	/// Note that we've received a particular hash range.
-	void noteHash(unsigned _index, unsigned count);
-
-	/// Nothing doing here.
-	void doneFetch() { resetFetch(); }
-
-	bool askedContains(unsigned _i) const { Guard l(m_fetch); return m_asked.contains(_i); }
-	RangeMask<unsigned> const& asked() const { return m_asked; }
-
-private:
-	void resetFetch();		// Called by DownloadMan when we need to reset the download.
-
-	HashDownloadMan* m_man = nullptr;
-	mutable Mutex m_fetch;
-	unsigned m_remaining;
-	RangeMask<unsigned> m_asked;
-};
-
-class HashDownloadMan
-{
-	friend class HashDownloadSub;
-
-public:
-	~HashDownloadMan()
-	{
-		for (auto i: m_subs)
-			i->m_man = nullptr;
-	}
-
-	void resetToRange(unsigned _start, unsigned _count)
-	{
-		{
-			ReadGuard l(x_subs);
-			for (auto i: m_subs)
-				i->resetFetch();
-		}
-		WriteGuard l(m_lock);
-		m_chainStart = _start;
-		m_chainCount = _count;
-		m_got += RangeMask<unsigned>(_start, _start + _count);
-		{
-			ReadGuard l(x_subs);
-			for (auto i: m_subs)
-				i->resetFetch();
-		}
-	}
-
-	void reset(unsigned _start)
-	{
-		WriteGuard l(m_lock);
-		m_chainStart = _start;
-		m_chainCount = 0;
-		m_got = RangeMask<unsigned>(_start, _start);
-	}
-
-	RangeMask<unsigned> taken(bool _desperate = false) const
-	{
-		ReadGuard l(m_lock);
-		auto ret = m_got;
-		if (!_desperate)
-		{
-			ReadGuard l(x_subs);
-			for (auto i: m_subs)
-				ret += i->m_asked;
-		}
-		return ret;
-	}
-
-	bool isComplete() const
-	{
-		ReadGuard l(m_lock);
-		return m_got.full();
-	}
-
-	size_t chainSize() const { ReadGuard l(m_lock); return m_chainCount; }
-	size_t chainEmpty() const { ReadGuard l(m_lock); return m_chainCount == 0; }
-	void foreachSub(std::function<void(HashDownloadSub const&)> const& _f) const { ReadGuard l(x_subs); for(auto i: m_subs) _f(*i); }
-	unsigned subCount() const { ReadGuard l(x_subs); return m_subs.size(); }
-	RangeMask<unsigned> hashesGot() const { ReadGuard l(m_lock); return m_got; }
-
-private:
-	mutable SharedMutex m_lock;
-	unsigned m_chainStart = 0;
-	unsigned m_chainCount = 0;
-	RangeMask<unsigned> m_got;
-
-	mutable SharedMutex x_subs;
-	std::unordered_set<HashDownloadSub*> m_subs;
-};
-
 }
 
 }

libethereum/TransactionQueue.cpp

@@ -107,6 +107,7 @@ ImportResult TransactionQueue::import(Transaction const& _transaction, IfDropped
 			return ir;
 
 		{
+			_transaction.safeSender(); // Perform EC recovery outside of the write lock
 			UpgradeGuard ul(l);
 			ret = manageImport_WITH_LOCK(h, _transaction);
 		}