From 3bd1aecd2dd1a16aa63f61c642bdd49c47a96b3f Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 16 Jul 2015 09:57:27 +0200 Subject: [PATCH] performance optimizations --- libethereum/BlockChain.cpp | 25 ++++---- libethereum/BlockChain.h | 14 +++-- libethereum/BlockDetails.h | 2 +- libethereum/BlockQueue.cpp | 2 +- libethereum/BlockQueue.h | 2 +- libethereum/DownloadMan.cpp | 56 ++--------------- libethereum/DownloadMan.h | 104 ------------------------------- libethereum/TransactionQueue.cpp | 1 + 8 files changed, 33 insertions(+), 173 deletions(-) diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index a992c5851..00117e679 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -84,17 +84,22 @@ std::ostream& dev::eth::operator<<(std::ostream& _out, BlockChain const& _bc) ldb::Slice dev::eth::toSlice(h256 const& _h, unsigned _sub) { -#if ALL_COMPILERS_ARE_CPP11_COMPLIANT - static thread_local h256 h = _h ^ sha3(h256(u256(_sub))); - return ldb::Slice((char const*)&h, 32); -#else static boost::thread_specific_ptr> t_h; if (!t_h.get()) t_h.reset(new FixedHash<33>); *t_h = FixedHash<33>(_h); (*t_h)[32] = (uint8_t)_sub; return (ldb::Slice)t_h->ref();//(char const*)t_h.get(), 32); -#endif +} + +ldb::Slice dev::eth::toSlice(uint64_t _n, unsigned _sub) +{ + static boost::thread_specific_ptr> t_h; + if (!t_h.get()) + t_h.reset(new FixedHash<33>); + toBigEndian(_n, bytesRef(t_h->data() + 24, 8)); + (*t_h)[32] = (uint8_t)_sub; + return (ldb::Slice)t_h->ref(); } namespace dev @@ -289,7 +294,7 @@ void BlockChain::rebuild(std::string const& _path, std::function(h256(u256(d)), m_blockHashes, x_blockHashes, NullBlockHash, oldExtrasDB).value); + bytes b = block(queryExtras(d, m_blockHashes, x_blockHashes, NullBlockHash, oldExtrasDB).value); BlockInfo bi(&b); if (_prepPoW) @@ -359,7 +364,8 @@ tuple BlockChain::sync(BlockQueue& _bq, OverlayDB c r = import(block.verified, _stateDB, ImportRequirements::Everything & ~ImportRequirements::ValidSeal & ~ImportRequirements::CheckUncles); fresh += r.liveBlocks; dead += r.deadBlocks; - goodTransactions += r.goodTranactions; + goodTransactions.reserve(goodTransactions.size() + r.goodTranactions.size()); + std::move(std::begin(r.goodTranactions), std::end(r.goodTranactions), std::back_inserter(goodTransactions)); ++count; } catch (dev::eth::UnknownParent) @@ -947,7 +953,7 @@ void BlockChain::noteUsed(h256 const& _h, unsigned _extra) const m_inUse.insert(id); } -template static unsigned getHashSize(unordered_map const& _map) +template static unsigned getHashSize(unordered_map const& _map) { unsigned ret = 0; for (auto const& i: _map) @@ -1005,9 +1011,6 @@ void BlockChain::garbageCollect(bool _force) case ExtraDetails: m_details.erase(id.first); break; - case ExtraBlockHash: - m_blockHashes.erase(id.first); - break; case ExtraReceipts: m_receipts.erase(id.first); break; diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h index d60be548a..415581f29 100644 --- a/libethereum/BlockChain.h +++ b/libethereum/BlockChain.h @@ -72,6 +72,7 @@ struct BlockChainDebug: public LogChannel { static const char* name(); static co std::unordered_map const& genesisState(); ldb::Slice toSlice(h256 const& _h, unsigned _sub = 0); +ldb::Slice toSlice(uint64_t _n, unsigned _sub = 0); using BlocksHash = std::unordered_map; using TransactionHashes = h256s; @@ -168,7 +169,7 @@ public: UncleHashes uncleHashes() const { return uncleHashes(currentHash()); } /// Get the hash for a given block's number. - h256 numberHash(unsigned _i) const { if (!_i) return genesisHash(); return queryExtras(h256(_i), m_blockHashes, x_blockHashes, NullBlockHash).value; } + h256 numberHash(unsigned _i) const { if (!_i) return genesisHash(); return queryExtras(_i, m_blockHashes, x_blockHashes, NullBlockHash).value; } /// Get the last N hashes for a given block. (N is determined by the LastHashes type.) LastHashes lastHashes() const { return lastHashes(number()); } @@ -293,7 +294,7 @@ protected: unsigned open(std::string const& _path, WithExisting _we = WithExisting::Trust); void close(); - template T queryExtras(h256 const& _h, std::unordered_map& _m, boost::shared_mutex& _x, T const& _n, ldb::DB* _extrasDB = nullptr) const + template T queryExtras(K const& _h, std::unordered_map& _m, boost::shared_mutex& _x, T const& _n, ldb::DB* _extrasDB = nullptr) const { { ReadGuard l(_x); @@ -305,10 +306,7 @@ protected: std::string s; (_extrasDB ? _extrasDB : m_extrasDB)->Get(m_readOptions, toSlice(_h, N), &s); if (s.empty()) - { -// cout << "Not found in DB: " << _h << endl; return _n; - } noteUsed(_h, N); @@ -317,6 +315,11 @@ protected: return ret.first->second; } + template T queryExtras(h256 const& _h, std::unordered_map& _m, boost::shared_mutex& _x, T const& _n, ldb::DB* _extrasDB = nullptr) const + { + return queryExtras(_h, _m, _x, _n, _extrasDB); + } + void checkConsistency(); /// The caches of the disk DB and their locks. @@ -340,6 +343,7 @@ protected: mutable std::deque> m_cacheUsage; mutable std::unordered_set m_inUse; void noteUsed(h256 const& _h, unsigned _extra = (unsigned)-1) const; + void noteUsed(uint64_t const& _h, unsigned _extra = (unsigned)-1) const { (void)_h; (void)_extra; } // don't note non-hash types std::chrono::system_clock::time_point m_lastCollection; void noteCanonChanged() const { Guard l(x_lastLastHashes); m_lastLastHashes.clear(); } diff --git a/libethereum/BlockDetails.h b/libethereum/BlockDetails.h index 73026834e..81f3cecdb 100644 --- a/libethereum/BlockDetails.h +++ b/libethereum/BlockDetails.h @@ -114,7 +114,7 @@ using BlockDetailsHash = std::unordered_map; using BlockLogBloomsHash = std::unordered_map; using BlockReceiptsHash = std::unordered_map; using TransactionAddressHash = std::unordered_map; -using BlockHashHash = std::unordered_map; +using BlockHashHash = std::unordered_map; using BlocksBloomsHash = std::unordered_map; static const BlockDetails NullBlockDetails; diff --git a/libethereum/BlockQueue.cpp b/libethereum/BlockQueue.cpp index d0ca34b1c..97352fbf1 100644 --- a/libethereum/BlockQueue.cpp +++ b/libethereum/BlockQueue.cpp @@ -292,7 +292,7 @@ void BlockQueue::updateBad_WITH_LOCK(h256 const& _bad) while (moreBad) { moreBad = false; - std::vector oldVerified; + std::deque oldVerified; swap(m_verified, oldVerified); for (auto& b: oldVerified) if (m_knownBad.count(b.verified.info.parentHash()) || m_knownBad.count(b.verified.info.hash())) diff --git a/libethereum/BlockQueue.h b/libethereum/BlockQueue.h index 97fca7c72..0d90aac21 100644 --- a/libethereum/BlockQueue.h +++ b/libethereum/BlockQueue.h @@ -154,7 +154,7 @@ private: mutable Mutex m_verification; ///< Mutex that allows writing to m_verified, m_verifying and m_unverified. std::condition_variable m_moreToVerify; ///< Signaled when m_unverified has a new entry. - std::vector m_verified; ///< List of blocks, in correct order, verified and ready for chain-import. + std::deque m_verified; ///< List of blocks, in correct order, verified and ready for chain-import. std::deque m_verifying; ///< List of blocks being verified; as long as the block component (bytes) is empty, it's not finished. std::deque m_unverified; ///< List of in correct order, ready for verification. diff --git a/libethereum/DownloadMan.cpp b/libethereum/DownloadMan.cpp index a9d353292..84931def2 100644 --- a/libethereum/DownloadMan.cpp +++ b/libethereum/DownloadMan.cpp @@ -24,6 +24,8 @@ using namespace std; using namespace dev; using namespace dev::eth; +size_t const c_maxDownloadAhead = 50000; // Must not be higher than BlockQueue::c_maxUnknownCount + DownloadMan::Overview DownloadMan::overview() const { ReadGuard l(m_lock); @@ -63,9 +65,10 @@ h256Hash DownloadSub::nextFetch(unsigned _n) if (!m_man || m_man->chainEmpty()) return h256Hash(); - m_asked = (~(m_man->taken() + m_attempted)).lowest(_n); - if (m_asked.empty()) - m_asked = (~(m_man->taken(true) + m_attempted)).lowest(_n); + RangeMask downloaded = m_man->taken(true); + m_asked = (~(m_man->taken(false) + m_attempted)).lowest(_n); + if (m_asked.empty() || m_asked.lastIn() - downloaded.firstOut() >= c_maxDownloadAhead) + m_asked = (~(downloaded + m_attempted)).lowest(_n); m_attempted += m_asked; for (auto i: m_asked) { @@ -85,50 +88,3 @@ bool DownloadSub::noteBlock(h256 _hash) m_remaining.erase(_hash); return ret; } - -HashDownloadSub::HashDownloadSub(HashDownloadMan& _man): m_man(&_man) -{ - WriteGuard l(m_man->x_subs); - m_asked = RangeMask(m_man->m_chainStart, m_man->m_chainStart + m_man->m_chainCount); - m_man->m_subs.insert(this); -} - -HashDownloadSub::~HashDownloadSub() -{ - if (m_man) - { - WriteGuard l(m_man->x_subs); - m_man->m_subs.erase(this); - } -} - -void HashDownloadSub::resetFetch() -{ - Guard l(m_fetch); - m_remaining = 0; - m_asked = RangeMask(m_man->m_chainStart, m_man->m_chainStart + m_man->m_chainCount); -} - -unsigned HashDownloadSub::nextFetch(unsigned _n) -{ - Guard l(m_fetch); - - m_asked = RangeMask(m_man->m_chainStart, m_man->m_chainStart + m_man->m_chainCount); - - if (!m_man || m_man->chainEmpty()) - return 0; - - m_asked = (~(m_man->taken())).lowest(_n); - if (m_asked.empty()) - m_asked = (~(m_man->taken(true))).lowest(_n); - return *m_asked.begin(); -} - -void HashDownloadSub::noteHash(unsigned _index, unsigned _size) -{ - Guard l(m_fetch); - if (m_man) - for(unsigned i = _index; i < _index + _size; ++i) - if (i >= m_man->m_got.all().first && i < m_man->m_got.all().second) - m_man->m_got += i; -} diff --git a/libethereum/DownloadMan.h b/libethereum/DownloadMan.h index b697d87ae..b6ca58845 100644 --- a/libethereum/DownloadMan.h +++ b/libethereum/DownloadMan.h @@ -170,110 +170,6 @@ private: std::unordered_set m_subs; }; - -class HashDownloadMan; - -class HashDownloadSub -{ - friend class HashDownloadMan; - -public: - HashDownloadSub(HashDownloadMan& _man); - ~HashDownloadSub(); - - /// Finished last fetch - grab the next hash index to download - unsigned nextFetch(unsigned _n); - - /// Note that we've received a particular hash range. - void noteHash(unsigned _index, unsigned count); - - /// Nothing doing here. - void doneFetch() { resetFetch(); } - - bool askedContains(unsigned _i) const { Guard l(m_fetch); return m_asked.contains(_i); } - RangeMask const& asked() const { return m_asked; } - -private: - void resetFetch(); // Called by DownloadMan when we need to reset the download. - - HashDownloadMan* m_man = nullptr; - mutable Mutex m_fetch; - unsigned m_remaining; - RangeMask m_asked; -}; - -class HashDownloadMan -{ - friend class HashDownloadSub; - -public: - ~HashDownloadMan() - { - for (auto i: m_subs) - i->m_man = nullptr; - } - - void resetToRange(unsigned _start, unsigned _count) - { - { - ReadGuard l(x_subs); - for (auto i: m_subs) - i->resetFetch(); - } - WriteGuard l(m_lock); - m_chainStart = _start; - m_chainCount = _count; - m_got += RangeMask(_start, _start + _count); - { - ReadGuard l(x_subs); - for (auto i: m_subs) - i->resetFetch(); - } - } - - void reset(unsigned _start) - { - WriteGuard l(m_lock); - m_chainStart = _start; - m_chainCount = 0; - m_got = RangeMask(_start, _start); - } - - RangeMask taken(bool _desperate = false) const - { - ReadGuard l(m_lock); - auto ret = m_got; - if (!_desperate) - { - ReadGuard l(x_subs); - for (auto i: m_subs) - ret += i->m_asked; - } - return ret; - } - - bool isComplete() const - { - ReadGuard l(m_lock); - return m_got.full(); - } - - size_t chainSize() const { ReadGuard l(m_lock); return m_chainCount; } - size_t chainEmpty() const { ReadGuard l(m_lock); return m_chainCount == 0; } - void foreachSub(std::function const& _f) const { ReadGuard l(x_subs); for(auto i: m_subs) _f(*i); } - unsigned subCount() const { ReadGuard l(x_subs); return m_subs.size(); } - RangeMask hashesGot() const { ReadGuard l(m_lock); return m_got; } - -private: - mutable SharedMutex m_lock; - unsigned m_chainStart = 0; - unsigned m_chainCount = 0; - RangeMask m_got; - - mutable SharedMutex x_subs; - std::unordered_set m_subs; -}; - } } diff --git a/libethereum/TransactionQueue.cpp b/libethereum/TransactionQueue.cpp index f461cae87..5de86c818 100644 --- a/libethereum/TransactionQueue.cpp +++ b/libethereum/TransactionQueue.cpp @@ -107,6 +107,7 @@ ImportResult TransactionQueue::import(Transaction const& _transaction, IfDropped return ir; { + _transaction.safeSender(); // Perform EC recovery outside of the write lock UpgradeGuard ul(l); ret = manageImport_WITH_LOCK(h, _transaction); }