From 0a5869739bef3bc51888b264e91412122aea43b7 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 5 Mar 2015 21:19:21 +0100 Subject: [PATCH] Caching. --- alethzero/Main.ui | 7 ++ alethzero/MainWin.cpp | 8 +++ alethzero/MainWin.h | 1 + libethereum/BlockChain.cpp | 124 +++++++++++++++++++++++++++++++++++ libethereum/BlockChain.h | 34 +++++++--- libethereum/BlockDetails.cpp | 5 +- libethereum/BlockDetails.h | 17 +++-- libethereum/Client.cpp | 1 + 8 files changed, 181 insertions(+), 16 deletions(-) diff --git a/alethzero/Main.ui b/alethzero/Main.ui index 9025aa846..8f6597ca5 100644 --- a/alethzero/Main.ui +++ b/alethzero/Main.ui @@ -38,6 +38,13 @@ + + + + 0 bytes used + + + diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index 8c86cc4b7..fc230f034 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -141,6 +141,7 @@ Main::Main(QWidget *parent) : ui->configDock->close(); on_verbosity_valueChanged(); + statusBar()->addPermanentWidget(ui->cacheUsage); statusBar()->addPermanentWidget(ui->balance); statusBar()->addPermanentWidget(ui->peerCount); statusBar()->addPermanentWidget(ui->mineStatus); @@ -1144,6 +1145,12 @@ void Main::on_refresh_triggered() refreshAll(); } +void Main::refreshCache() +{ + BlockChain::Statistics s = ethereum()->blockChain().usage(); + ui->cacheUsage->setText(QString("%1 bytes used").arg(s.memTotal())); +} + void Main::timerEvent(QTimerEvent*) { // 7/18, Alex: aggregating timers, prelude to better threading? @@ -1172,6 +1179,7 @@ void Main::timerEvent(QTimerEvent*) interval = 0; refreshNetwork(); refreshWhispers(); + refreshCache(); poll(); } else diff --git a/alethzero/MainWin.h b/alethzero/MainWin.h index b6a04ca6d..cc867f06b 100644 --- a/alethzero/MainWin.h +++ b/alethzero/MainWin.h @@ -212,6 +212,7 @@ private: void refreshNetwork(); void refreshMining(); void refreshWhispers(); + void refreshCache(); void refreshAll(); void refreshPending(); diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 10f44576b..4415120c4 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -76,8 +76,33 @@ ldb::Slice dev::eth::toSlice(h256 _h, unsigned _sub) #endif } +#if ETH_DEBUG +static const chrono::system_clock::duration c_collectionDuration = chrono::seconds(15); +static const unsigned c_collectionQueueSize = 2; +static const unsigned c_maxCacheSize = 1024 * 1024 * 1; +static const unsigned c_minCacheSize = 1; +#else + +/// Duration between flushes. +static const chrono::system_clock::duration c_collectionDuration = chrono::seconds(60); + +/// Length of death row (total time in cache is multiple of this and collection duration). +static const unsigned c_collectionQueueSize = 20; + +/// Max size, above which we start forcing cache reduction. +static const unsigned c_maxCacheSize = 1024 * 1024 * 64; + +/// Min size, below which we don't bother flushing it. +static const unsigned c_minCacheSize = 1024 * 1024 * 32; + +#endif + BlockChain::BlockChain(bytes const& _genesisBlock, std::string _path, bool _killExisting) { + // initialise deathrow. + m_cacheUsage.resize(c_collectionQueueSize); + m_lastCollection = chrono::system_clock::now(); + // Initialise with the genesis as the last block on the longest chain. m_genesisBlock = _genesisBlock; m_genesisHash = sha3(RLP(m_genesisBlock)[0].data()); @@ -428,6 +453,103 @@ h256s BlockChain::treeRoute(h256 _from, h256 _to, h256* o_common, bool _pre, boo return ret; } +void BlockChain::noteUsed(h256 const& _h, unsigned _extra) const +{ + auto id = CacheID(_h, _extra); + Guard l(x_cacheUsage); + m_cacheUsage[0].insert(id); + if (m_cacheUsage[1].count(id)) + m_cacheUsage[1].erase(id); + else + m_inUse.insert(id); +} + +template static unsigned getHashSize(map const& _map) +{ + unsigned ret = 0; + for (auto const& i: _map) + ret += i.second.size + 64; + return ret; +} + +void BlockChain::updateStats() const +{ + { + ReadGuard l1(x_blocks); + m_lastStats.memBlocks = 0; + for (auto const& i: m_blocks) + m_lastStats.memBlocks += i.second.size() + 64; + } + { + ReadGuard l2(x_details); + m_lastStats.memDetails = getHashSize(m_details); + } + { + ReadGuard l5(x_logBlooms); + m_lastStats.memLogBlooms = getHashSize(m_logBlooms); + } + { + ReadGuard l4(x_receipts); + m_lastStats.memReceipts = getHashSize(m_receipts); + } + { + ReadGuard l3(x_blockHashes); + m_lastStats.memBlockHashes = getHashSize(m_blockHashes); + } + { + ReadGuard l6(x_transactionAddresses); + m_lastStats.memTransactionAddresses = getHashSize(m_transactionAddresses); + } +} + +void BlockChain::garbageCollect(bool _force) +{ + updateStats(); + + if (!_force && chrono::system_clock::now() < m_lastCollection + c_collectionDuration && m_lastStats.memTotal() < c_maxCacheSize) + return; + if (m_lastStats.memTotal() < c_minCacheSize) + return; + + m_lastCollection = chrono::system_clock::now(); + + Guard l(x_cacheUsage); + WriteGuard l1(x_blocks); + WriteGuard l2(x_details); + WriteGuard l3(x_blockHashes); + WriteGuard l4(x_receipts); + WriteGuard l5(x_logBlooms); + WriteGuard l6(x_transactionAddresses); + for (CacheID const& id: m_cacheUsage.back()) + { + m_inUse.erase(id); + // kill i from cache. + switch (id.second) + { + case (unsigned)-1: + m_blocks.erase(id.first); + break; + case ExtraDetails: + m_details.erase(id.first); + break; + case ExtraBlockHash: + m_blockHashes.erase(id.first); + break; + case ExtraReceipts: + m_receipts.erase(id.first); + break; + case ExtraLogBlooms: + m_logBlooms.erase(id.first); + break; + case ExtraTransactionAddress: + m_transactionAddresses.erase(id.first); + break; + } + } + m_cacheUsage.pop_back(); + m_cacheUsage.push_front({}); +} + void BlockChain::checkConsistency() { { @@ -511,5 +633,7 @@ bytes BlockChain::block(h256 _hash) const m_blocks[_hash].resize(d.size()); memcpy(m_blocks[_hash].data(), d.data(), d.size()); + noteUsed(_hash); + return m_blocks[_hash]; } diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h index 62a5bf792..20c41b553 100644 --- a/libethereum/BlockChain.h +++ b/libethereum/BlockChain.h @@ -26,7 +26,7 @@ #include #pragma warning(pop) -#include +#include #include #include #include @@ -171,18 +171,20 @@ public: struct Statistics { + unsigned memBlocks; unsigned memDetails; unsigned memLogBlooms; unsigned memReceipts; unsigned memTransactionAddresses; - unsigned memCache; + unsigned memBlockHashes; + unsigned memTotal() const { return memBlocks + memDetails + memLogBlooms + memReceipts + memTransactionAddresses + memBlockHashes; } }; /// @returns statistics about memory usage. - Statistics usage() const; + Statistics usage(bool _freshen = false) const { if (_freshen) updateStats(); return m_lastStats; } /// Deallocate unused data. - void garbageCollect(); + void garbageCollect(bool _force = false); private: void open(std::string _path, bool _killExisting = false); @@ -205,6 +207,8 @@ private: return _n; } + noteUsed(_h, N); + WriteGuard l(_x); auto ret = _m.insert(std::make_pair(_h, T(RLP(s)))); return ret.first->second; @@ -213,19 +217,29 @@ private: void checkConsistency(); /// The caches of the disk DB and their locks. - mutable boost::shared_mutex x_blocks; + mutable SharedMutex x_blocks; mutable BlocksHash m_blocks; - mutable boost::shared_mutex x_details; + mutable SharedMutex x_details; mutable BlockDetailsHash m_details; - mutable boost::shared_mutex x_logBlooms; + mutable SharedMutex x_logBlooms; mutable BlockLogBloomsHash m_logBlooms; - mutable boost::shared_mutex x_receipts; + mutable SharedMutex x_receipts; mutable BlockReceiptsHash m_receipts; - mutable boost::shared_mutex x_transactionAddresses; + mutable SharedMutex x_transactionAddresses; mutable TransactionAddressHash m_transactionAddresses; - mutable boost::shared_mutex x_blockHashes; + mutable SharedMutex x_blockHashes; mutable BlockHashHash m_blockHashes; + using CacheID = std::pair; + mutable Mutex x_cacheUsage; + mutable std::deque> m_cacheUsage; + mutable std::set m_inUse; + void noteUsed(h256 const& _h, unsigned _extra = (unsigned)-1) const; + std::chrono::system_clock::time_point m_lastCollection; + + void updateStats() const; + mutable Statistics m_lastStats; + /// The disk DBs. Thread-safe, so no need for locks. ldb::DB* m_blocksDB; ldb::DB* m_extrasDB; diff --git a/libethereum/BlockDetails.cpp b/libethereum/BlockDetails.cpp index 58f37b7aa..c15939fdb 100644 --- a/libethereum/BlockDetails.cpp +++ b/libethereum/BlockDetails.cpp @@ -32,9 +32,12 @@ BlockDetails::BlockDetails(RLP const& _r) totalDifficulty = _r[1].toInt(); parent = _r[2].toHash(); children = _r[3].toVector(); + size = _r.size(); } bytes BlockDetails::rlp() const { - return rlpList(number, totalDifficulty, parent, children); + auto ret = rlpList(number, totalDifficulty, parent, children); + size = ret.size(); + return ret; } diff --git a/libethereum/BlockDetails.h b/libethereum/BlockDetails.h index f9bccea1e..ed478568d 100644 --- a/libethereum/BlockDetails.h +++ b/libethereum/BlockDetails.h @@ -46,28 +46,32 @@ struct BlockDetails bool isNull() const { return !totalDifficulty; } explicit operator bool() const { return !isNull(); } - unsigned number; // TODO: remove? + unsigned number; u256 totalDifficulty; h256 parent; h256s children; + + mutable unsigned size; }; struct BlockLogBlooms { BlockLogBlooms() {} - BlockLogBlooms(RLP const& _r) { blooms = _r.toVector(); } - bytes rlp() const { RLPStream s; s << blooms; return s.out(); } + BlockLogBlooms(RLP const& _r) { blooms = _r.toVector(); size = _r.data().size(); } + bytes rlp() const { RLPStream s; s << blooms; size = s.out().size(); return s.out(); } LogBlooms blooms; + mutable unsigned size; }; struct BlockReceipts { BlockReceipts() {} - BlockReceipts(RLP const& _r) { for (auto const& i: _r) receipts.emplace_back(i.data()); } - bytes rlp() const { RLPStream s(receipts.size()); for (TransactionReceipt const& i: receipts) i.streamRLP(s); return s.out(); } + BlockReceipts(RLP const& _r) { for (auto const& i: _r) receipts.emplace_back(i.data()); size = _r.data().size(); } + bytes rlp() const { RLPStream s(receipts.size()); for (TransactionReceipt const& i: receipts) i.streamRLP(s); size = s.out().size(); return s.out(); } TransactionReceipts receipts; + mutable unsigned size; }; struct BlockHash @@ -77,6 +81,7 @@ struct BlockHash bytes rlp() const { return dev::rlp(value); } h256 value; + static const unsigned size = 65; }; struct TransactionAddress @@ -89,6 +94,8 @@ struct TransactionAddress h256 blockHash; unsigned index = 0; + + static const unsigned size = 67; }; using BlockDetailsHash = std::map; diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index 827346c95..38d2f16ea 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -635,6 +635,7 @@ void Client::doWork() uninstallWatch(i); m_lastGarbageCollection = chrono::system_clock::now(); } + m_bc.garbageCollect(); } unsigned Client::numberOf(int _n) const