From 1e2aebbe1bf0908f81b5274eff9f69292c6da0cf Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 5 Mar 2015 12:48:28 +0100 Subject: [PATCH 1/5] Make RLP JSON compatible by default. --- rlp/main.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rlp/main.cpp b/rlp/main.cpp index 85c08e0e6..663bbf14d 100644 --- a/rlp/main.cpp +++ b/rlp/main.cpp @@ -36,6 +36,7 @@ void help() << " -r,--render Render the given RLP. Options:" << endl << " --indent Use string as the level indentation (default ' ')." << endl << " --hex-ints Render integers in hex." << endl + << " --ascii-strings Render data as C-style strings or hex depending on content being ASCII." << endl << " --force-string Force all data to be rendered as C-style strings." << endl << " --force-escape When rendering as C-style strings, force all characters to be escaped." << endl << " --force-hex Force all data to be rendered as raw hex." << endl @@ -86,7 +87,7 @@ public: { string indent = " "; bool hexInts = false; - bool forceString = false; + bool forceString = true; bool escapeAll = false; bool forceHex = false; }; @@ -150,6 +151,8 @@ int main(int argc, char** argv) prefs.indent = argv[++i]; else if (arg == "--hex-ints") prefs.hexInts = true; + else if (arg == "--ascii-strings") + prefs.forceString = prefs.forceHex = false; else if (arg == "--force-string") prefs.forceString = true; else if (arg == "--force-hex") From 228f41b8eda986cfe43afe14f3152dbbf87474de Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 5 Mar 2015 15:58:20 +0100 Subject: [PATCH 2/5] For Marek :) BlockChain::transaction(h256 _transactionHash) BlockChain::transactionHashes(h256 _blockHash) --- alethzero/MainWin.cpp | 5 +- libethcore/BlockInfo.h | 1 + libethcore/Ethasher.cpp | 115 +++++++++++++++++++++++++++++++++++++ libethcore/Ethasher.h | 97 +++++++++++++++++++++++++++++++ libethcore/ProofOfWork.cpp | 98 ++----------------------------- libethcore/ProofOfWork.h | 10 ---- libethereum/BlockChain.cpp | 6 ++ libethereum/BlockChain.h | 14 ++++- libethereum/BlockDetails.h | 14 +++++ test/dagger.cpp | 76 +++++++++++++++++------- 10 files changed, 308 insertions(+), 128 deletions(-) create mode 100644 libethcore/Ethasher.cpp create mode 100644 libethcore/Ethasher.h diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index e6f380e47..3c5dcf25a 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -36,6 +36,7 @@ #endif #include #include +#include #include #include #include @@ -1365,7 +1366,7 @@ void Main::on_blocks_currentItemChanged() s << "
Difficulty: " << info.difficulty << ""; if (info.number) { - auto e = ProofOfWork::eval(info); + auto e = Ethasher::eval(info); s << "
Proof-of-Work: " << e.value << " <= " << (h256)u256((bigint(1) << 256) / info.difficulty) << " (mixhash: " << e.mixHash.abridged() << ")"; } else @@ -1389,7 +1390,7 @@ void Main::on_blocks_currentItemChanged() s << line << "Nonce: " << uncle.nonce << ""; s << line << "Hash w/o nonce: " << uncle.headerHash(WithoutNonce) << ""; s << line << "Difficulty: " << uncle.difficulty << ""; - auto e = ProofOfWork::eval(uncle); + auto e = Ethasher::eval(uncle); s << line << "Proof-of-Work: " << e.value << " <= " << (h256)u256((bigint(1) << 256) / uncle.difficulty) << " (mixhash: " << e.mixHash.abridged() << ")"; } if (info.parentHash) diff --git a/libethcore/BlockInfo.h b/libethcore/BlockInfo.h index a119d92eb..95519b57d 100644 --- a/libethcore/BlockInfo.h +++ b/libethcore/BlockInfo.h @@ -85,6 +85,7 @@ public: static h256 headerHash(bytes const& _block) { return headerHash(&_block); } static h256 headerHash(bytesConstRef _block); + static BlockInfo fromHeader(bytes const& _block) { return fromHeader(bytesConstRef(&_block)); } static BlockInfo fromHeader(bytesConstRef _block); explicit operator bool() const { return timestamp != Invalid256; } diff --git a/libethcore/Ethasher.cpp b/libethcore/Ethasher.cpp new file mode 100644 index 000000000..a28895a1a --- /dev/null +++ b/libethcore/Ethasher.cpp @@ -0,0 +1,115 @@ +/* + This file is part of cpp-ethereum. + + cpp-ethereum is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + cpp-ethereum is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with cpp-ethereum. If not, see . +*/ +/** @file Ethasher.cpp + * @author Gav Wood + * @date 2014 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "BlockInfo.h" +#include "Ethasher.h" +using namespace std; +using namespace chrono; +using namespace dev; +using namespace eth; + +Ethasher* dev::eth::Ethasher::s_this = nullptr; + +bytes const& Ethasher::cache(BlockInfo const& _header) +{ + RecursiveGuard l(x_this); + if (!m_caches.count(_header.seedHash)) + { + try { + boost::filesystem::create_directories(getDataDir() + "/ethashcache"); + } catch (...) {} + std::string memoFile = getDataDir() + "/ethashcache/" + toHex(_header.seedHash.ref().cropped(0, 4)) + ".cache"; + m_caches[_header.seedHash] = contents(memoFile); + if (m_caches[_header.seedHash].empty()) + { + ethash_params p = params((unsigned)_header.number); + m_caches[_header.seedHash].resize(p.cache_size); + ethash_prep_light(m_caches[_header.seedHash].data(), &p, _header.seedHash.data()); + writeFile(memoFile, m_caches[_header.seedHash]); + } + } + return m_caches[_header.seedHash]; +} + +bytesConstRef Ethasher::full(BlockInfo const& _header) +{ + RecursiveGuard l(x_this); + if (!m_fulls.count(_header.seedHash)) + { + if (!m_fulls.empty()) + { + delete [] m_fulls.begin()->second.data(); + m_fulls.erase(m_fulls.begin()); + } + std::string memoFile = getDataDir() + "/ethashcache/" + toHex(_header.seedHash.ref().cropped(0, 4)) + ".full"; + m_fulls[_header.seedHash] = contentsNew(memoFile); + if (!m_fulls[_header.seedHash]) + { + ethash_params p = params((unsigned)_header.number); + m_fulls[_header.seedHash] = bytesRef(new byte[p.full_size], p.full_size); + auto c = cache(_header); + ethash_prep_full(m_fulls[_header.seedHash].data(), &p, c.data()); + writeFile(memoFile, m_fulls[_header.seedHash]); + } + } + return m_fulls[_header.seedHash]; +} + +ethash_params Ethasher::params(BlockInfo const& _header) +{ + return params((unsigned)_header.number); +} + +ethash_params Ethasher::params(unsigned _n) +{ + ethash_params p; + p.cache_size = ethash_get_cachesize(_n); + p.full_size = ethash_get_datasize(_n); + return p; +} + +bool Ethasher::verify(BlockInfo const& _header) +{ + bigint boundary = (bigint(1) << 256) / _header.difficulty; + auto e = eval(_header, _header.nonce); + return (u256)e.value <= boundary && e.mixHash == _header.mixHash; +} + +Ethasher::Result Ethasher::eval(BlockInfo const& _header, Nonce const& _nonce) +{ + auto p = Ethasher::params(_header); + ethash_return_value r; + ethash_compute_light(&r, Ethasher::get()->cache(_header).data(), &p, _header.headerHash(WithoutNonce).data(), (uint64_t)(u64)_nonce); + return Result{h256(r.result, h256::ConstructFromPointer), h256(r.mix_hash, h256::ConstructFromPointer)}; +} + diff --git a/libethcore/Ethasher.h b/libethcore/Ethasher.h new file mode 100644 index 000000000..cfe0d1c82 --- /dev/null +++ b/libethcore/Ethasher.h @@ -0,0 +1,97 @@ +/* + This file is part of cpp-ethereum. + + cpp-ethereum is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + cpp-ethereum is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with cpp-ethereum. If not, see . +*/ +/** @file Ethasher.h + * @author Gav Wood + * @date 2014 + * + * ProofOfWork algorithm. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include // TODO: REMOVE once everything merged into this class and an opaque API can be provided. +#include "Common.h" +#include "BlockInfo.h" + +namespace dev +{ +namespace eth +{ + +class Ethasher +{ +public: + Ethasher() {} + + static Ethasher* get() { if (!s_this) s_this = new Ethasher(); return s_this; } + + bytes const& cache(BlockInfo const& _header); + bytesConstRef full(BlockInfo const& _header); + static ethash_params params(BlockInfo const& _header); + static ethash_params params(unsigned _n); + + struct Result + { + h256 value; + h256 mixHash; + }; + + static Result eval(BlockInfo const& _header) { return eval(_header, _header.nonce); } + static Result eval(BlockInfo const& _header, Nonce const& _nonce); + static bool verify(BlockInfo const& _header); + + class Miner + { + public: + Miner(BlockInfo const& _header): + m_headerHash(_header.headerHash(WithoutNonce)), + m_params(Ethasher::params(_header)), + m_datasetPointer(Ethasher::get()->full(_header).data()) + {} + + inline h256 mine(uint64_t _nonce) + { + ethash_compute_full(&m_ethashReturn, m_datasetPointer, &m_params, m_headerHash.data(), _nonce); + return h256(m_ethashReturn.result, h256::ConstructFromPointer); + } + + inline h256 lastMixHash() const + { + return h256(m_ethashReturn.mix_hash, h256::ConstructFromPointer); + } + + private: + ethash_return_value m_ethashReturn; + h256 m_headerHash; + ethash_params m_params; + void const* m_datasetPointer; + }; + +private: + static Ethasher* s_this; + RecursiveMutex x_this; + std::map m_caches; + std::map m_fulls; +}; + +} +} diff --git a/libethcore/ProofOfWork.cpp b/libethcore/ProofOfWork.cpp index c879df2ce..d261ccb1c 100644 --- a/libethcore/ProofOfWork.cpp +++ b/libethcore/ProofOfWork.cpp @@ -30,8 +30,8 @@ #include #include #include -#include #include "BlockInfo.h" +#include "Ethasher.h" #include "ProofOfWork.h" using namespace std; using namespace std::chrono; @@ -41,100 +41,14 @@ namespace dev namespace eth { -class Ethasher -{ -public: - Ethasher() {} - - static Ethasher* get() { if (!s_this) s_this = new Ethasher(); return s_this; } - - bytes const& cache(BlockInfo const& _header) - { - RecursiveGuard l(x_this); - if (!m_caches.count(_header.seedHash)) - { - try { - boost::filesystem::create_directories(getDataDir() + "/ethashcache"); - } catch (...) {} - std::string memoFile = getDataDir() + "/ethashcache/" + toHex(_header.seedHash.ref().cropped(0, 4)) + ".cache"; - m_caches[_header.seedHash] = contents(memoFile); - if (m_caches[_header.seedHash].empty()) - { - ethash_params p = params((unsigned)_header.number); - m_caches[_header.seedHash].resize(p.cache_size); - ethash_prep_light(m_caches[_header.seedHash].data(), &p, _header.seedHash.data()); - writeFile(memoFile, m_caches[_header.seedHash]); - } - } - return m_caches[_header.seedHash]; - } - - byte const* full(BlockInfo const& _header) - { - RecursiveGuard l(x_this); - if (!m_fulls.count(_header.seedHash)) - { - if (!m_fulls.empty()) - { - delete [] m_fulls.begin()->second.data(); - m_fulls.erase(m_fulls.begin()); - } - std::string memoFile = getDataDir() + "/ethashcache/" + toHex(_header.seedHash.ref().cropped(0, 4)) + ".full"; - m_fulls[_header.seedHash] = contentsNew(memoFile); - if (!m_fulls[_header.seedHash]) - { - ethash_params p = params((unsigned)_header.number); - m_fulls[_header.seedHash] = bytesRef(new byte[p.full_size], p.full_size); - auto c = cache(_header); - ethash_prep_full(m_fulls[_header.seedHash].data(), &p, c.data()); - writeFile(memoFile, m_fulls[_header.seedHash]); - } - } - return m_fulls[_header.seedHash].data(); - } - - static ethash_params params(BlockInfo const& _header) - { - return params((unsigned)_header.number); - } - - static ethash_params params(unsigned _n) - { - ethash_params p; - p.cache_size = ethash_get_cachesize(_n); - p.full_size = ethash_get_datasize(_n); - return p; - } - -private: - static Ethasher* s_this; - RecursiveMutex x_this; - std::map m_caches; - std::map m_fulls; -}; - -Ethasher* Ethasher::s_this = nullptr; - bool Ethash::verify(BlockInfo const& _header) { - bigint boundary = (bigint(1) << 256) / _header.difficulty; - auto e = eval(_header, _header.nonce); - return (u256)e.value <= boundary && e.mixHash == _header.mixHash; -} - -Ethash::Result Ethash::eval(BlockInfo const& _header, Nonce const& _nonce) -{ - auto p = Ethasher::params(_header); - ethash_return_value r; - ethash_compute_light(&r, Ethasher::get()->cache(_header).data(), &p, _header.headerHash(WithoutNonce).data(), (uint64_t)(u64)_nonce); - return Result{h256(r.result, h256::ConstructFromPointer), h256(r.mix_hash, h256::ConstructFromPointer)}; + return Ethasher::verify(_header); } std::pair Ethash::mine(BlockInfo const& _header, unsigned _msTimeout, bool _continue, bool _turbo) { - auto h = _header.headerHash(WithoutNonce); - auto p = Ethasher::params(_header); - auto d = Ethasher::get()->full(_header); + Ethasher::Miner m(_header); std::pair ret; static std::mt19937_64 s_eng((time(0) + *reinterpret_cast(m_last.data()))); @@ -152,17 +66,15 @@ std::pair Ethash::mine(BlockInfo const& _header, unsign std::this_thread::sleep_for(std::chrono::milliseconds(_msTimeout * 90 / 100)); double best = 1e99; // high enough to be effectively infinity :) Proof result; - ethash_return_value ethashReturn; unsigned hashCount = 0; for (; (std::chrono::steady_clock::now() - startTime) < std::chrono::milliseconds(_msTimeout) && _continue; tryNonce++, hashCount++) { - ethash_compute_full(ðashReturn, d, &p, h.data(), tryNonce); - u256 val(h256(ethashReturn.result, h256::ConstructFromPointer)); + u256 val(m.mine(tryNonce)); best = std::min(best, log2((double)val)); if (val <= boundary) { ret.first.completed = true; - result.mixHash = h256(ethashReturn.mix_hash, h256::ConstructFromPointer); + result.mixHash = m.lastMixHash(); result.nonce = u64(tryNonce); break; } diff --git a/libethcore/ProofOfWork.h b/libethcore/ProofOfWork.h index 7006f6f61..250ddb73d 100644 --- a/libethcore/ProofOfWork.h +++ b/libethcore/ProofOfWork.h @@ -49,20 +49,12 @@ struct MineInfo class Ethash { public: - // bit-compatible with ethash_return_value struct Proof { Nonce nonce; h256 mixHash; }; - struct Result - { - h256 value; - h256 mixHash; - }; - static Result eval(BlockInfo const& _header) { return eval(_header, _header.nonce); } - static Result eval(BlockInfo const& _header, Nonce const& _nonce); static bool verify(BlockInfo const& _header); std::pair mine(BlockInfo const& _header, unsigned _msTimeout = 100, bool _continue = true, bool _turbo = false); static void assignResult(Proof const& _r, BlockInfo& _header) { _header.nonce = _r.nonce; _header.mixHash = _r.mixHash; } @@ -78,9 +70,7 @@ public: using Proof = Nonce; static bool verify(BlockInfo const& _header) { return (bigint)(u256)Evaluator::eval(_header.headerHash(WithoutNonce), _header.nonce) <= (bigint(1) << 256) / _header.difficulty; } - inline std::pair mine(BlockInfo const& _header, unsigned _msTimeout = 100, bool _continue = true, bool _turbo = false); - static void assignResult(Proof const& _r, BlockInfo& _header) { _header.nonce = _r; } protected: diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index d5c5e710c..30e9e4204 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -307,7 +307,13 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db) m_extrasDB->Put(m_writeOptions, toSlice(bi.parentHash), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp())); m_extrasDB->Put(m_writeOptions, toSlice(newHash, 3), (ldb::Slice)dev::ref(m_logBlooms[newHash].rlp())); m_extrasDB->Put(m_writeOptions, toSlice(newHash, 4), (ldb::Slice)dev::ref(m_receipts[newHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, 4), (ldb::Slice)dev::ref(m_receipts[newHash].rlp())); m_db->Put(m_writeOptions, toSlice(newHash), (ldb::Slice)ref(_block)); + RLP blockRLP(_block); + TransactionAddress ta; + ta.blockHash = newHash; + for (ta.index = 0; ta.index < blockRLP[1].itemCount(); ++ta.index) + m_extrasDB->Put(m_writeOptions, toSlice(sha3(blockRLP[1][ta.index].data()), 5), (ldb::Slice)dev::ref(ta.rlp())); #if ETH_PARANOIA checkConsistency(); diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h index dbcab2580..58bd7bf38 100644 --- a/libethereum/BlockChain.h +++ b/libethereum/BlockChain.h @@ -34,6 +34,7 @@ #include #include "BlockDetails.h" #include "Account.h" +#include "Transaction.h" #include "BlockQueue.h" namespace ldb = leveldb; @@ -61,6 +62,8 @@ std::map const& genesisState(); ldb::Slice toSlice(h256 _h, unsigned _sub = 0); +using TransactionHashes = h256s; + /** * @brief Implements the blockchain database. All data this gives is disk-backed. * @threadsafe @@ -108,12 +111,19 @@ public: BlockReceipts receipts(h256 _hash) const { return queryExtras(_hash, m_receipts, x_receipts, NullBlockReceipts); } BlockReceipts receipts() const { return receipts(currentHash()); } + /// Get a list of transaction hashes for a given block. Thread-safe. + TransactionHashes transactionHashes(h256 _hash) const { auto b = block(_hash); RLP rlp(b); h256s ret; for (auto t: rlp[1]) ret.push_back(sha3(t.data())); return ret; } + TransactionHashes transactionHashes() const { return transactionHashes(currentHash()); } + /// Get a block (RLP format) for the given hash (or the most recent mined if none given). Thread-safe. bytes block(h256 _hash) const; bytes block() const { return block(currentHash()); } + /// Get a transaction from its hash. Thread-safe. + bytes transaction(h256 _transactionHash) const { TransactionAddress ta = queryExtras(_transactionHash, m_transactionAddresses, x_transactionAddresses, NullTransactionAddress); if (!ta) return bytes(); return transaction(ta.blockHash, ta.index); } + /// Get a block's transaction (RLP format) for the given block hash (or the most recent mined if none given) & index. Thread-safe. - bytes transaction(h256 _hash, unsigned _i) const { bytes b = block(_hash); return RLP(b)[1][_i].data().toBytes(); } + bytes transaction(h256 _blockHash, unsigned _i) const { bytes b = block(_blockHash); return RLP(b)[1][_i].data().toBytes(); } bytes transaction(unsigned _i) const { return transaction(currentHash(), _i); } /// Get a number for the given hash (or the most recent mined if none given). Thread-safe. @@ -185,6 +195,8 @@ private: mutable BlockLogBloomsHash m_logBlooms; mutable boost::shared_mutex x_receipts; mutable BlockReceiptsHash m_receipts; + mutable boost::shared_mutex x_transactionAddresses; + mutable TransactionAddressHash m_transactionAddresses; mutable boost::shared_mutex x_cache; mutable std::map m_cache; diff --git a/libethereum/BlockDetails.h b/libethereum/BlockDetails.h index 9a3ac9ff1..b59d165ab 100644 --- a/libethereum/BlockDetails.h +++ b/libethereum/BlockDetails.h @@ -70,13 +70,27 @@ struct BlockReceipts TransactionReceipts receipts; }; +struct TransactionAddress +{ + TransactionAddress() {} + TransactionAddress(RLP const& _rlp) { blockHash = _rlp[0].toHash(); index = _rlp[1].toInt(); } + bytes rlp() const { RLPStream s(2); s << blockHash << index; return s.out(); } + + explicit operator bool() const { return !!blockHash; } + + h256 blockHash; + unsigned index = 0; +}; + using BlockDetailsHash = std::map; using BlockLogBloomsHash = std::map; using BlockReceiptsHash = std::map; +using TransactionAddressHash = std::map; static const BlockDetails NullBlockDetails; static const BlockLogBlooms NullBlockLogBlooms; static const BlockReceipts NullBlockReceipts; +static const TransactionAddress NullTransactionAddress; } } diff --git a/test/dagger.cpp b/test/dagger.cpp index 87c49bd7d..dec753fe7 100644 --- a/test/dagger.cpp +++ b/test/dagger.cpp @@ -17,37 +17,69 @@ /** @file dagger.cpp * @author Gav Wood * @date 2014 - * ProofOfWork test functions. + * Dashimoto test functions. */ -#include -#include +#include +#include +#include "JsonSpiritHeaders.h" +#include #include +#include +#include +#include "TestHelper.h" + using namespace std; -using namespace std::chrono; using namespace dev; using namespace dev::eth; -int daggerTest() +namespace js = json_spirit; + +using dev::operator <<; + +BOOST_AUTO_TEST_SUITE(DashimotoTests) + +BOOST_AUTO_TEST_CASE(basic_test) { -#if 0 - cnote << "Testing ProofOfWork..."; - // Test dagger - { - auto s = steady_clock::now(); - cout << hex << ProofOfWork().eval((h256)(u256)1, (h256)(u256)0); - cout << " " << dec << duration_cast(steady_clock::now() - s).count() << " ms" << endl; - cout << hex << ProofOfWork().eval((h256)(u256)1, (h256)(u256)1); - cout << " " << dec << duration_cast(steady_clock::now() - s).count() << " ms" << endl; - } + string testPath = test::getTestPath(); + + testPath += "/PoWTests"; + + cnote << "Testing Secure Trie..."; + js::mValue v; + string s = asString(contents(testPath + "/ethash_tests.json")); + BOOST_REQUIRE_MESSAGE(s.length() > 0, "Contents of 'ethash_tests.json' is empty. Have you cloned the 'tests' repo branch develop?"); + js::read_string(s, v); + for (auto& i: v.get_obj()) { - auto s = steady_clock::now(); - cout << hex << ProofOfWork().eval((h256)(u256)1, (h256)(u256)0); - cout << " " << dec << duration_cast(steady_clock::now() - s).count() << " ms" << endl; - cout << hex << ProofOfWork().eval((h256)(u256)1, (h256)(u256)1); - cout << " " << dec << duration_cast(steady_clock::now() - s).count() << " ms" << endl; - } + cnote << i.first; + js::mObject& o = i.second.get_obj(); + vector> ss; + BlockInfo header = BlockInfo::fromHeader(fromHex(o["header"].get_str())); + h256 headerHash(o["header_hash"].get_str()); + Nonce nonce(o["nonce"].get_str()); + BOOST_REQUIRE_EQUAL(headerHash, header.headerHash(WithoutNonce)); + BOOST_REQUIRE_EQUAL(nonce, header.nonce); + + unsigned cacheSize(o["cache_size"].get_int()); + h256 cacheHash(o["cache_hash"].get_str()); + BOOST_REQUIRE_EQUAL(Ethasher::get()->cache(header).size(), cacheSize); + BOOST_REQUIRE_EQUAL(sha3(Ethasher::get()->cache(header)), cacheHash); + +#if TEST_FULL + unsigned fullSize(o["full_size"].get_int()); + h256 fullHash(o["full_hash"].get_str()); + BOOST_REQUIRE_EQUAL(Ethasher::get()->full(header).size(), fullSize); + BOOST_REQUIRE_EQUAL(sha3(Ethasher::get()->full(header)), fullHash); #endif - return 0; + + h256 result(o["result"].get_str()); + Ethasher::Result r = Ethasher::eval(header); + BOOST_REQUIRE_EQUAL(r.value, result); + BOOST_REQUIRE_EQUAL(r.mixHash, header.mixHash); + } } +BOOST_AUTO_TEST_SUITE_END() + + From 90bb001cf9c82c80fd266ad726f3176367b9f310 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 5 Mar 2015 19:38:31 +0100 Subject: [PATCH 3/5] Fast block hash from number. --- libethcore/Common.cpp | 9 +++-- libethereum/BlockChain.cpp | 79 ++++++++++++++++++++------------------ libethereum/BlockChain.h | 52 ++++++++++++++++++------- libethereum/BlockDetails.h | 11 ++++++ libethereum/Client.cpp | 11 +++++- libethereum/Client.h | 1 + libethereum/Miner.h | 3 ++ 7 files changed, 111 insertions(+), 55 deletions(-) diff --git a/libethcore/Common.cpp b/libethcore/Common.cpp index 65c0b8b92..9eb622fe3 100644 --- a/libethcore/Common.cpp +++ b/libethcore/Common.cpp @@ -33,13 +33,14 @@ namespace eth { const unsigned c_protocolVersion = 56; -const unsigned c_databaseVersion = 6 + +const unsigned c_databaseBaseVersion = 7; #if ETH_FATDB - 1000 +const unsigned c_databaseVersionModifier = 1000; #else - 0 +const unsigned c_databaseVersionModifier = 0; #endif -; + +const unsigned c_databaseVersion = c_databaseBaseVersion + c_databaseVersionModifier; vector> const& units() { diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 30e9e4204..65a55880c 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -45,7 +45,7 @@ namespace js = json_spirit; std::ostream& dev::eth::operator<<(std::ostream& _out, BlockChain const& _bc) { string cmp = toBigEndianString(_bc.currentHash()); - auto it = _bc.m_db->NewIterator(_bc.m_readOptions); + auto it = _bc.m_blocksDB->NewIterator(_bc.m_readOptions); for (it->SeekToFirst(); it->Valid(); it->Next()) if (it->key().ToString() != "best") { @@ -102,9 +102,9 @@ void BlockChain::open(std::string _path, bool _killExisting) ldb::Options o; o.create_if_missing = true; - ldb::DB::Open(o, _path + "/blocks", &m_db); + ldb::DB::Open(o, _path + "/blocks", &m_blocksDB); ldb::DB::Open(o, _path + "/details", &m_extrasDB); - if (!m_db) + if (!m_blocksDB) BOOST_THROW_EXCEPTION(DatabaseAlreadyOpen()); if (!m_extrasDB) BOOST_THROW_EXCEPTION(DatabaseAlreadyOpen()); @@ -132,10 +132,10 @@ void BlockChain::close() { cnote << "Closing blockchain DB"; delete m_extrasDB; - delete m_db; + delete m_blocksDB; m_lastBlockHash = m_genesisHash; m_details.clear(); - m_cache.clear(); + m_blocks.clear(); } template @@ -294,6 +294,23 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db) m_details[newHash] = BlockDetails((unsigned)pd.number + 1, td, bi.parentHash, {}); m_details[bi.parentHash].children.push_back(newHash); } + { + WriteGuard l(x_blockHashes); + m_blockHashes[h256(bi.number)].value = newHash; + } + // Collate transaction hashes and remember who they were. + h256s tas; + { + RLP blockRLP(_block); + TransactionAddress ta; + ta.blockHash = newHash; + WriteGuard l(x_transactionAddresses); + for (ta.index = 0; ta.index < blockRLP[1].itemCount(); ++ta.index) + { + tas.push_back(sha3(blockRLP[1][ta.index].data())); + m_transactionAddresses[tas.back()] = ta; + } + } { WriteGuard l(x_logBlooms); m_logBlooms[newHash] = blb; @@ -303,17 +320,14 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db) m_receipts[newHash] = br; } - m_extrasDB->Put(m_writeOptions, toSlice(newHash), (ldb::Slice)dev::ref(m_details[newHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(bi.parentHash), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(newHash, 3), (ldb::Slice)dev::ref(m_logBlooms[newHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(newHash, 4), (ldb::Slice)dev::ref(m_receipts[newHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(newHash, 4), (ldb::Slice)dev::ref(m_receipts[newHash].rlp())); - m_db->Put(m_writeOptions, toSlice(newHash), (ldb::Slice)ref(_block)); - RLP blockRLP(_block); - TransactionAddress ta; - ta.blockHash = newHash; - for (ta.index = 0; ta.index < blockRLP[1].itemCount(); ++ta.index) - m_extrasDB->Put(m_writeOptions, toSlice(sha3(blockRLP[1][ta.index].data()), 5), (ldb::Slice)dev::ref(ta.rlp())); + m_blocksDB->Put(m_writeOptions, toSlice(newHash), (ldb::Slice)ref(_block)); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[newHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(bi.parentHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(h256(bi.number), ExtraBlockHash), (ldb::Slice)dev::ref(m_blockHashes[h256(bi.number)].rlp())); + for (auto const& h: tas) + m_extrasDB->Put(m_writeOptions, toSlice(h, ExtraTransactionAddress), (ldb::Slice)dev::ref(m_transactionAddresses[h].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraLogBlooms), (ldb::Slice)dev::ref(m_logBlooms[newHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraReceipts), (ldb::Slice)dev::ref(m_receipts[newHash].rlp())); #if ETH_PARANOIA checkConsistency(); @@ -406,7 +420,7 @@ void BlockChain::checkConsistency() WriteGuard l(x_details); m_details.clear(); } - ldb::Iterator* it = m_db->NewIterator(m_readOptions); + ldb::Iterator* it = m_blocksDB->NewIterator(m_readOptions); for (it->SeekToFirst(); it->Valid(); it->Next()) if (it->key().size() == 32) { @@ -449,12 +463,12 @@ bool BlockChain::isKnown(h256 _hash) const if (_hash == m_genesisHash) return true; { - ReadGuard l(x_cache); - if (m_cache.count(_hash)) + ReadGuard l(x_blocks); + if (m_blocks.count(_hash)) return true; } string d; - m_db->Get(m_readOptions, ldb::Slice((char const*)&_hash, 32), &d); + m_blocksDB->Get(m_readOptions, ldb::Slice((char const*)&_hash, 32), &d); return !!d.size(); } @@ -464,14 +478,14 @@ bytes BlockChain::block(h256 _hash) const return m_genesisBlock; { - ReadGuard l(x_cache); - auto it = m_cache.find(_hash); - if (it != m_cache.end()) + ReadGuard l(x_blocks); + auto it = m_blocks.find(_hash); + if (it != m_blocks.end()) return it->second; } string d; - m_db->Get(m_readOptions, ldb::Slice((char const*)&_hash, 32), &d); + m_blocksDB->Get(m_readOptions, ldb::Slice((char const*)&_hash, 32), &d); if (!d.size()) { @@ -479,18 +493,9 @@ bytes BlockChain::block(h256 _hash) const return bytes(); } - WriteGuard l(x_cache); - m_cache[_hash].resize(d.size()); - memcpy(m_cache[_hash].data(), d.data(), d.size()); - - return m_cache[_hash]; -} + WriteGuard l(x_blocks); + m_blocks[_hash].resize(d.size()); + memcpy(m_blocks[_hash].data(), d.data(), d.size()); -h256 BlockChain::numberHash(unsigned _n) const -{ - if (!_n) - return genesisHash(); - h256 ret = currentHash(); - for (; _n < details().number; ++_n, ret = details(ret).parent) {} - return ret; + return m_blocks[_hash]; } diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h index 58bd7bf38..62a5bf792 100644 --- a/libethereum/BlockChain.h +++ b/libethereum/BlockChain.h @@ -62,8 +62,17 @@ std::map const& genesisState(); ldb::Slice toSlice(h256 _h, unsigned _sub = 0); +using BlocksHash = std::map; using TransactionHashes = h256s; +enum { + ExtraDetails = 0, + ExtraBlockHash, + ExtraTransactionAddress, + ExtraLogBlooms, + ExtraReceipts +}; + /** * @brief Implements the blockchain database. All data this gives is disk-backed. * @threadsafe @@ -99,28 +108,31 @@ public: BlockInfo info(h256 _hash) const { return BlockInfo(block(_hash)); } BlockInfo info() const { return BlockInfo(block()); } + /// Get a block (RLP format) for the given hash (or the most recent mined if none given). Thread-safe. + bytes block(h256 _hash) const; + bytes block() const { return block(currentHash()); } + /// Get the familial details concerning a block (or the most recent mined if none given). Thread-safe. - BlockDetails details(h256 _hash) const { return queryExtras(_hash, m_details, x_details, NullBlockDetails); } + BlockDetails details(h256 _hash) const { return queryExtras(_hash, m_details, x_details, NullBlockDetails); } BlockDetails details() const { return details(currentHash()); } /// Get the transactions' log blooms of a block (or the most recent mined if none given). Thread-safe. - BlockLogBlooms logBlooms(h256 _hash) const { return queryExtras(_hash, m_logBlooms, x_logBlooms, NullBlockLogBlooms); } + BlockLogBlooms logBlooms(h256 _hash) const { return queryExtras(_hash, m_logBlooms, x_logBlooms, NullBlockLogBlooms); } BlockLogBlooms logBlooms() const { return logBlooms(currentHash()); } /// Get the transactions' receipts of a block (or the most recent mined if none given). Thread-safe. - BlockReceipts receipts(h256 _hash) const { return queryExtras(_hash, m_receipts, x_receipts, NullBlockReceipts); } + BlockReceipts receipts(h256 _hash) const { return queryExtras(_hash, m_receipts, x_receipts, NullBlockReceipts); } BlockReceipts receipts() const { return receipts(currentHash()); } /// Get a list of transaction hashes for a given block. Thread-safe. TransactionHashes transactionHashes(h256 _hash) const { auto b = block(_hash); RLP rlp(b); h256s ret; for (auto t: rlp[1]) ret.push_back(sha3(t.data())); return ret; } TransactionHashes transactionHashes() const { return transactionHashes(currentHash()); } - /// Get a block (RLP format) for the given hash (or the most recent mined if none given). Thread-safe. - bytes block(h256 _hash) const; - bytes block() const { return block(currentHash()); } + /// Get a list of transaction hashes for a given block. Thread-safe. + h256 numberHash(u256 _index) const { if (!_index) return genesisHash(); return queryExtras(h256(_index), m_blockHashes, x_blockHashes, NullBlockHash).value; } /// Get a transaction from its hash. Thread-safe. - bytes transaction(h256 _transactionHash) const { TransactionAddress ta = queryExtras(_transactionHash, m_transactionAddresses, x_transactionAddresses, NullTransactionAddress); if (!ta) return bytes(); return transaction(ta.blockHash, ta.index); } + bytes transaction(h256 _transactionHash) const { TransactionAddress ta = queryExtras(_transactionHash, m_transactionAddresses, x_transactionAddresses, NullTransactionAddress); if (!ta) return bytes(); return transaction(ta.blockHash, ta.index); } /// Get a block's transaction (RLP format) for the given block hash (or the most recent mined if none given) & index. Thread-safe. bytes transaction(h256 _blockHash, unsigned _i) const { bytes b = block(_blockHash); return RLP(b)[1][_i].data().toBytes(); } @@ -136,9 +148,6 @@ public: /// Get the hash of the genesis block. Thread-safe. h256 genesisHash() const { return m_genesisHash; } - /// Get the hash of a block of a given number. Slow; try not to use it too much. - h256 numberHash(unsigned _n) const; - /// Get all blocks not allowed as uncles given a parent (i.e. featured as uncles/main in parent, parent + 1, ... parent + 5). /// @returns set including the header-hash of every parent (including @a _parent) up to and including generation +5 /// togther with all their quoted uncles. @@ -160,6 +169,21 @@ public: */ h256s treeRoute(h256 _from, h256 _to, h256* o_common = nullptr, bool _pre = true, bool _post = true) const; + struct Statistics + { + unsigned memDetails; + unsigned memLogBlooms; + unsigned memReceipts; + unsigned memTransactionAddresses; + unsigned memCache; + }; + + /// @returns statistics about memory usage. + Statistics usage() const; + + /// Deallocate unused data. + void garbageCollect(); + private: void open(std::string _path, bool _killExisting = false); void close(); @@ -189,6 +213,8 @@ private: void checkConsistency(); /// The caches of the disk DB and their locks. + mutable boost::shared_mutex x_blocks; + mutable BlocksHash m_blocks; mutable boost::shared_mutex x_details; mutable BlockDetailsHash m_details; mutable boost::shared_mutex x_logBlooms; @@ -197,11 +223,11 @@ private: mutable BlockReceiptsHash m_receipts; mutable boost::shared_mutex x_transactionAddresses; mutable TransactionAddressHash m_transactionAddresses; - mutable boost::shared_mutex x_cache; - mutable std::map m_cache; + mutable boost::shared_mutex x_blockHashes; + mutable BlockHashHash m_blockHashes; /// The disk DBs. Thread-safe, so no need for locks. - ldb::DB* m_db; + ldb::DB* m_blocksDB; ldb::DB* m_extrasDB; /// Hash of the last (valid) block on the longest chain. diff --git a/libethereum/BlockDetails.h b/libethereum/BlockDetails.h index b59d165ab..f9bccea1e 100644 --- a/libethereum/BlockDetails.h +++ b/libethereum/BlockDetails.h @@ -70,6 +70,15 @@ struct BlockReceipts TransactionReceipts receipts; }; +struct BlockHash +{ + BlockHash() {} + BlockHash(RLP const& _r) { value = _r.toHash(); } + bytes rlp() const { return dev::rlp(value); } + + h256 value; +}; + struct TransactionAddress { TransactionAddress() {} @@ -86,11 +95,13 @@ using BlockDetailsHash = std::map; using BlockLogBloomsHash = std::map; using BlockReceiptsHash = std::map; using TransactionAddressHash = std::map; +using BlockHashHash = std::map; static const BlockDetails NullBlockDetails; static const BlockLogBlooms NullBlockLogBlooms; static const BlockReceipts NullBlockReceipts; static const TransactionAddress NullTransactionAddress; +static const BlockHash NullBlockHash; } } diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index 0462d1c7c..4db4afe58 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -516,9 +516,18 @@ void Client::doWork() { if (m.isComplete()) { - cwork << "CHAIN <== postSTATE"; + // TODO: enable a short-circuit option since we mined it. will need to get the end state from the miner. + auto lm = dynamic_cast(&m); h256s hs; + if (false && lm && !m_verifyOwnBlocks) { + // TODO: implement + //m_bc.attemptImport(m_blockData(), m_stateDB, lm->state()); + // TODO: derive hs from lm->state() + } + else + { + cwork << "CHAIN <== postSTATE"; WriteGuard l(x_stateDB); hs = m_bc.attemptImport(m.blockData(), m_stateDB); } diff --git a/libethereum/Client.h b/libethereum/Client.h index c4f2b3705..02fc8b91b 100644 --- a/libethereum/Client.h +++ b/libethereum/Client.h @@ -359,6 +359,7 @@ private: bool m_paranoia = false; ///< Should we be paranoid about our state? bool m_turboMining = false; ///< Don't squander all of our time mining actually just sleeping. bool m_forceMining = false; ///< Mine even when there are no transactions pending? + bool m_verifyOwnBlocks = true; ///< Shoudl be verify blocks that we mined? mutable Mutex m_filterLock; std::map m_filters; diff --git a/libethereum/Miner.h b/libethereum/Miner.h index bdd49abb9..45cf9944b 100644 --- a/libethereum/Miner.h +++ b/libethereum/Miner.h @@ -130,6 +130,9 @@ public: /// Get and clear the mining history. std::list miningHistory() { Guard l(x_mineInfo); auto ret = m_mineHistory; m_mineHistory.clear(); return ret; } + /// @returns the state on which we mined. + State const& state() const { return m_mineState; } + private: /// Do some work on the mining. virtual void doWork(); From 114712aa43136d36c49f24f0cf435ca5dfba1795 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 5 Mar 2015 21:19:21 +0100 Subject: [PATCH 4/5] Caching. --- alethzero/Main.ui | 7 ++ alethzero/MainWin.cpp | 8 +++ alethzero/MainWin.h | 1 + libethereum/BlockChain.cpp | 124 +++++++++++++++++++++++++++++++++++ libethereum/BlockChain.h | 34 +++++++--- libethereum/BlockDetails.cpp | 5 +- libethereum/BlockDetails.h | 17 +++-- libethereum/Client.cpp | 1 + 8 files changed, 181 insertions(+), 16 deletions(-) diff --git a/alethzero/Main.ui b/alethzero/Main.ui index 9025aa846..8f6597ca5 100644 --- a/alethzero/Main.ui +++ b/alethzero/Main.ui @@ -38,6 +38,13 @@ + + + + 0 bytes used + + + diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index 3c5dcf25a..978676795 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -137,6 +137,7 @@ Main::Main(QWidget *parent) : ui->configDock->close(); on_verbosity_valueChanged(); + statusBar()->addPermanentWidget(ui->cacheUsage); statusBar()->addPermanentWidget(ui->balance); statusBar()->addPermanentWidget(ui->peerCount); statusBar()->addPermanentWidget(ui->mineStatus); @@ -1140,6 +1141,12 @@ void Main::on_refresh_triggered() refreshAll(); } +void Main::refreshCache() +{ + BlockChain::Statistics s = ethereum()->blockChain().usage(); + ui->cacheUsage->setText(QString("%1 bytes used").arg(s.memTotal())); +} + void Main::timerEvent(QTimerEvent*) { // 7/18, Alex: aggregating timers, prelude to better threading? @@ -1168,6 +1175,7 @@ void Main::timerEvent(QTimerEvent*) interval = 0; refreshNetwork(); refreshWhispers(); + refreshCache(); poll(); } else diff --git a/alethzero/MainWin.h b/alethzero/MainWin.h index 072911ff7..487272a7a 100644 --- a/alethzero/MainWin.h +++ b/alethzero/MainWin.h @@ -211,6 +211,7 @@ private: void refreshNetwork(); void refreshMining(); void refreshWhispers(); + void refreshCache(); void refreshAll(); void refreshPending(); diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 65a55880c..f4a5acea9 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -75,8 +75,33 @@ ldb::Slice dev::eth::toSlice(h256 _h, unsigned _sub) #endif } +#if ETH_DEBUG +static const chrono::system_clock::duration c_collectionDuration = chrono::seconds(15); +static const unsigned c_collectionQueueSize = 2; +static const unsigned c_maxCacheSize = 1024 * 1024 * 1; +static const unsigned c_minCacheSize = 1; +#else + +/// Duration between flushes. +static const chrono::system_clock::duration c_collectionDuration = chrono::seconds(60); + +/// Length of death row (total time in cache is multiple of this and collection duration). +static const unsigned c_collectionQueueSize = 20; + +/// Max size, above which we start forcing cache reduction. +static const unsigned c_maxCacheSize = 1024 * 1024 * 64; + +/// Min size, below which we don't bother flushing it. +static const unsigned c_minCacheSize = 1024 * 1024 * 32; + +#endif + BlockChain::BlockChain(bytes const& _genesisBlock, std::string _path, bool _killExisting) { + // initialise deathrow. + m_cacheUsage.resize(c_collectionQueueSize); + m_lastCollection = chrono::system_clock::now(); + // Initialise with the genesis as the last block on the longest chain. m_genesisBlock = _genesisBlock; m_genesisHash = sha3(RLP(m_genesisBlock)[0].data()); @@ -414,6 +439,103 @@ h256s BlockChain::treeRoute(h256 _from, h256 _to, h256* o_common, bool _pre, boo return ret; } +void BlockChain::noteUsed(h256 const& _h, unsigned _extra) const +{ + auto id = CacheID(_h, _extra); + Guard l(x_cacheUsage); + m_cacheUsage[0].insert(id); + if (m_cacheUsage[1].count(id)) + m_cacheUsage[1].erase(id); + else + m_inUse.insert(id); +} + +template static unsigned getHashSize(map const& _map) +{ + unsigned ret = 0; + for (auto const& i: _map) + ret += i.second.size + 64; + return ret; +} + +void BlockChain::updateStats() const +{ + { + ReadGuard l1(x_blocks); + m_lastStats.memBlocks = 0; + for (auto const& i: m_blocks) + m_lastStats.memBlocks += i.second.size() + 64; + } + { + ReadGuard l2(x_details); + m_lastStats.memDetails = getHashSize(m_details); + } + { + ReadGuard l5(x_logBlooms); + m_lastStats.memLogBlooms = getHashSize(m_logBlooms); + } + { + ReadGuard l4(x_receipts); + m_lastStats.memReceipts = getHashSize(m_receipts); + } + { + ReadGuard l3(x_blockHashes); + m_lastStats.memBlockHashes = getHashSize(m_blockHashes); + } + { + ReadGuard l6(x_transactionAddresses); + m_lastStats.memTransactionAddresses = getHashSize(m_transactionAddresses); + } +} + +void BlockChain::garbageCollect(bool _force) +{ + updateStats(); + + if (!_force && chrono::system_clock::now() < m_lastCollection + c_collectionDuration && m_lastStats.memTotal() < c_maxCacheSize) + return; + if (m_lastStats.memTotal() < c_minCacheSize) + return; + + m_lastCollection = chrono::system_clock::now(); + + Guard l(x_cacheUsage); + WriteGuard l1(x_blocks); + WriteGuard l2(x_details); + WriteGuard l3(x_blockHashes); + WriteGuard l4(x_receipts); + WriteGuard l5(x_logBlooms); + WriteGuard l6(x_transactionAddresses); + for (CacheID const& id: m_cacheUsage.back()) + { + m_inUse.erase(id); + // kill i from cache. + switch (id.second) + { + case (unsigned)-1: + m_blocks.erase(id.first); + break; + case ExtraDetails: + m_details.erase(id.first); + break; + case ExtraBlockHash: + m_blockHashes.erase(id.first); + break; + case ExtraReceipts: + m_receipts.erase(id.first); + break; + case ExtraLogBlooms: + m_logBlooms.erase(id.first); + break; + case ExtraTransactionAddress: + m_transactionAddresses.erase(id.first); + break; + } + } + m_cacheUsage.pop_back(); + m_cacheUsage.push_front({}); +} + void BlockChain::checkConsistency() { { @@ -497,5 +619,7 @@ bytes BlockChain::block(h256 _hash) const m_blocks[_hash].resize(d.size()); memcpy(m_blocks[_hash].data(), d.data(), d.size()); + noteUsed(_hash); + return m_blocks[_hash]; } diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h index 62a5bf792..20c41b553 100644 --- a/libethereum/BlockChain.h +++ b/libethereum/BlockChain.h @@ -26,7 +26,7 @@ #include #pragma warning(pop) -#include +#include #include #include #include @@ -171,18 +171,20 @@ public: struct Statistics { + unsigned memBlocks; unsigned memDetails; unsigned memLogBlooms; unsigned memReceipts; unsigned memTransactionAddresses; - unsigned memCache; + unsigned memBlockHashes; + unsigned memTotal() const { return memBlocks + memDetails + memLogBlooms + memReceipts + memTransactionAddresses + memBlockHashes; } }; /// @returns statistics about memory usage. - Statistics usage() const; + Statistics usage(bool _freshen = false) const { if (_freshen) updateStats(); return m_lastStats; } /// Deallocate unused data. - void garbageCollect(); + void garbageCollect(bool _force = false); private: void open(std::string _path, bool _killExisting = false); @@ -205,6 +207,8 @@ private: return _n; } + noteUsed(_h, N); + WriteGuard l(_x); auto ret = _m.insert(std::make_pair(_h, T(RLP(s)))); return ret.first->second; @@ -213,19 +217,29 @@ private: void checkConsistency(); /// The caches of the disk DB and their locks. - mutable boost::shared_mutex x_blocks; + mutable SharedMutex x_blocks; mutable BlocksHash m_blocks; - mutable boost::shared_mutex x_details; + mutable SharedMutex x_details; mutable BlockDetailsHash m_details; - mutable boost::shared_mutex x_logBlooms; + mutable SharedMutex x_logBlooms; mutable BlockLogBloomsHash m_logBlooms; - mutable boost::shared_mutex x_receipts; + mutable SharedMutex x_receipts; mutable BlockReceiptsHash m_receipts; - mutable boost::shared_mutex x_transactionAddresses; + mutable SharedMutex x_transactionAddresses; mutable TransactionAddressHash m_transactionAddresses; - mutable boost::shared_mutex x_blockHashes; + mutable SharedMutex x_blockHashes; mutable BlockHashHash m_blockHashes; + using CacheID = std::pair; + mutable Mutex x_cacheUsage; + mutable std::deque> m_cacheUsage; + mutable std::set m_inUse; + void noteUsed(h256 const& _h, unsigned _extra = (unsigned)-1) const; + std::chrono::system_clock::time_point m_lastCollection; + + void updateStats() const; + mutable Statistics m_lastStats; + /// The disk DBs. Thread-safe, so no need for locks. ldb::DB* m_blocksDB; ldb::DB* m_extrasDB; diff --git a/libethereum/BlockDetails.cpp b/libethereum/BlockDetails.cpp index 58f37b7aa..c15939fdb 100644 --- a/libethereum/BlockDetails.cpp +++ b/libethereum/BlockDetails.cpp @@ -32,9 +32,12 @@ BlockDetails::BlockDetails(RLP const& _r) totalDifficulty = _r[1].toInt(); parent = _r[2].toHash(); children = _r[3].toVector(); + size = _r.size(); } bytes BlockDetails::rlp() const { - return rlpList(number, totalDifficulty, parent, children); + auto ret = rlpList(number, totalDifficulty, parent, children); + size = ret.size(); + return ret; } diff --git a/libethereum/BlockDetails.h b/libethereum/BlockDetails.h index f9bccea1e..ed478568d 100644 --- a/libethereum/BlockDetails.h +++ b/libethereum/BlockDetails.h @@ -46,28 +46,32 @@ struct BlockDetails bool isNull() const { return !totalDifficulty; } explicit operator bool() const { return !isNull(); } - unsigned number; // TODO: remove? + unsigned number; u256 totalDifficulty; h256 parent; h256s children; + + mutable unsigned size; }; struct BlockLogBlooms { BlockLogBlooms() {} - BlockLogBlooms(RLP const& _r) { blooms = _r.toVector(); } - bytes rlp() const { RLPStream s; s << blooms; return s.out(); } + BlockLogBlooms(RLP const& _r) { blooms = _r.toVector(); size = _r.data().size(); } + bytes rlp() const { RLPStream s; s << blooms; size = s.out().size(); return s.out(); } LogBlooms blooms; + mutable unsigned size; }; struct BlockReceipts { BlockReceipts() {} - BlockReceipts(RLP const& _r) { for (auto const& i: _r) receipts.emplace_back(i.data()); } - bytes rlp() const { RLPStream s(receipts.size()); for (TransactionReceipt const& i: receipts) i.streamRLP(s); return s.out(); } + BlockReceipts(RLP const& _r) { for (auto const& i: _r) receipts.emplace_back(i.data()); size = _r.data().size(); } + bytes rlp() const { RLPStream s(receipts.size()); for (TransactionReceipt const& i: receipts) i.streamRLP(s); size = s.out().size(); return s.out(); } TransactionReceipts receipts; + mutable unsigned size; }; struct BlockHash @@ -77,6 +81,7 @@ struct BlockHash bytes rlp() const { return dev::rlp(value); } h256 value; + static const unsigned size = 65; }; struct TransactionAddress @@ -89,6 +94,8 @@ struct TransactionAddress h256 blockHash; unsigned index = 0; + + static const unsigned size = 67; }; using BlockDetailsHash = std::map; diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index 4db4afe58..448fdb9d6 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -629,6 +629,7 @@ void Client::doWork() uninstallWatch(i); m_lastGarbageCollection = chrono::system_clock::now(); } + m_bc.garbageCollect(); } unsigned Client::numberOf(int _n) const From 8d6cb82083dfe86c6a83d85730ba5e0f319e9011 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 5 Mar 2015 22:42:34 +0100 Subject: [PATCH 5/5] Cache reporting in AZ. --- alethzero/MainWin.cpp | 34 +++++++++++++++++++++++++++++++++- libethereum/Client.cpp | 7 +++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index 978676795..a155207ba 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -1141,10 +1141,42 @@ void Main::on_refresh_triggered() refreshAll(); } +static std::string niceUsed(unsigned _n) +{ + static const vector c_units = { "bytes", "KB", "MB", "GB", "TB", "PB" }; + unsigned u = 0; + while (_n > 10240) + { + _n /= 1024; + u++; + } + if (_n > 1000) + return toString(_n / 1000) + "." + toString((min(949, _n % 1000) + 50) / 100) + " " + c_units[u + 1]; + else + return toString(_n) + " " + c_units[u]; +} + void Main::refreshCache() { BlockChain::Statistics s = ethereum()->blockChain().usage(); - ui->cacheUsage->setText(QString("%1 bytes used").arg(s.memTotal())); + QString t; + auto f = [&](unsigned n, QString l) + { + t += ("%1 " + l).arg(QString::fromStdString(niceUsed(n))); + }; + f(s.memTotal(), "total"); + t += " ("; + f(s.memBlocks, "blocks"); + t += ", "; + f(s.memReceipts, "receipts"); + t += ", "; + f(s.memLogBlooms, "blooms"); + t += ", "; + f(s.memBlockHashes + s.memTransactionAddresses, "hashes"); + t += ", "; + f(s.memDetails, "family"); + t += ")"; + ui->cacheUsage->setText(t); } void Main::timerEvent(QTimerEvent*) diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index 448fdb9d6..9a249a645 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -614,7 +614,7 @@ void Client::doWork() this_thread::sleep_for(chrono::milliseconds(100)); if (chrono::system_clock::now() - m_lastGarbageCollection > chrono::seconds(5)) { - // garbage collect on watches + // watches garbage collection vector toUninstall; { Guard l(m_filterLock); @@ -627,9 +627,12 @@ void Client::doWork() } for (auto i: toUninstall) uninstallWatch(i); + + // blockchain GC + m_bc.garbageCollect(); + m_lastGarbageCollection = chrono::system_clock::now(); } - m_bc.garbageCollect(); } unsigned Client::numberOf(int _n) const