From d16737c7070a31b74be0f1507e1737d6064abb2b Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 5 Apr 2015 14:47:39 +0200 Subject: [PATCH 01/10] New DAG file scheme. --- libethcore/Ethasher.cpp | 39 ++++++++++++++++++++++++--------------- libethcore/Ethasher.h | 4 ++-- test/dagger.cpp | 2 +- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/libethcore/Ethasher.cpp b/libethcore/Ethasher.cpp index 64db69187..58a0aa8d7 100644 --- a/libethcore/Ethasher.cpp +++ b/libethcore/Ethasher.cpp @@ -43,21 +43,21 @@ Ethasher* dev::eth::Ethasher::s_this = nullptr; Ethasher::~Ethasher() { - while (!m_caches.empty()) - killCache(m_caches.begin()->first); + while (!m_lights.empty()) + killCache(m_lights.begin()->first); } void Ethasher::killCache(h256 const& _s) { RecursiveGuard l(x_this); - if (m_caches.count(_s)) + if (m_lights.count(_s)) { - ethash_delete_light(m_caches.at(_s)); - m_caches.erase(_s); + ethash_delete_light(m_lights.at(_s)); + m_lights.erase(_s); } } -void const* Ethasher::cache(BlockInfo const& _header) +void const* Ethasher::light(BlockInfo const& _header) { RecursiveGuard l(x_this); if (_header.number > c_ethashEpochLength * 2048) @@ -67,14 +67,16 @@ void const* Ethasher::cache(BlockInfo const& _header) throw std::invalid_argument( error.str() ); } - if (!m_caches.count(_header.seedHash())) + if (!m_lights.count(_header.seedHash())) { ethash_params p = params((unsigned)_header.number); - m_caches[_header.seedHash()] = ethash_new_light(&p, _header.seedHash().data()); + m_lights[_header.seedHash()] = ethash_new_light(&p, _header.seedHash().data()); } - return m_caches[_header.seedHash()]; + return m_lights[_header.seedHash()]; } +#define IGNORE_EXCEPTIONS(X) try { X; } catch (...) {} + bytesConstRef Ethasher::full(BlockInfo const& _header) { RecursiveGuard l(x_this); @@ -89,19 +91,26 @@ bytesConstRef Ethasher::full(BlockInfo const& _header) boost::filesystem::create_directories(getDataDir("ethash")); } catch (...) {} - std::string memoFile = getDataDir("ethash") + "/full"; auto info = rlpList(c_ethashRevision, _header.seedHash()); - if (boost::filesystem::exists(memoFile) && contents(memoFile + ".info") != info) - boost::filesystem::remove(memoFile); + std::string oldMemoFile = getDataDir("ethash") + "/full"; + std::string memoFile = getDataDir("ethash") + "/full-R" + toString(c_ethashRevision) + "-" + toHex(_header.seedHash().ref().cropped(0, 8)); + if (boost::filesystem::exists(oldMemoFile) && contents(oldMemoFile + ".info") == info) + { + // memofile valid - rename. + boost::filesystem::rename(oldMemoFile, memoFile); + } + + IGNORE_EXCEPTIONS(boost::filesystem::remove(oldMemoFile)); + IGNORE_EXCEPTIONS(boost::filesystem::remove(oldMemoFile + ".info")); + m_fulls[_header.seedHash()] = contentsNew(memoFile); if (!m_fulls[_header.seedHash()]) { ethash_params p = params((unsigned)_header.number); m_fulls[_header.seedHash()] = bytesRef(new byte[p.full_size], p.full_size); - auto c = cache(_header); + auto c = light(_header); ethash_prep_full(m_fulls[_header.seedHash()].data(), &p, c); writeFile(memoFile, m_fulls[_header.seedHash()]); - writeFile(memoFile + ".info", info); } } return m_fulls[_header.seedHash()]; @@ -162,7 +171,7 @@ Ethasher::Result Ethasher::eval(BlockInfo const& _header, Nonce const& _nonce) { auto p = Ethasher::params(_header); ethash_return_value r; - ethash_compute_light(&r, Ethasher::get()->cache(_header), &p, _header.headerHash(WithoutNonce).data(), (uint64_t)(u64)_nonce); + ethash_compute_light(&r, Ethasher::get()->light(_header), &p, _header.headerHash(WithoutNonce).data(), (uint64_t)(u64)_nonce); // cdebug << "Ethasher::eval sha3(cache):" << sha3(Ethasher::get()->cache(_header)) << "hh:" << _header.headerHash(WithoutNonce) << "nonce:" << _nonce << " => " << h256(r.result, h256::ConstructFromPointer); return Result{h256(r.result, h256::ConstructFromPointer), h256(r.mix_hash, h256::ConstructFromPointer)}; } diff --git a/libethcore/Ethasher.h b/libethcore/Ethasher.h index a10c206d1..8d1a1e84c 100644 --- a/libethcore/Ethasher.h +++ b/libethcore/Ethasher.h @@ -51,7 +51,7 @@ public: using LightType = void const*; using FullType = void const*; - LightType cache(BlockInfo const& _header); + LightType light(BlockInfo const& _header); bytesConstRef full(BlockInfo const& _header); static ethash_params params(BlockInfo const& _header); static ethash_params params(unsigned _n); @@ -99,7 +99,7 @@ private: static Ethasher* s_this; RecursiveMutex x_this; - std::map m_caches; + std::map m_lights; std::map m_fulls; }; diff --git a/test/dagger.cpp b/test/dagger.cpp index 4dda9c4fc..4abba5090 100644 --- a/test/dagger.cpp +++ b/test/dagger.cpp @@ -64,7 +64,7 @@ BOOST_AUTO_TEST_CASE(basic_test) unsigned cacheSize(o["cache_size"].get_int()); h256 cacheHash(o["cache_hash"].get_str()); BOOST_REQUIRE_EQUAL(Ethasher::get()->params(header).cache_size, cacheSize); - BOOST_REQUIRE_EQUAL(sha3(bytesConstRef((byte const*)Ethasher::get()->cache(header), cacheSize)), cacheHash); + BOOST_REQUIRE_EQUAL(sha3(bytesConstRef((byte const*)Ethasher::get()->light(header), cacheSize)), cacheHash); #if TEST_FULL unsigned fullSize(o["full_size"].get_int()); From 3053282582721d99ea6a190e43e9685a420c5bf2 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 5 Apr 2015 15:17:39 +0200 Subject: [PATCH 02/10] Avoid killing old DAGs for now. Presents a memory leak to the tune of 1GB/epoch (!!!) but this will all be fixed when @LefterisJP introduces ethash-side memoisation. --- libethcore/Ethasher.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libethcore/Ethasher.cpp b/libethcore/Ethasher.cpp index 58a0aa8d7..e8e4d0b0e 100644 --- a/libethcore/Ethasher.cpp +++ b/libethcore/Ethasher.cpp @@ -82,11 +82,13 @@ bytesConstRef Ethasher::full(BlockInfo const& _header) RecursiveGuard l(x_this); if (!m_fulls.count(_header.seedHash())) { - if (!m_fulls.empty()) + // @memoryleak @bug place it on a pile for deletion - perhaps use shared_ptr. +/* if (!m_fulls.empty()) { delete [] m_fulls.begin()->second.data(); m_fulls.erase(m_fulls.begin()); - } + }*/ + try { boost::filesystem::create_directories(getDataDir("ethash")); } catch (...) {} From 105be32bb4bfaf1f4e49f5993ffb3b863228b5eb Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 5 Apr 2015 16:33:51 +0200 Subject: [PATCH 03/10] Decent transaction import result provision. Give network a hint about what's going on for peer backoffs. Avoid sleeping in main loop when there's still work on. --- eth/main.cpp | 8 ++++---- libethcore/Common.h | 10 ++++++++++ libethereum/BlockChain.cpp | 6 +++--- libethereum/BlockChain.h | 2 +- libethereum/BlockQueue.h | 14 +++----------- libethereum/Client.cpp | 15 +++++++++++---- libethereum/ClientBase.cpp | 4 ++-- libethereum/EthereumPeer.cpp | 20 ++++++++++++++++++-- libethereum/TransactionQueue.cpp | 12 +++++++----- libethereum/TransactionQueue.h | 8 ++++---- libp2p/Capability.cpp | 2 +- libp2p/Capability.h | 2 +- libp2p/Session.cpp | 4 +++- libp2p/Session.h | 2 +- test/blockchain.cpp | 2 +- 15 files changed, 70 insertions(+), 41 deletions(-) diff --git a/eth/main.cpp b/eth/main.cpp index 3cdcba472..9ec6dec98 100644 --- a/eth/main.cpp +++ b/eth/main.cpp @@ -120,7 +120,7 @@ void help() << " -j,--json-rpc Enable JSON-RPC server (default: off)." << endl << " --json-rpc-port Specify JSON-RPC server port (implies '-j', default: " << SensibleHttpPort << ")." << endl #endif - << " -K,--kill-blockchain First kill the blockchain." << endl + << " -K,--kill First kill the blockchain." << endl << " --listen-ip Listen on the given port for incoming connections (default: 30303)." << endl << " -l,--listen Listen on the given IP for incoming connections (default: 0.0.0.0)." << endl << " -u,--public-ip Force public ip to given (default: auto)." << endl @@ -129,7 +129,7 @@ void help() << " -o,--mode Start a full node or a peer node (Default: full)." << endl << " -p,--port Connect to remote port (default: 30303)." << endl << " -P,--priority <0 - 100> Default % priority of a transaction (default: 50)." << endl - << " -R,--rebuild-blockchain First rebuild the blockchain from the existing database." << endl + << " -R,--rebuild First rebuild the blockchain from the existing database." << endl << " -r,--remote Connect to remote host (default: none)." << endl << " -s,--secret Set the secret key for use with send command (default: auto)." << endl << " -t,--miners Number of mining threads to start (Default: " << thread::hardware_concurrency() << ")" << endl @@ -274,9 +274,9 @@ int main(int argc, char** argv) return -1; } } - else if (arg == "-K" || arg == "--kill-blockchain") + else if (arg == "-K" || arg == "--kill-blockchain" || arg == "--kill") killChain = WithExisting::Kill; - else if (arg == "-B" || arg == "--rebuild-blockchain") + else if (arg == "-B" || arg == "--rebuild") killChain = WithExisting::Verify; else if ((arg == "-c" || arg == "--client-name") && i + 1 < argc) clientName = argv[++i]; diff --git a/libethcore/Common.h b/libethcore/Common.h index aabe663cd..56f1b704a 100644 --- a/libethcore/Common.h +++ b/libethcore/Common.h @@ -85,5 +85,15 @@ enum class RelativeBlock: BlockNumber Pending = PendingBlock }; +enum class ImportResult +{ + Success = 0, + UnknownParent, + FutureTime, + AlreadyInChain, + AlreadyKnown, + Malformed +}; + } } diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 832d9465f..ae319cc14 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -267,7 +267,7 @@ LastHashes BlockChain::lastHashes(unsigned _n) const return m_lastLastHashes; } -h256s BlockChain::sync(BlockQueue& _bq, OverlayDB const& _stateDB, unsigned _max) +pair BlockChain::sync(BlockQueue& _bq, OverlayDB const& _stateDB, unsigned _max) { _bq.tick(*this); @@ -295,8 +295,8 @@ h256s BlockChain::sync(BlockQueue& _bq, OverlayDB const& _stateDB, unsigned _max catch (...) {} } - _bq.doneDrain(); - return ret; + bool yetMore = _bq.doneDrain(); + return make_pair(ret, yetMore); } h256s BlockChain::attemptImport(bytes const& _block, OverlayDB const& _stateDB, bool _force) noexcept diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h index a45be0ab0..add4b43cd 100644 --- a/libethereum/BlockChain.h +++ b/libethereum/BlockChain.h @@ -99,7 +99,7 @@ public: void process(); /// Sync the chain with any incoming blocks. All blocks should, if processed in order - h256s sync(BlockQueue& _bq, OverlayDB const& _stateDB, unsigned _max); + std::pair sync(BlockQueue& _bq, OverlayDB const& _stateDB, unsigned _max); /// Attempt to import the given block directly into the CanonBlockChain and sync with the state DB. /// @returns the block hashes of any blocks that came into/went out of the canonical block chain. diff --git a/libethereum/BlockQueue.h b/libethereum/BlockQueue.h index 9c473d766..f34a53812 100644 --- a/libethereum/BlockQueue.h +++ b/libethereum/BlockQueue.h @@ -26,6 +26,7 @@ #include #include #include +#include namespace dev { @@ -37,16 +38,6 @@ class BlockChain; struct BlockQueueChannel: public LogChannel { static const char* name() { return "[]Q"; } static const int verbosity = 4; }; #define cblockq dev::LogOutputStream() -enum class ImportResult -{ - Success = 0, - UnknownParent, - FutureTime, - AlreadyInChain, - AlreadyKnown, - Malformed -}; - /** * @brief A queue of blocks. Sits between network or other I/O and the BlockChain. * Sorts them ready for blockchain insertion (with the BlockChain::sync() method). @@ -66,7 +57,8 @@ public: void drain(std::vector& o_out, unsigned _max); /// Must be called after a drain() call. Notes that the drained blocks have been imported into the blockchain, so we can forget about them. - void doneDrain() { WriteGuard l(m_lock); m_drainingSet.clear(); } + /// @returns true iff there are additional blocks ready to be processed. + bool doneDrain() { WriteGuard l(m_lock); m_drainingSet.clear(); return !m_readySet.empty(); } /// Notify the queue that the chain has changed and a new block has attained 'ready' status (i.e. is in the chain). void noteReady(h256 _b) { WriteGuard l(m_lock); noteReadyWithoutWriteGuard(_b); } diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index 3d9f17335..2f65e5f4b 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -120,7 +120,7 @@ void BasicGasPricer::update(BlockChain const& _bc) Client::Client(p2p::Host* _extNet, std::string const& _dbPath, WithExisting _forceAction, u256 _networkId, int _miners): Worker("eth"), m_vc(_dbPath), - m_bc(_dbPath, max(m_vc.action(), _forceAction), [](unsigned d, unsigned t){ cerr << "REVISING BLOCKCHAIN: Processed " << d << " of " << t << "..." << endl; }), + m_bc(_dbPath, max(m_vc.action(), _forceAction), [](unsigned d, unsigned t){ cerr << "REVISING BLOCKCHAIN: Processed " << d << " of " << t << "...\r"; }), m_gp(new TrivialGasPricer), m_stateDB(State::openDB(_dbPath, max(m_vc.action(), _forceAction))), m_preMine(Address(), m_stateDB), @@ -441,6 +441,8 @@ void Client::doWork() { // TODO: Use condition variable rather than polling. + bool stillGotWork = false; + cworkin << "WORK"; h256Set changeds; @@ -496,7 +498,10 @@ void Client::doWork() cwork << "BQ ==> CHAIN ==> STATE"; OverlayDB db = m_stateDB; x_stateDB.unlock(); - h256s newBlocks = m_bc.sync(m_bq, db, 100); // TODO: remove transactions from m_tq nicely rather than relying on out of date nonce later on. + h256s newBlocks; + bool sgw; + tie(newBlocks, sgw) = m_bc.sync(m_bq, db, 100); // TODO: remove transactions from m_tq nicely rather than relying on out of date nonce later on. + stillGotWork = stillGotWork | sgw; if (newBlocks.size()) { for (auto i: newBlocks) @@ -544,7 +549,9 @@ void Client::doWork() noteChanged(changeds); cworkout << "WORK"; - this_thread::sleep_for(chrono::milliseconds(100)); + if (!stillGotWork) + this_thread::sleep_for(chrono::milliseconds(100)); + if (chrono::system_clock::now() - m_lastGarbageCollection > chrono::seconds(5)) { // watches garbage collection @@ -601,7 +608,7 @@ void Client::inject(bytesConstRef _rlp) { startWorking(); - m_tq.attemptImport(_rlp); + m_tq.import(_rlp); } void Client::flushTransactions() diff --git a/libethereum/ClientBase.cpp b/libethereum/ClientBase.cpp index f949391fd..fae534ff8 100644 --- a/libethereum/ClientBase.cpp +++ b/libethereum/ClientBase.cpp @@ -44,7 +44,7 @@ void ClientBase::submitTransaction(Secret _secret, u256 _value, Address _dest, b u256 n = postMine().transactionsFrom(toAddress(_secret)); Transaction t(_value, _gasPrice, _gas, _dest, _data, n, _secret); - m_tq.attemptImport(t.rlp()); + m_tq.import(t.rlp()); StructuredLogger::transactionReceived(t.sha3().abridged(), t.sender().abridged()); cnote << "New transaction " << t; @@ -56,7 +56,7 @@ Address ClientBase::submitTransaction(Secret _secret, u256 _endowment, bytes con u256 n = postMine().transactionsFrom(toAddress(_secret)); Transaction t(_endowment, _gasPrice, _gas, _init, n, _secret); - m_tq.attemptImport(t.rlp()); + m_tq.import(t.rlp()); StructuredLogger::transactionReceived(t.sha3().abridged(), t.sender().abridged()); cnote << "New transaction " << t; diff --git a/libethereum/EthereumPeer.cpp b/libethereum/EthereumPeer.cpp index f318a1757..15ce5c3f6 100644 --- a/libethereum/EthereumPeer.cpp +++ b/libethereum/EthereumPeer.cpp @@ -326,15 +326,27 @@ bool EthereumPeer::interpret(unsigned _id, RLP const& _r) case TransactionsPacket: { clogS(NetMessageSummary) << "Transactions (" << dec << _r.itemCount() << "entries)"; - addRating(_r.itemCount()); Guard l(x_knownTransactions); for (unsigned i = 0; i < _r.itemCount(); ++i) { auto h = sha3(_r[i].data()); m_knownTransactions.insert(h); - if (!host()->m_tq.import(_r[i].data())) + ImportResult ir = host()->m_tq.import(_r[i].data()); + switch (ir) + { + case ImportResult::Malformed: + addRating(-100); + break; + case ImportResult::AlreadyKnown: // if we already had the transaction, then don't bother sending it on. + addRating(0); + break; + case ImportResult::Success: + addRating(100); host()->m_transactionsSent.insert(h); + break; + default:; + } } break; } @@ -352,6 +364,7 @@ bool EthereumPeer::interpret(unsigned _id, RLP const& _r) for (unsigned i = 0; i < c && p; ++i, p = host()->m_chain.details(p).parent) s << p; sealAndSend(s); + addRating(0); break; } case BlockHashesPacket: @@ -370,6 +383,7 @@ bool EthereumPeer::interpret(unsigned _id, RLP const& _r) } for (unsigned i = 0; i < _r.itemCount(); ++i) { + addRating(1); auto h = _r[i].toHash(); if (host()->m_chain.isKnown(h)) { @@ -398,6 +412,7 @@ bool EthereumPeer::interpret(unsigned _id, RLP const& _r) ++n; } } + addRating(0); RLPStream s; prep(s, BlocksPacket, n).appendRaw(rlp, n); sealAndSend(s); @@ -497,6 +512,7 @@ bool EthereumPeer::interpret(unsigned _id, RLP const& _r) setNeedsSyncing(h, _r[1].toInt()); break; } + Guard l(x_knownBlocks); m_knownBlocks.insert(h); } diff --git a/libethereum/TransactionQueue.cpp b/libethereum/TransactionQueue.cpp index 803d320ee..26962ed90 100644 --- a/libethereum/TransactionQueue.cpp +++ b/libethereum/TransactionQueue.cpp @@ -28,14 +28,16 @@ using namespace std; using namespace dev; using namespace dev::eth; -bool TransactionQueue::import(bytesConstRef _transactionRLP) +ImportResult TransactionQueue::import(bytesConstRef _transactionRLP) { // Check if we already know this transaction. h256 h = sha3(_transactionRLP); UpgradableGuard l(m_lock); + // TODO: keep old transactions around and check in State for nonce validity + if (m_known.count(h)) - return false; + return ImportResult::AlreadyKnown; try { @@ -52,15 +54,15 @@ bool TransactionQueue::import(bytesConstRef _transactionRLP) catch (Exception const& _e) { cwarn << "Ignoring invalid transaction: " << diagnostic_information(_e); - return false; + return ImportResult::Malformed; } catch (std::exception const& _e) { cwarn << "Ignoring invalid transaction: " << _e.what(); - return false; + return ImportResult::Malformed; } - return true; + return ImportResult::Success; } void TransactionQueue::setFuture(std::pair const& _t) diff --git a/libethereum/TransactionQueue.h b/libethereum/TransactionQueue.h index b104b98ca..cf40e1209 100644 --- a/libethereum/TransactionQueue.h +++ b/libethereum/TransactionQueue.h @@ -23,8 +23,8 @@ #include #include -#include "libethcore/Common.h" #include +#include "libethcore/Common.h" #include "Transaction.h" namespace dev @@ -34,6 +34,7 @@ namespace eth class BlockChain; + /** * @brief A queue of Transactions, each stored as RLP. * @threadsafe @@ -41,9 +42,8 @@ class BlockChain; class TransactionQueue { public: - bool attemptImport(bytesConstRef _tx) { try { import(_tx); return true; } catch (...) { return false; } } - bool attemptImport(bytes const& _tx) { return attemptImport(&_tx); } - bool import(bytesConstRef _tx); + ImportResult import(bytes const& _tx) { return import(&_tx); } + ImportResult import(bytesConstRef _tx); void drop(h256 _txHash); diff --git a/libp2p/Capability.cpp b/libp2p/Capability.cpp index dccc130cd..f59fd8cdd 100644 --- a/libp2p/Capability.cpp +++ b/libp2p/Capability.cpp @@ -53,7 +53,7 @@ void Capability::sealAndSend(RLPStream& _s) m_session->sealAndSend(_s); } -void Capability::addRating(unsigned _r) +void Capability::addRating(int _r) { m_session->addRating(_r); } diff --git a/libp2p/Capability.h b/libp2p/Capability.h index ad8127bb5..d09391655 100644 --- a/libp2p/Capability.h +++ b/libp2p/Capability.h @@ -52,7 +52,7 @@ protected: RLPStream& prep(RLPStream& _s, unsigned _id, unsigned _args = 0); void sealAndSend(RLPStream& _s); - void addRating(unsigned _r); + void addRating(int _r); private: Session* m_session; diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index 133d8a30b..0bf07bbe1 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -77,12 +77,14 @@ NodeId Session::id() const return m_peer ? m_peer->id : NodeId(); } -void Session::addRating(unsigned _r) +void Session::addRating(int _r) { if (m_peer) { m_peer->m_rating += _r; m_peer->m_score += _r; + if (_r >= 0) + m_peer->noteSessionGood(); } } diff --git a/libp2p/Session.h b/libp2p/Session.h index 51db5adc3..95053d2a9 100644 --- a/libp2p/Session.h +++ b/libp2p/Session.h @@ -74,7 +74,7 @@ public: void sealAndSend(RLPStream& _s); int rating() const; - void addRating(unsigned _r); + void addRating(int _r); void addNote(std::string const& _k, std::string const& _v) { m_info.notes[_k] = _v; } diff --git a/test/blockchain.cpp b/test/blockchain.cpp index ffb55da30..21345abfd 100644 --- a/test/blockchain.cpp +++ b/test/blockchain.cpp @@ -98,7 +98,7 @@ void doBlockchainTests(json_spirit::mValue& _v, bool _fillin) { mObject tx = txObj.get_obj(); importer.importTransaction(tx); - if (!txs.attemptImport(importer.m_transaction.rlp())) + if (txs.import(importer.m_transaction.rlp()) != ImportResult::Success) cnote << "failed importing transaction\n"; } From 80df4e11fce9ce66810f5b3aa03ee69c1887b3a9 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 5 Apr 2015 16:59:51 +0200 Subject: [PATCH 04/10] Nicer cmake read out. Add profiling support. --- CMakeLists.txt | 77 ++++++++++++++++++++++++++++++++- cmake/EthCompilerSettings.cmake | 6 ++- 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 80bad7929..1fbb83ff8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,7 @@ function(createDefaultCacheConfig) set(SOLIDITY ON CACHE BOOL "Build the Solidity language components (requried unless HEADLESS)") set(USENPM OFF CACHE BOOL "Use npm to recompile ethereum.js if it was changed") set(ETHASHCL OFF CACHE BOOL "Build in support for GPU mining via OpenCL") + set(PROFILING OFF CACHE BOOL "Build in support for profiling") endfunction() @@ -56,6 +57,7 @@ function(configureProject) if (HEADLESS OR JUSTTESTS) add_definitions(-DETH_HEADLESS) endif() + endfunction() set(CPPETHEREUM 1) @@ -133,8 +135,79 @@ configureProject() # Force chromium. set (ETH_HAVE_WEBENGINE 1) -message(STATUS "CMAKE_VERSION: ${CMAKE_VERSION}") -message("-- VMTRACE: ${VMTRACE}; PARANOIA: ${PARANOIA}; HEADLESS: ${HEADLESS}; JSONRPC: ${JSONRPC}; EVMJIT: ${EVMJIT}; FATDB: ${FATDB}; CHROMIUM: ${ETH_HAVE_WEBENGINE}; USENPM: ${USENPM}; ETHASHCL: ${ETHASHCL}") +# Normalise build options +# TODO: Abstract into something sensible and move into a function. +if (PARANOIA) + set(PARANOIA ON) +else () + set (PARANOIA OFF) +endif () +if (VMTRACE) + set(VMTRACE ON) +else () + set (VMTRACE OFF) +endif () +if (ETHASHCL) + set(ETHASHCL ON) +else () + set (ETHASHCL OFF) +endif() +if (EVMJIT) + set(EVMJIT ON) +else () + set (EVMJIT OFF) +endif() +if (FATDB) + set(FATDB ON) +else () + set (FATDB OFF) +endif() +if (SOLIDITY) + set(SOLIDITY ON) +else () + set (SOLIDITY OFF) +endif() +if (HEADLESS) + set(HEADLESS ON) +else () + set (HEADLESS OFF) +endif () +if (JUSTTESTS) + set(JUSTTESTS ON) +else () + set (JUSTTESTS OFF) +endif () +if (JSONRPC) + set(JSONRPC ON) +else () + set (JSONRPC OFF) +endif () +if (USENPM) + set(USENPM ON) +else () + set (USENPM OFF) +endif () +if (PROFILING) + set(PROFILING ON) +else () + set (PROFILING OFF) +endif () + +message(STATUS "CMake Version: ${CMAKE_VERSION}") +message("-- VM execution tracing VMTRACE ${VMTRACE}") +message("-- Profiling support PROFILING ${PROFILING}") +message("-- Additional (SLOW) database checking PARANOIA ${PARANOIA}") +message("-- Full database exploring FATDB ${FATDB}") +message("-- Build Javascript components from source USENPM ${USENPM}") +message("-- Build only headless components HEADLESS ${HEADLESS}") +message("-- Build only tests JUSTTESTS ${JUSTTESTS}") +message("-- Build Solidity language components SOLIDITY ${SOLIDITY}") +message("-- Build OpenCL components ETHASHCL ${ETHASHCL}") +message("-- Build LLVM-based JIT EVM EVMJIT ${EVMJIT}") +message("-- Build with support for JSON-RPC JSONRPC ${JSONRPC}") +message("-- Build with support for Chromium ${ETH_HAVE_WEBENGINE}") +message("------------------------------------------------------------------------") +message("") # Default TARGET_PLATFORM to "linux". diff --git a/cmake/EthCompilerSettings.cmake b/cmake/EthCompilerSettings.cmake index dd14b0650..731b258c7 100755 --- a/cmake/EthCompilerSettings.cmake +++ b/cmake/EthCompilerSettings.cmake @@ -7,9 +7,13 @@ if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -DETH_DEBUG") set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os -DNDEBUG -DETH_RELEASE") set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG -DETH_RELEASE") - set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DETH_DEBUG") + set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DETH_RELEASE") set(ETH_SHARED 1) + if (PROFILING) + set(CMAKE_CXX_FLAGS "-pg ${CMAKE_CXX_FLAGS}") + endif () + execute_process( COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) if (NOT (GCC_VERSION VERSION_GREATER 4.7 OR GCC_VERSION VERSION_EQUAL 4.7)) From 8111a3e6b327f59a5f05f418d814f2f361220a11 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 5 Apr 2015 17:28:52 +0200 Subject: [PATCH 05/10] Ability to disable serpent building. --- CMakeLists.txt | 66 ++++++++++++++++++++------------- cmake/EthCompilerSettings.cmake | 2 +- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fbb83ff8..b174f9a28 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,11 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") # user defined, defaults # Normally, set(...CACHE...) creates cache variables, but does not modify them. function(createDefaultCacheConfig) + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") + set(SERPENT_DEFAULT OFF) + else () + set(SERPENT_DEFAULT ON) + endif () set(HEADLESS OFF CACHE BOOL "Do not compile GUI (AlethZero)") set(VMTRACE OFF CACHE BOOL "VM tracing and run-time checks (useful for cross-implementation VM debugging)") set(PARANOIA OFF CACHE BOOL "Additional run-time checks") @@ -21,7 +26,8 @@ function(createDefaultCacheConfig) set(EVMJIT OFF CACHE BOOL "Build a just-in-time compiler for EVM code (requires LLVM)") set(FATDB OFF CACHE BOOL "Build with ability to list entries in the Trie. Doubles DB size, slows everything down, but good for looking at state diffs and trie contents.") set(JUSTTESTS OFF CACHE BOOL "Build only for tests.") - set(SOLIDITY ON CACHE BOOL "Build the Solidity language components (requried unless HEADLESS)") + set(SOLIDITY ON CACHE BOOL "Build the Solidity language components (required unless HEADLESS)") + set(SERPENT ${SERPENT_DEFAULT} CACHE BOOL "Build the Serpent language components (required unless HEADLESS)") set(USENPM OFF CACHE BOOL "Use npm to recompile ethereum.js if it was changed") set(ETHASHCL OFF CACHE BOOL "Build in support for GPU mining via OpenCL") set(PROFILING OFF CACHE BOOL "Build in support for profiling") @@ -140,72 +146,82 @@ set (ETH_HAVE_WEBENGINE 1) if (PARANOIA) set(PARANOIA ON) else () - set (PARANOIA OFF) + set(PARANOIA OFF) endif () if (VMTRACE) set(VMTRACE ON) else () - set (VMTRACE OFF) + set(VMTRACE OFF) endif () if (ETHASHCL) set(ETHASHCL ON) else () - set (ETHASHCL OFF) + set(ETHASHCL OFF) endif() if (EVMJIT) set(EVMJIT ON) else () - set (EVMJIT OFF) + set(EVMJIT OFF) endif() if (FATDB) set(FATDB ON) else () - set (FATDB OFF) + set(FATDB OFF) endif() if (SOLIDITY) set(SOLIDITY ON) else () - set (SOLIDITY OFF) + set(SOLIDITY OFF) +endif() +if (SERPENT) + set(SERPENT ON) +else () + set(SERPENT OFF) endif() if (HEADLESS) set(HEADLESS ON) else () - set (HEADLESS OFF) + set(HEADLESS OFF) endif () if (JUSTTESTS) set(JUSTTESTS ON) else () - set (JUSTTESTS OFF) + set(JUSTTESTS OFF) endif () if (JSONRPC) set(JSONRPC ON) else () - set (JSONRPC OFF) + set(JSONRPC OFF) endif () if (USENPM) set(USENPM ON) else () - set (USENPM OFF) + set(USENPM OFF) endif () if (PROFILING) set(PROFILING ON) else () - set (PROFILING OFF) + set(PROFILING OFF) +endif () +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Release") endif () message(STATUS "CMake Version: ${CMAKE_VERSION}") -message("-- VM execution tracing VMTRACE ${VMTRACE}") -message("-- Profiling support PROFILING ${PROFILING}") -message("-- Additional (SLOW) database checking PARANOIA ${PARANOIA}") -message("-- Full database exploring FATDB ${FATDB}") -message("-- Build Javascript components from source USENPM ${USENPM}") -message("-- Build only headless components HEADLESS ${HEADLESS}") -message("-- Build only tests JUSTTESTS ${JUSTTESTS}") -message("-- Build Solidity language components SOLIDITY ${SOLIDITY}") -message("-- Build OpenCL components ETHASHCL ${ETHASHCL}") -message("-- Build LLVM-based JIT EVM EVMJIT ${EVMJIT}") -message("-- Build with support for JSON-RPC JSONRPC ${JSONRPC}") -message("-- Build with support for Chromium ${ETH_HAVE_WEBENGINE}") +message("-- Build type CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE}") +message("-- VM execution tracing VMTRACE ${VMTRACE}") +message("-- Profiling support PROFILING ${PROFILING}") +message("-- Additional (SLOW) database checking PARANOIA ${PARANOIA}") +message("-- Full database exploring FATDB ${FATDB}") +message("-- Build Javascript components from source USENPM ${USENPM}") +message("-- Build only headless components HEADLESS ${HEADLESS}") +message("-- Build only tests JUSTTESTS ${JUSTTESTS}") +message("-- Build Solidity language components SOLIDITY ${SOLIDITY}") +message("-- Build Serpent language components SERPENT ${SERPENT}") +message("-- Build OpenCL components ETHASHCL ${ETHASHCL}") +message("-- Build LLVM-based JIT EVM EVMJIT ${EVMJIT}") +message("-- Build with support for JSON-RPC JSONRPC ${JSONRPC}") +message("-- Build with support for Chromium ${ETH_HAVE_WEBENGINE}") message("------------------------------------------------------------------------") message("") @@ -239,7 +255,7 @@ add_subdirectory(libdevcore) add_subdirectory(libevmcore) add_subdirectory(liblll) -if (NOT ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")) +if (SERPENT) add_subdirectory(libserpent) add_subdirectory(sc) endif () diff --git a/cmake/EthCompilerSettings.cmake b/cmake/EthCompilerSettings.cmake index 731b258c7..b88c3adcb 100755 --- a/cmake/EthCompilerSettings.cmake +++ b/cmake/EthCompilerSettings.cmake @@ -11,7 +11,7 @@ if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") set(ETH_SHARED 1) if (PROFILING) - set(CMAKE_CXX_FLAGS "-pg ${CMAKE_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS "-pg -g ${CMAKE_CXX_FLAGS}") endif () execute_process( From 9fb30b141e57cd8b87cf066efe5fb0be438f6f7b Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 5 Apr 2015 17:46:12 +0200 Subject: [PATCH 06/10] Allow Solidity build to be disabled. --- CMakeLists.txt | 29 +++++++++++++++++--------- libethereum/BlockChain.cpp | 2 ++ test/Assembly.cpp | 3 +++ test/CMakeLists.txt | 4 +++- test/SolidityABIJSON.cpp | 3 +++ test/SolidityCompiler.cpp | 3 +++ test/SolidityEndToEndTest.cpp | 4 +++- test/SolidityExpressionCompiler.cpp | 4 +++- test/SolidityInterface.cpp | 4 ++++ test/SolidityNameAndTypeResolution.cpp | 3 +++ test/SolidityNatspecJSON.cpp | 4 ++++ test/SolidityOptimizer.cpp | 5 ++++- test/SolidityParser.cpp | 3 +++ test/SolidityScanner.cpp | 4 ++++ test/SolidityTypes.cpp | 4 ++++ test/solidityExecutionFramework.h | 1 - 16 files changed, 65 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b174f9a28..8c02948a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -203,12 +203,23 @@ if (PROFILING) else () set(PROFILING OFF) endif () -if (NOT CMAKE_BUILD_TYPE) + +# Default CMAKE_BUILD_TYPE to "Release". +set(CMAKE_BUILD_TYPE CACHE STRING "Relase") +if ("x${CMAKE_BUILD_TYPE}" STREQUAL "x") set(CMAKE_BUILD_TYPE "Release") endif () -message(STATUS "CMake Version: ${CMAKE_VERSION}") +# Default TARGET_PLATFORM to "linux". +set(TARGET_PLATFORM CACHE STRING "linux") +if ("x${TARGET_PLATFORM}" STREQUAL "x") + set(TARGET_PLATFORM "linux") +endif () + +message("------------------------------------------------------------------------") +message("-- CMake Version ${CMAKE_VERSION}") message("-- Build type CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE}") +message("-- Target platform TARGET_PLATFORM ${TARGET_PLATFORM}") message("-- VM execution tracing VMTRACE ${VMTRACE}") message("-- Profiling support PROFILING ${PROFILING}") message("-- Additional (SLOW) database checking PARANOIA ${PARANOIA}") @@ -226,12 +237,6 @@ message("----------------------------------------------------------------------- message("") -# Default TARGET_PLATFORM to "linux". -set(TARGET_PLATFORM CACHE STRING "linux") -if ("x${TARGET_PLATFORM}" STREQUAL "x") - set(TARGET_PLATFORM "linux") -endif () - if ("${TARGET_PLATFORM}" STREQUAL "linux") set(CMAKE_THREAD_LIBS_INIT pthread) endif () @@ -260,11 +265,15 @@ if (SERPENT) add_subdirectory(sc) endif () -add_subdirectory(libsolidity) +if (SOLIDITY) + add_subdirectory(libsolidity) +endif () if (NOT JUSTTESTS) add_subdirectory(lllc) - add_subdirectory(solc) + if (SOLIDITY) + add_subdirectory(solc) + endif () endif() if (JSONRPC) diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index ae319cc14..488937c41 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -208,6 +208,8 @@ void BlockChain::rebuild(std::string const& _path, std::function 1000) + exit(0); try { bytes b = block(queryExtras(h256(u256(d)), m_blockHashes, x_blockHashes, NullBlockHash, oldExtrasDB).value); diff --git a/test/Assembly.cpp b/test/Assembly.cpp index fbc8e47b5..fab260a9d 100644 --- a/test/Assembly.cpp +++ b/test/Assembly.cpp @@ -20,6 +20,8 @@ * Unit tests for Assembly Items from evmcore/Assembly.h */ +#if ETH_SOLIDITY + #include #include #include @@ -119,3 +121,4 @@ BOOST_AUTO_TEST_SUITE_END() } } // end namespaces +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 01681dbef..5bd44091d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -44,7 +44,9 @@ target_link_libraries(testeth ${CURL_LIBRARIES}) target_link_libraries(testeth ethereum) target_link_libraries(testeth ethcore) target_link_libraries(testeth secp256k1) -target_link_libraries(testeth solidity) +if (SOLIDITY) + target_link_libraries(testeth solidity) +endif () target_link_libraries(testeth testutils) if (NOT HEADLESS AND NOT JUSTTESTS) target_link_libraries(testeth webthree) diff --git a/test/SolidityABIJSON.cpp b/test/SolidityABIJSON.cpp index f7f968ea8..bbe5fd8c4 100644 --- a/test/SolidityABIJSON.cpp +++ b/test/SolidityABIJSON.cpp @@ -19,6 +19,7 @@ * @date 2014 * Unit tests for the solidity compiler JSON Interface output. */ +#if ETH_SOLIDITY #include "TestHelper.h" #include @@ -500,3 +501,5 @@ BOOST_AUTO_TEST_SUITE_END() } } } + +#endif diff --git a/test/SolidityCompiler.cpp b/test/SolidityCompiler.cpp index 1369b038f..bb16c88cd 100644 --- a/test/SolidityCompiler.cpp +++ b/test/SolidityCompiler.cpp @@ -20,6 +20,8 @@ * Unit tests for the solidity compiler. */ +#if ETH_SOLIDITY + #include #include #include @@ -192,3 +194,4 @@ BOOST_AUTO_TEST_SUITE_END() } } // end namespaces +#endif diff --git a/test/SolidityEndToEndTest.cpp b/test/SolidityEndToEndTest.cpp index ea6ada603..b4da07892 100644 --- a/test/SolidityEndToEndTest.cpp +++ b/test/SolidityEndToEndTest.cpp @@ -1,4 +1,3 @@ - /* This file is part of cpp-ethereum. @@ -22,6 +21,8 @@ * Unit tests for the solidity expression compiler, testing the behaviour of the code. */ +#if ETH_SOLIDITY + #include #include #include @@ -3627,3 +3628,4 @@ BOOST_AUTO_TEST_SUITE_END() } } // end namespaces +#endif diff --git a/test/SolidityExpressionCompiler.cpp b/test/SolidityExpressionCompiler.cpp index 7034085ef..b748d887d 100644 --- a/test/SolidityExpressionCompiler.cpp +++ b/test/SolidityExpressionCompiler.cpp @@ -1,4 +1,3 @@ - /* This file is part of cpp-ethereum. @@ -21,6 +20,8 @@ * Unit tests for the solidity expression compiler. */ +#if ETH_SOLIDITY + #include #include @@ -491,3 +492,4 @@ BOOST_AUTO_TEST_SUITE_END() } } // end namespaces +#endif diff --git a/test/SolidityInterface.cpp b/test/SolidityInterface.cpp index 48e2fd7aa..c836f0fa7 100644 --- a/test/SolidityInterface.cpp +++ b/test/SolidityInterface.cpp @@ -20,6 +20,8 @@ * Unit tests for generating source interfaces for Solidity contracts. */ +#if ETH_SOLIDITY + #include "TestHelper.h" #include #include @@ -147,3 +149,5 @@ BOOST_AUTO_TEST_SUITE_END() } } } + +#endif diff --git a/test/SolidityNameAndTypeResolution.cpp b/test/SolidityNameAndTypeResolution.cpp index 531f3bc13..74a488883 100644 --- a/test/SolidityNameAndTypeResolution.cpp +++ b/test/SolidityNameAndTypeResolution.cpp @@ -20,6 +20,8 @@ * Unit tests for the name and type resolution of the solidity parser. */ +#if ETH_SOLIDITY + #include #include @@ -1627,3 +1629,4 @@ BOOST_AUTO_TEST_SUITE_END() } } // end namespaces +#endif diff --git a/test/SolidityNatspecJSON.cpp b/test/SolidityNatspecJSON.cpp index aeaad1966..28d657357 100644 --- a/test/SolidityNatspecJSON.cpp +++ b/test/SolidityNatspecJSON.cpp @@ -20,6 +20,8 @@ * Unit tests for the solidity compiler JSON Interface output. */ +#if ETH_SOLIDITY + #include "TestHelper.h" #include #include @@ -537,3 +539,5 @@ BOOST_AUTO_TEST_SUITE_END() } } } + +#endif diff --git a/test/SolidityOptimizer.cpp b/test/SolidityOptimizer.cpp index e69d5120e..4fedd642d 100644 --- a/test/SolidityOptimizer.cpp +++ b/test/SolidityOptimizer.cpp @@ -1,4 +1,3 @@ - /* This file is part of cpp-ethereum. @@ -21,6 +20,8 @@ * Tests for the Solidity optimizer. */ +#if ETH_SOLIDITY + #include #include #include @@ -573,3 +574,5 @@ BOOST_AUTO_TEST_SUITE_END() } } } // end namespaces + +#endif diff --git a/test/SolidityParser.cpp b/test/SolidityParser.cpp index 7640f91ad..b76f00656 100644 --- a/test/SolidityParser.cpp +++ b/test/SolidityParser.cpp @@ -20,6 +20,8 @@ * Unit tests for the solidity parser. */ +#if ETH_SOLIDITY + #include #include #include @@ -845,3 +847,4 @@ BOOST_AUTO_TEST_SUITE_END() } } // end namespaces +#endif diff --git a/test/SolidityScanner.cpp b/test/SolidityScanner.cpp index 8d3e53929..20b946ee0 100644 --- a/test/SolidityScanner.cpp +++ b/test/SolidityScanner.cpp @@ -20,6 +20,8 @@ * Unit tests for the solidity scanner. */ +#if ETH_SOLIDITY + #include #include @@ -286,3 +288,5 @@ BOOST_AUTO_TEST_SUITE_END() } } } // end namespaces + +#endif diff --git a/test/SolidityTypes.cpp b/test/SolidityTypes.cpp index 6b6306479..da8b48303 100644 --- a/test/SolidityTypes.cpp +++ b/test/SolidityTypes.cpp @@ -20,6 +20,8 @@ * Unit tests for the type system of Solidity. */ +#if ETH_SOLIDITY + #include #include @@ -91,3 +93,5 @@ BOOST_AUTO_TEST_SUITE_END() } } } + +#endif diff --git a/test/solidityExecutionFramework.h b/test/solidityExecutionFramework.h index 2451aa381..2134d424d 100644 --- a/test/solidityExecutionFramework.h +++ b/test/solidityExecutionFramework.h @@ -1,4 +1,3 @@ - /* This file is part of cpp-ethereum. From a33de145e71f17fa38f53351bc589cdbf33ddac0 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 5 Apr 2015 19:54:37 +0200 Subject: [PATCH 07/10] Don't check nonce by default. --- cmake/EthCompilerSettings.cmake | 6 +++++- libethcore/BlockInfo.cpp | 4 ++-- libethcore/BlockInfo.h | 14 +++++++------- libethereum/BlockChain.cpp | 19 +++++++++++++++++-- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/cmake/EthCompilerSettings.cmake b/cmake/EthCompilerSettings.cmake index b88c3adcb..d781ad2b3 100755 --- a/cmake/EthCompilerSettings.cmake +++ b/cmake/EthCompilerSettings.cmake @@ -11,7 +11,11 @@ if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") set(ETH_SHARED 1) if (PROFILING) - set(CMAKE_CXX_FLAGS "-pg -g ${CMAKE_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS "-g ${CMAKE_CXX_FLAGS}") + add_definitions(-DETH_PROFILING_GPERF) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -lprofiler") +# set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} -lprofiler") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lprofiler") endif () execute_process( diff --git a/libethcore/BlockInfo.cpp b/libethcore/BlockInfo.cpp index bbd78224f..f4b6f3b4e 100644 --- a/libethcore/BlockInfo.cpp +++ b/libethcore/BlockInfo.cpp @@ -69,10 +69,10 @@ h256 const& BlockInfo::seedHash() const return m_seedHash; } -BlockInfo BlockInfo::fromHeader(bytesConstRef _block, Strictness _s) +BlockInfo BlockInfo::fromHeader(bytesConstRef _header, Strictness _s) { BlockInfo ret; - ret.populateFromHeader(RLP(_block), _s); + ret.populateFromHeader(RLP(_header), _s); return ret; } diff --git a/libethcore/BlockInfo.h b/libethcore/BlockInfo.h index d262afca7..2ef196934 100644 --- a/libethcore/BlockInfo.h +++ b/libethcore/BlockInfo.h @@ -86,14 +86,14 @@ public: Nonce nonce; BlockInfo(); - explicit BlockInfo(bytes const& _block, Strictness _s = CheckEverything): BlockInfo(&_block, _s) {} - explicit BlockInfo(bytesConstRef _block, Strictness _s = CheckEverything); + explicit BlockInfo(bytes const& _block, Strictness _s = IgnoreNonce): BlockInfo(&_block, _s) {} + explicit BlockInfo(bytesConstRef _block, Strictness _s = IgnoreNonce); static h256 headerHash(bytes const& _block) { return headerHash(&_block); } static h256 headerHash(bytesConstRef _block); - static BlockInfo fromHeader(bytes const& _block, Strictness _s = CheckEverything) { return fromHeader(bytesConstRef(&_block), _s); } - static BlockInfo fromHeader(bytesConstRef _block, Strictness _s = CheckEverything); + static BlockInfo fromHeader(bytes const& _header, Strictness _s = IgnoreNonce) { return fromHeader(bytesConstRef(&_header), _s); } + static BlockInfo fromHeader(bytesConstRef _header, Strictness _s = IgnoreNonce); explicit operator bool() const { return timestamp != Invalid256; } @@ -119,9 +119,9 @@ public: void setEmpty(); - void populateFromHeader(RLP const& _header, Strictness _s = CheckEverything); - void populate(bytesConstRef _block, Strictness _s = CheckEverything); - void populate(bytes const& _block, Strictness _s = CheckEverything) { populate(&_block, _s); } + void populateFromHeader(RLP const& _header, Strictness _s = IgnoreNonce); + void populate(bytesConstRef _block, Strictness _s = IgnoreNonce); + void populate(bytes const& _block, Strictness _s = IgnoreNonce) { populate(&_block, _s); } void verifyInternals(bytesConstRef _block) const; void verifyParent(BlockInfo const& _parent) const; void populateFromParent(BlockInfo const& parent); diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 488937c41..0aea83de7 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -155,7 +155,9 @@ void BlockChain::open(std::string const& _path, WithExisting _we) m_extrasDB->Put(m_writeOptions, toSlice(m_genesisHash, ExtraDetails), (ldb::Slice)dev::ref(r)); } +#if ETH_PARANOIA checkConsistency(); +#endif // TODO: Implement ability to rebuild details map from DB. std::string l; @@ -175,10 +177,15 @@ void BlockChain::close() m_blocks.clear(); } +#include #define IGNORE_EXCEPTIONS(X) try { X; } catch (...) {} void BlockChain::rebuild(std::string const& _path, std::function const& _progress) { +#if ETH_PROFILING_GPERF + ProfilerStart("BlockChain_rebuild.log"); +#endif + unsigned originalNumber = number(); // Keep extras DB around, but under a temp name @@ -206,10 +213,14 @@ void BlockChain::rebuild(std::string const& _path, std::function 1000) - exit(0); + if (!(d % 1000)) + { + cerr << "\n1000 blocks in " << t.elapsed() << "s = " << (1000.0 / t.elapsed()) << "b/s" << endl; + t.restart(); + } try { bytes b = block(queryExtras(h256(u256(d)), m_blockHashes, x_blockHashes, NullBlockHash, oldExtrasDB).value); @@ -233,6 +244,10 @@ void BlockChain::rebuild(std::string const& _path, std::function Date: Sun, 5 Apr 2015 21:08:22 +0200 Subject: [PATCH 08/10] Move hash to a hidden function, and provide a datapath to it so avoid unneeded SHA3s. --- alethzero/MainWin.cpp | 4 +-- libethcore/BlockInfo.cpp | 33 ++++++++++++++--------- libethcore/BlockInfo.h | 21 ++++++++------- libethcore/Ethasher.cpp | 5 +++- libethereum/BlockChain.cpp | 18 ++++++++----- libethereum/BlockChain.h | 4 +-- libethereum/Client.cpp | 16 +++++------ libethereum/ClientBase.cpp | 2 +- libethereum/State.cpp | 31 ++++++++++----------- libethereum/State.h | 19 ++++++++++--- libtestutils/StateLoader.cpp | 2 +- libweb3jsonrpc/WebThreeStubServerBase.cpp | 2 +- mix/ClientModel.cpp | 2 +- mix/MixClient.cpp | 2 +- test/ClientBase.cpp | 18 +++++++++---- test/TestHelper.cpp | 7 +++-- test/TestHelper.h | 2 +- test/blockchain.cpp | 6 ++--- test/stateOriginal.cpp | 2 +- test/vm.cpp | 4 +-- 20 files changed, 122 insertions(+), 78 deletions(-) diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index af6e1a731..48b47c3a5 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -148,7 +148,7 @@ Main::Main(QWidget *parent) : cerr << "State root: " << CanonBlockChain::genesis().stateRoot << endl; auto block = CanonBlockChain::createGenesisBlock(); - cerr << "Block Hash: " << CanonBlockChain::genesis().hash << endl; + cerr << "Block Hash: " << CanonBlockChain::genesis().hash() << endl; cerr << "Block RLP: " << RLP(block) << endl; cerr << "Block Hex: " << toHex(block) << endl; cerr << "eth Network protocol version: " << eth::c_protocolVersion << endl; @@ -1491,7 +1491,7 @@ void Main::on_blocks_currentItemChanged() { BlockInfo uncle = BlockInfo::fromHeader(u.data()); char const* line = "
 "; - s << line << "Hash: " << uncle.hash << "" << "
"; + s << line << "Hash: " << uncle.hash() << "" << ""; s << line << "Parent: " << uncle.parentHash << "" << ""; s << line << "Number: " << uncle.number << "" << ""; s << line << "Coinbase: " << pretty(uncle.coinbaseAddress).toHtmlEscaped().toStdString() << " " << uncle.coinbaseAddress << "" << ""; diff --git a/libethcore/BlockInfo.cpp b/libethcore/BlockInfo.cpp index f4b6f3b4e..66f08d2bf 100644 --- a/libethcore/BlockInfo.cpp +++ b/libethcore/BlockInfo.cpp @@ -35,9 +35,9 @@ BlockInfo::BlockInfo(): timestamp(Invalid256) { } -BlockInfo::BlockInfo(bytesConstRef _block, Strictness _s) +BlockInfo::BlockInfo(bytesConstRef _block, Strictness _s, h256 const& _h) { - populate(_block, _s); + populate(_block, _s, _h); } void BlockInfo::setEmpty() @@ -57,8 +57,7 @@ void BlockInfo::setEmpty() extraData.clear(); mixHash = h256(); nonce = Nonce(); - m_seedHash = h256(); - hash = headerHash(WithNonce); + m_hash = m_seedHash = h256(); } h256 const& BlockInfo::seedHash() const @@ -69,10 +68,17 @@ h256 const& BlockInfo::seedHash() const return m_seedHash; } -BlockInfo BlockInfo::fromHeader(bytesConstRef _header, Strictness _s) +h256 const& BlockInfo::hash() const +{ + if (!m_hash) + m_hash = headerHash(WithNonce); + return m_hash; +} + +BlockInfo BlockInfo::fromHeader(bytesConstRef _header, Strictness _s, h256 const& _h) { BlockInfo ret; - ret.populateFromHeader(RLP(_header), _s); + ret.populateFromHeader(RLP(_header), _s, _h); return ret; } @@ -97,9 +103,11 @@ h256 BlockInfo::headerHash(bytesConstRef _block) return sha3(RLP(_block)[0].data()); } -void BlockInfo::populateFromHeader(RLP const& _header, Strictness _s) +void BlockInfo::populateFromHeader(RLP const& _header, Strictness _s, h256 const& _h) { - hash = dev::sha3(_header.data()); +// m_hash = dev::sha3(_header.data()); + m_hash = _h; + m_seedHash = h256(); int field = 0; try @@ -149,14 +157,14 @@ void BlockInfo::populateFromHeader(RLP const& _header, Strictness _s) } } -void BlockInfo::populate(bytesConstRef _block, Strictness _s) +void BlockInfo::populate(bytesConstRef _block, Strictness _s, h256 const& _h) { RLP root(_block); RLP header = root[0]; if (!header.isList()) BOOST_THROW_EXCEPTION(InvalidBlockFormat() << errinfo_comment("block header needs to be a list") << BadFieldError(0, header.data().toString())); - populateFromHeader(header, _s); + populateFromHeader(header, _s, _h); if (!root[1].isList()) BOOST_THROW_EXCEPTION(InvalidBlockFormat() << errinfo_comment("block transactions need to be a list") << BadFieldError(1, root[1].data().toString())); @@ -191,8 +199,9 @@ void BlockInfo::verifyInternals(bytesConstRef _block) const void BlockInfo::populateFromParent(BlockInfo const& _parent) { + m_hash = m_seedHash = h256(); stateRoot = _parent.stateRoot; - parentHash = _parent.hash; + parentHash = _parent.hash(); number = _parent.number + 1; gasLimit = selectGasLimit(_parent); gasUsed = 0; @@ -230,7 +239,7 @@ void BlockInfo::verifyParent(BlockInfo const& _parent) const // Check timestamp is after previous timestamp. if (parentHash) { - if (parentHash != _parent.hash) + if (parentHash != _parent.hash()) BOOST_THROW_EXCEPTION(InvalidParentHash()); if (timestamp <= _parent.timestamp) diff --git a/libethcore/BlockInfo.h b/libethcore/BlockInfo.h index 2ef196934..6a3ca68b2 100644 --- a/libethcore/BlockInfo.h +++ b/libethcore/BlockInfo.h @@ -68,7 +68,6 @@ struct BlockInfo { public: // TODO: make them all private! - h256 hash; ///< SHA3 hash of the block header! Not serialised (the only member not contained in a block header). h256 parentHash; h256 sha3Uncles; Address coinbaseAddress; @@ -86,14 +85,14 @@ public: Nonce nonce; BlockInfo(); - explicit BlockInfo(bytes const& _block, Strictness _s = IgnoreNonce): BlockInfo(&_block, _s) {} - explicit BlockInfo(bytesConstRef _block, Strictness _s = IgnoreNonce); + explicit BlockInfo(bytes const& _block, Strictness _s = IgnoreNonce, h256 const& _h = h256()): BlockInfo(&_block, _s, _h) {} + explicit BlockInfo(bytesConstRef _block, Strictness _s = IgnoreNonce, h256 const& _h = h256()); static h256 headerHash(bytes const& _block) { return headerHash(&_block); } static h256 headerHash(bytesConstRef _block); - static BlockInfo fromHeader(bytes const& _header, Strictness _s = IgnoreNonce) { return fromHeader(bytesConstRef(&_header), _s); } - static BlockInfo fromHeader(bytesConstRef _header, Strictness _s = IgnoreNonce); + static BlockInfo fromHeader(bytes const& _header, Strictness _s = IgnoreNonce, h256 const& _h = h256()) { return fromHeader(bytesConstRef(&_header), _s, _h); } + static BlockInfo fromHeader(bytesConstRef _header, Strictness _s = IgnoreNonce, h256 const& _h = h256()); explicit operator bool() const { return timestamp != Invalid256; } @@ -119,9 +118,11 @@ public: void setEmpty(); - void populateFromHeader(RLP const& _header, Strictness _s = IgnoreNonce); - void populate(bytesConstRef _block, Strictness _s = IgnoreNonce); - void populate(bytes const& _block, Strictness _s = IgnoreNonce) { populate(&_block, _s); } + void noteDirty() const { m_hash = m_seedHash= h256(); } + + void populateFromHeader(RLP const& _header, Strictness _s = IgnoreNonce, h256 const& _h = h256()); + void populate(bytesConstRef _block, Strictness _s = IgnoreNonce, h256 const& _h = h256()); + void populate(bytes const& _block, Strictness _s = IgnoreNonce, h256 const& _h = h256()) { populate(&_block, _s, _h); } void verifyInternals(bytesConstRef _block) const; void verifyParent(BlockInfo const& _parent) const; void populateFromParent(BlockInfo const& parent); @@ -129,6 +130,7 @@ public: u256 calculateDifficulty(BlockInfo const& _parent) const; u256 selectGasLimit(BlockInfo const& _parent) const; h256 const& seedHash() const; + h256 const& hash() const; /// sha3 of the header only. h256 headerHash(IncludeNonce _n) const; @@ -136,11 +138,12 @@ public: private: mutable h256 m_seedHash; + mutable h256 m_hash; ///< SHA3 hash of the block header! Not serialised. }; inline std::ostream& operator<<(std::ostream& _out, BlockInfo const& _bi) { - _out << _bi.hash << " " << _bi.parentHash << " " << _bi.sha3Uncles << " " << _bi.coinbaseAddress << " " << _bi.stateRoot << " " << _bi.transactionsRoot << " " << + _out << _bi.hash() << " " << _bi.parentHash << " " << _bi.sha3Uncles << " " << _bi.coinbaseAddress << " " << _bi.stateRoot << " " << _bi.transactionsRoot << " " << _bi.receiptsRoot << " " << _bi.logBloom << " " << _bi.difficulty << " " << _bi.number << " " << _bi.gasLimit << " " << _bi.gasUsed << " " << _bi.timestamp << " " << _bi.mixHash << " " << _bi.nonce << " (" << _bi.seedHash() << ")"; return _out; diff --git a/libethcore/Ethasher.cpp b/libethcore/Ethasher.cpp index e8e4d0b0e..75f0bcd5a 100644 --- a/libethcore/Ethasher.cpp +++ b/libethcore/Ethasher.cpp @@ -173,7 +173,10 @@ Ethasher::Result Ethasher::eval(BlockInfo const& _header, Nonce const& _nonce) { auto p = Ethasher::params(_header); ethash_return_value r; - ethash_compute_light(&r, Ethasher::get()->light(_header), &p, _header.headerHash(WithoutNonce).data(), (uint64_t)(u64)_nonce); + if (Ethasher::get()->m_fulls.count(_header.seedHash())) + ethash_compute_full(&r, Ethasher::get()->full(_header).data(), &p, _header.headerHash(WithoutNonce).data(), (uint64_t)(u64)_nonce); + else + ethash_compute_light(&r, Ethasher::get()->light(_header), &p, _header.headerHash(WithoutNonce).data(), (uint64_t)(u64)_nonce); // cdebug << "Ethasher::eval sha3(cache):" << sha3(Ethasher::get()->cache(_header)) << "hh:" << _header.headerHash(WithoutNonce) << "nonce:" << _nonce << " => " << h256(r.result, h256::ConstructFromPointer); return Result{h256(r.result, h256::ConstructFromPointer), h256(r.mix_hash, h256::ConstructFromPointer)}; } diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 0aea83de7..cad23171a 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include "GenesisInfo.h" #include "State.h" @@ -200,7 +201,7 @@ void BlockChain::rebuild(std::string const& _path, std::function(h256(u256(d)), m_blockHashes, x_blockHashes, NullBlockHash, oldExtrasDB).value); BlockInfo bi(b); + if (bi.number % c_ethashEpochLength == 1) + Ethasher::get()->full(bi); + if (bi.parentHash != lastHash) { - cwarn << "DISJOINT CHAIN DETECTED; " << bi.hash.abridged() << "#" << d << " -> parent is" << bi.parentHash.abridged() << "; expected" << lastHash.abridged() << "#" << (d - 1); + cwarn << "DISJOINT CHAIN DETECTED; " << bi.hash().abridged() << "#" << d << " -> parent is" << bi.parentHash.abridged() << "; expected" << lastHash.abridged() << "#" << (d - 1); return; } - lastHash = bi.hash; + lastHash = bi.hash(); import(b, s.db(), true); } catch (...) @@ -414,7 +418,7 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db, bool _force) { // Check transactions are valid and that they result in a state equivalent to our state_root. // Get total difficulty increase and update state, checking it. - State s(bi.coinbaseAddress, _db); + State s(_db); //, bi.coinbaseAddress auto tdIncrease = s.enactOn(&_block, bi, *this); BlockLogBlooms blb; @@ -493,7 +497,7 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db, bool _force) { clog(BlockChainWarn) << " Malformed block: " << diagnostic_information(_e); _e << errinfo_comment("Malformed block "); - clog(BlockChainWarn) << "Block: " << bi.hash; + clog(BlockChainWarn) << "Block: " << bi.hash(); clog(BlockChainWarn) << bi; clog(BlockChainWarn) << "Block parent: " << bi.parentHash; clog(BlockChainWarn) << BlockInfo(block(bi.parentHash)); @@ -560,7 +564,7 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db, bool _force) { RLP blockRLP(b); TransactionAddress ta; - ta.blockHash = bi.hash; + ta.blockHash = bi.hash(); WriteGuard l(x_transactionAddresses); for (ta.index = 0; ta.index < blockRLP[1].itemCount(); ++ta.index) { @@ -570,7 +574,7 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db, bool _force) } { WriteGuard l(x_blockHashes); - m_blockHashes[h256(bi.number)].value = bi.hash; + m_blockHashes[h256(bi.number)].value = bi.hash(); } // Update database with them. diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h index add4b43cd..2e2d8cf34 100644 --- a/libethereum/BlockChain.h +++ b/libethereum/BlockChain.h @@ -113,8 +113,8 @@ public: bool isKnown(h256 const& _hash) const; /// Get the familial details concerning a block (or the most recent mined if none given). Thread-safe. - BlockInfo info(h256 const& _hash) const { return BlockInfo(block(_hash)); } - BlockInfo info() const { return BlockInfo(block()); } + BlockInfo info(h256 const& _hash) const { return BlockInfo(block(_hash), IgnoreNonce, _hash); } + BlockInfo info() const { return info(currentHash()); } /// Get a block (RLP format) for the given hash (or the most recent mined if none given). Thread-safe. bytes block(h256 const& _hash) const; diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index 2f65e5f4b..95fd2364f 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -90,7 +90,7 @@ void BasicGasPricer::update(BlockChain const& _bc) { auto bb = _bc.block(p); RLP r(bb); - BlockReceipts brs(_bc.receipts(bi.hash)); + BlockReceipts brs(_bc.receipts(bi.hash())); for (unsigned i = 0; i < r[1].size(); ++i) { auto gu = brs.receipts[i].gasUsed(); @@ -123,8 +123,8 @@ Client::Client(p2p::Host* _extNet, std::string const& _dbPath, WithExisting _for m_bc(_dbPath, max(m_vc.action(), _forceAction), [](unsigned d, unsigned t){ cerr << "REVISING BLOCKCHAIN: Processed " << d << " of " << t << "...\r"; }), m_gp(new TrivialGasPricer), m_stateDB(State::openDB(_dbPath, max(m_vc.action(), _forceAction))), - m_preMine(Address(), m_stateDB), - m_postMine(Address(), m_stateDB) + m_preMine(m_stateDB), + m_postMine(m_stateDB) { m_gp->update(m_bc); @@ -148,8 +148,8 @@ Client::Client(p2p::Host* _extNet, std::shared_ptr _gp, std::string c m_bc(_dbPath, max(m_vc.action(), _forceAction), [](unsigned d, unsigned t){ cerr << "REVISING BLOCKCHAIN: Processed " << d << " of " << t << "...\r"; }), m_gp(_gp), m_stateDB(State::openDB(_dbPath, max(m_vc.action(), _forceAction))), - m_preMine(Address(), m_stateDB), - m_postMine(Address(), m_stateDB) + m_preMine(m_stateDB), + m_postMine(m_stateDB) { m_gp->update(m_bc); @@ -221,8 +221,8 @@ void Client::killChain() } m_bc.reopen(Defaults::dbPath(), WithExisting::Kill); - m_preMine = State(Address(), m_stateDB); - m_postMine = State(Address(), m_stateDB); + m_preMine = State(m_stateDB); + m_postMine = State(m_stateDB); if (auto h = m_host.lock()) h->reset(); @@ -313,7 +313,7 @@ void Client::appendFromNewBlock(h256 const& _block, h256Set& io_changed) auto m = i.second.filter.matches(tr); if (m.size()) { - auto transactionHash = transaction(d.hash, j).sha3(); + auto transactionHash = transaction(d.hash(), j).sha3(); // filter catches them for (LogEntry const& l: m) i.second.changes.push_back(LocalisedLogEntry(l, (unsigned)d.number, transactionHash)); diff --git a/libethereum/ClientBase.cpp b/libethereum/ClientBase.cpp index fae534ff8..b9c2fa878 100644 --- a/libethereum/ClientBase.cpp +++ b/libethereum/ClientBase.cpp @@ -182,7 +182,7 @@ LocalisedLogEntries ClientBase::logs(LogFilter const& _f) const if (_f.matches(receipt.bloom())) { auto info = bc().info(h); - auto th = transaction(info.hash, i).sha3(); + auto th = transaction(info.hash(), i).sha3(); LogEntries le = _f.matches(receipt); if (le.size()) { diff --git a/libethereum/State.cpp b/libethereum/State.cpp index 74d200381..a58ba9d0f 100644 --- a/libethereum/State.cpp +++ b/libethereum/State.cpp @@ -78,23 +78,24 @@ OverlayDB State::openDB(std::string _path, WithExisting _we) return OverlayDB(db); } -State::State(Address _coinbaseAddress, OverlayDB const& _db, BaseState _bs): +State::State(OverlayDB const& _db, BaseState _bs, Address _coinbaseAddress): m_db(_db), m_state(&m_db), m_ourAddress(_coinbaseAddress), m_blockReward(c_blockReward) { - // Initialise to the state entailed by the genesis block; this guarantees the trie is built correctly. - m_state.init(); + if (_bs != BaseState::PreExisting) + // Initialise to the state entailed by the genesis block; this guarantees the trie is built correctly. + m_state.init(); - paranoia("beginning of normal construction.", true); + paranoia("beginning of Genesis construction.", true); if (_bs == BaseState::CanonGenesis) { dev::eth::commit(genesisState(), m_db, m_state); m_db.commit(); - paranoia("after DB commit of normal construction.", true); + paranoia("after DB commit of Genesis construction.", true); m_previousBlock = CanonBlockChain::genesis(); } else @@ -321,7 +322,7 @@ bool State::sync(BlockChain const& _bc, h256 _block, BlockInfo const& _bi) std::vector chain; while (bi.number != 0 && m_db.lookup(bi.stateRoot).empty()) // while we don't have the state root of the latest block... { - chain.push_back(bi.hash); // push back for later replay. + chain.push_back(bi.hash()); // push back for later replay. bi.populate(_bc.block(bi.parentHash)); // move to parent. } @@ -537,18 +538,18 @@ u256 State::enact(bytesConstRef _block, BlockChain const& _bc, bool _checkNonce) { // m_currentBlock is assumed to be prepopulated and reset. -#if !ETH_RELEASE BlockInfo bi(_block, _checkNonce ? CheckEverything : IgnoreNonce); - assert(m_previousBlock.hash == bi.parentHash); +#if !ETH_RELEASE + assert(m_previousBlock.hash() == bi.parentHash); assert(m_currentBlock.parentHash == bi.parentHash); assert(rootHash() == m_previousBlock.stateRoot); #endif - if (m_currentBlock.parentHash != m_previousBlock.hash) + if (m_currentBlock.parentHash != m_previousBlock.hash()) BOOST_THROW_EXCEPTION(InvalidParentHash()); // Populate m_currentBlock with the correct values. - m_currentBlock.populate(_block, _checkNonce ? CheckEverything : IgnoreNonce); + m_currentBlock = bi; m_currentBlock.verifyInternals(_block); // cnote << "playback begins:" << m_state.root(); @@ -598,7 +599,7 @@ u256 State::enact(bytesConstRef _block, BlockChain const& _bc, bool _checkNonce) cwarn << TransactionReceipt(&b); } cwarn << "Recorded: " << m_currentBlock.receiptsRoot; - auto rs = _bc.receipts(m_currentBlock.hash); + auto rs = _bc.receipts(m_currentBlock.hash()); for (unsigned j = 0; j < rs.receipts.size(); ++j) { auto b = rs.receipts[j].rlp(); @@ -840,7 +841,7 @@ void State::commitToMine(BlockChain const& _bc) m_currentBlock.gasUsed = gasUsed(); m_currentBlock.stateRoot = m_state.root(); - m_currentBlock.parentHash = m_previousBlock.hash; + m_currentBlock.parentHash = m_previousBlock.hash(); } MineInfo State::mine(unsigned _msTimeout, bool _turbo) @@ -890,10 +891,10 @@ void State::completeMine() ret.appendRaw(m_currentTxs); ret.appendRaw(m_currentUncles); ret.swapOut(m_currentBytes); - m_currentBlock.hash = sha3(RLP(m_currentBytes)[0].data()); - cnote << "Mined " << m_currentBlock.hash.abridged() << "(parent: " << m_currentBlock.parentHash.abridged() << ")"; + m_currentBlock.noteDirty(); + cnote << "Mined " << m_currentBlock.hash().abridged() << "(parent: " << m_currentBlock.parentHash.abridged() << ")"; StructuredLogger::minedNewBlock( - m_currentBlock.hash.abridged(), + m_currentBlock.hash().abridged(), m_currentBlock.nonce.abridged(), "", //TODO: chain head hash here ?? m_currentBlock.parentHash.abridged() diff --git a/libethereum/State.h b/libethereum/State.h index e64a6f0e3..662426e17 100644 --- a/libethereum/State.h +++ b/libethereum/State.h @@ -54,7 +54,12 @@ struct StateTrace: public LogChannel { static const char* name() { return "=S="; struct StateDetail: public LogChannel { static const char* name() { return "/S/"; } static const int verbosity = 14; }; struct StateSafeExceptions: public LogChannel { static const char* name() { return "(S)"; } static const int verbosity = 21; }; -enum class BaseState { Empty, CanonGenesis }; +enum class BaseState +{ + PreExisting, + Empty, + CanonGenesis +}; enum class TransactionPriority { @@ -103,8 +108,15 @@ class State friend class Executive; public: - /// Construct state object. - State(Address _coinbaseAddress = Address(), OverlayDB const& _db = OverlayDB(), BaseState _bs = BaseState::CanonGenesis); + /// Default constructor; creates with a blank database prepopulated with the genesis block. + State(): State(OverlayDB(), BaseState::Empty) {} + + /// Basic state object from database. + /// Use the default when you already have a database and you just want to make a State object + /// which uses it. If you have no preexisting database then set BaseState to something other + /// than BaseState::PreExisting in order to prepopulate the Trie. + /// You can also set the coinbase address. + explicit State(OverlayDB const& _db, BaseState _bs = BaseState::PreExisting, Address _coinbaseAddress = Address()); /// Construct state object from arbitrary point in blockchain. State(OverlayDB const& _db, BlockChain const& _bc, h256 _hash); @@ -335,6 +347,7 @@ private: /// Debugging only. Good for checking the Trie is in shape. void paranoia(std::string const& _when, bool _enforceRefs = false) const; + OverlayDB m_db; ///< Our overlay for the state tree. SecureTrieDB m_state; ///< Our state tree, as an OverlayDB DB. Transactions m_transactions; ///< The current list of transactions that we've included in the state. diff --git a/libtestutils/StateLoader.cpp b/libtestutils/StateLoader.cpp index b56475b5a..464df0ec5 100644 --- a/libtestutils/StateLoader.cpp +++ b/libtestutils/StateLoader.cpp @@ -26,7 +26,7 @@ using namespace dev; using namespace dev::eth; using namespace dev::test; -StateLoader::StateLoader(Json::Value const& _json) : m_state(Address(), OverlayDB(), BaseState::Empty) +StateLoader::StateLoader(Json::Value const& _json) { for (string const& name: _json.getMemberNames()) { diff --git a/libweb3jsonrpc/WebThreeStubServerBase.cpp b/libweb3jsonrpc/WebThreeStubServerBase.cpp index 6f95bd529..fa273a0da 100644 --- a/libweb3jsonrpc/WebThreeStubServerBase.cpp +++ b/libweb3jsonrpc/WebThreeStubServerBase.cpp @@ -59,7 +59,7 @@ static Json::Value toJson(dev::eth::BlockInfo const& _bi) Json::Value res; if (_bi) { - res["hash"] = toJS(_bi.hash); + res["hash"] = toJS(_bi.hash()); res["parentHash"] = toJS(_bi.parentHash); res["sha3Uncles"] = toJS(_bi.sha3Uncles); res["miner"] = toJS(_bi.coinbaseAddress); diff --git a/mix/ClientModel.cpp b/mix/ClientModel.cpp index 767c2ac6f..08ef83e5e 100644 --- a/mix/ClientModel.cpp +++ b/mix/ClientModel.cpp @@ -469,7 +469,7 @@ RecordLogEntry* ClientModel::lastBlock() const strGas << blockInfo.gasUsed; std::stringstream strNumber; strNumber << blockInfo.number; - RecordLogEntry* record = new RecordLogEntry(0, QString::fromStdString(strNumber.str()), tr(" - Block - "), tr("Hash: ") + QString(QString::fromStdString(toHex(blockInfo.hash.ref()))), tr("Gas Used: ") + QString::fromStdString(strGas.str()), QString(), QString(), false, RecordLogEntry::RecordType::Block); + RecordLogEntry* record = new RecordLogEntry(0, QString::fromStdString(strNumber.str()), tr(" - Block - "), tr("Hash: ") + QString(QString::fromStdString(toHex(blockInfo.hash().ref()))), tr("Gas Used: ") + QString::fromStdString(strGas.str()), QString(), QString(), false, RecordLogEntry::RecordType::Block); QQmlEngine::setObjectOwnership(record, QQmlEngine::JavaScriptOwnership); return record; } diff --git a/mix/MixClient.cpp b/mix/MixClient.cpp index 7729c0ffe..cc200e62c 100644 --- a/mix/MixClient.cpp +++ b/mix/MixClient.cpp @@ -93,7 +93,7 @@ void MixClient::resetState(std::map _accounts) h256 stateRoot = accountState.root(); m_bc.reset(); m_bc.reset(new MixBlockChain(m_dbPath, stateRoot)); - m_state = eth::State(genesisState.begin()->first , m_stateDB, BaseState::Empty); + m_state = eth::State(m_stateDB, BaseState::PreExisting, genesisState.begin()->first); m_state.sync(bc()); m_startState = m_state; WriteGuard lx(x_executions); diff --git a/test/ClientBase.cpp b/test/ClientBase.cpp index 304182cfc..7597b6612 100644 --- a/test/ClientBase.cpp +++ b/test/ClientBase.cpp @@ -120,11 +120,15 @@ BOOST_AUTO_TEST_CASE(blocks) ETH_CHECK_EQUAL(expectedBlockInfoBloom, _blockInfo.logBloom); ETH_CHECK_EQUAL(expectedBlockInfoCoinbase, _blockInfo.coinbaseAddress); ETH_CHECK_EQUAL(expectedBlockInfoDifficulty, _blockInfo.difficulty); - ETH_CHECK_EQUAL_COLLECTIONS(expectedBlockInfoExtraData.begin(), expectedBlockInfoExtraData.end(), - _blockInfo.extraData.begin(), _blockInfo.extraData.end()); + ETH_CHECK_EQUAL_COLLECTIONS( + expectedBlockInfoExtraData.begin(), + expectedBlockInfoExtraData.end(), + _blockInfo.extraData.begin(), + _blockInfo.extraData.end() + ); ETH_CHECK_EQUAL(expectedBlockInfoGasLimit, _blockInfo.gasLimit); ETH_CHECK_EQUAL(expectedBlockInfoGasUsed, _blockInfo.gasUsed); - ETH_CHECK_EQUAL(expectedBlockInfoHash, _blockInfo.hash); + ETH_CHECK_EQUAL(expectedBlockInfoHash, _blockInfo.hash()); ETH_CHECK_EQUAL(expectedBlockInfoMixHash, _blockInfo.mixHash); ETH_CHECK_EQUAL(expectedBlockInfoNonce, _blockInfo.nonce); ETH_CHECK_EQUAL(expectedBlockInfoNumber, _blockInfo.number); @@ -155,8 +159,12 @@ BOOST_AUTO_TEST_CASE(blocks) u256 expectedTransactionSignatureS = h256(fromHex(_t["s"].asString())); // unsigned expectedTransactionSignatureV = jsToInt(t["v"].asString()); - ETH_CHECK_EQUAL_COLLECTIONS(expectedTransactionData.begin(), expectedTransactionData.end(), - _transaction.data().begin(), _transaction.data().end()); + ETH_CHECK_EQUAL_COLLECTIONS( + expectedTransactionData.begin(), + expectedTransactionData.end(), + _transaction.data().begin(), + _transaction.data().end() + ); ETH_CHECK_EQUAL(expectedTransactionGasLimit, _transaction.gas()); ETH_CHECK_EQUAL(expectedTransactionGasPrice, _transaction.gasPrice()); ETH_CHECK_EQUAL(expectedTransactionNonce, _transaction.nonce()); diff --git a/test/TestHelper.cpp b/test/TestHelper.cpp index 295b759f3..140efdb99 100644 --- a/test/TestHelper.cpp +++ b/test/TestHelper.cpp @@ -70,7 +70,10 @@ namespace test struct ValueTooLarge: virtual Exception {}; bigint const c_max256plus1 = bigint(1) << 256; -ImportTest::ImportTest(json_spirit::mObject& _o, bool isFiller) : m_statePre(Address(_o["env"].get_obj()["currentCoinbase"].get_str()), OverlayDB(), eth::BaseState::Empty), m_statePost(Address(_o["env"].get_obj()["currentCoinbase"].get_str()), OverlayDB(), eth::BaseState::Empty), m_TestObject(_o) +ImportTest::ImportTest(json_spirit::mObject& _o, bool isFiller): + m_statePre(OverlayDB(), eth::BaseState::Empty, Address(_o["env"].get_obj()["currentCoinbase"].get_str())), + m_statePost(OverlayDB(), eth::BaseState::Empty, Address(_o["env"].get_obj()["currentCoinbase"].get_str())), + m_TestObject(_o) { importEnv(_o["env"].get_obj()); importState(_o["pre"].get_obj(), m_statePre); @@ -92,7 +95,7 @@ void ImportTest::importEnv(json_spirit::mObject& _o) assert(_o.count("currentCoinbase") > 0); assert(_o.count("currentNumber") > 0); - m_environment.previousBlock.hash = h256(_o["previousHash"].get_str()); + m_environment.currentBlock.parentHash = h256(_o["parentHash"].get_str()); m_environment.currentBlock.number = toInt(_o["currentNumber"]); m_environment.currentBlock.gasLimit = toInt(_o["currentGasLimit"]); m_environment.currentBlock.difficulty = toInt(_o["currentDifficulty"]); diff --git a/test/TestHelper.h b/test/TestHelper.h index e5f96f51d..7f6d73365 100644 --- a/test/TestHelper.h +++ b/test/TestHelper.h @@ -101,7 +101,7 @@ namespace test class ImportTest { public: - ImportTest(json_spirit::mObject& _o) : m_statePre(Address(), OverlayDB(), eth::BaseState::Empty), m_statePost(Address(), OverlayDB(), eth::BaseState::Empty), m_TestObject(_o) {} + ImportTest(json_spirit::mObject& _o): m_TestObject(_o) {} ImportTest(json_spirit::mObject& _o, bool isFiller); // imports void importEnv(json_spirit::mObject& _o); diff --git a/test/blockchain.cpp b/test/blockchain.cpp index 21345abfd..50c17bdee 100644 --- a/test/blockchain.cpp +++ b/test/blockchain.cpp @@ -52,7 +52,7 @@ void doBlockchainTests(json_spirit::mValue& _v, bool _fillin) BOOST_REQUIRE(o.count("pre")); ImportTest importer(o["pre"].get_obj()); - State state(biGenesisBlock.coinbaseAddress, OverlayDB(), BaseState::Empty); + State state(OverlayDB(), BaseState::Empty, biGenesisBlock.coinbaseAddress); importer.importState(o["pre"].get_obj(), state); o["pre"] = fillJsonWithState(state); state.commit(); @@ -599,7 +599,7 @@ void updatePoW(BlockInfo& _bi) ret = pow.mine(_bi, 10000, true, true); Ethash::assignResult(ret.second, _bi); } - _bi.hash = _bi.headerHash(WithNonce); + _bi.noteDirty(); } void writeBlockHeaderToJson(mObject& _o, BlockInfo const& _bi) @@ -619,7 +619,7 @@ void writeBlockHeaderToJson(mObject& _o, BlockInfo const& _bi) _o["extraData"] ="0x" + toHex(_bi.extraData); _o["mixHash"] = toString(_bi.mixHash); _o["nonce"] = toString(_bi.nonce); - _o["hash"] = toString(_bi.hash); + _o["hash"] = toString(_bi.hash()); } RLPStream createFullBlockFromHeader(BlockInfo const& _bi, bytes const& _txs, bytes const& _uncles) diff --git a/test/stateOriginal.cpp b/test/stateOriginal.cpp index 384d85344..572e84dcf 100644 --- a/test/stateOriginal.cpp +++ b/test/stateOriginal.cpp @@ -58,7 +58,7 @@ BOOST_AUTO_TEST_CASE(Complex) CanonBlockChain bc; cout << bc; - State s(myMiner.address(), stateDB); + State s(stateDB, BaseState::Empty, myMiner.address()); cout << s; // Sync up - this won't do much until we use the last state. diff --git a/test/vm.cpp b/test/vm.cpp index cffbaa64d..ff8903523 100644 --- a/test/vm.cpp +++ b/test/vm.cpp @@ -96,7 +96,7 @@ void FakeExtVM::push(mArray& a, u256 _v) mObject FakeExtVM::exportEnv() { mObject ret; - ret["previousHash"] = toString(previousBlock.hash); + ret["previousHash"] = toString(currentBlock.parentHash); push(ret, "currentDifficulty", currentBlock.difficulty); push(ret, "currentTimestamp", currentBlock.timestamp); ret["currentCoinbase"] = toString(currentBlock.coinbaseAddress); @@ -115,7 +115,7 @@ void FakeExtVM::importEnv(mObject& _o) assert(_o.count("currentCoinbase") > 0); assert(_o.count("currentNumber") > 0); - previousBlock.hash = h256(_o["previousHash"].get_str()); + currentBlock.parentHash = h256(_o["previousHash"].get_str()); currentBlock.number = toInt(_o["currentNumber"]); lastHashes = test::lastHashes(currentBlock.number); currentBlock.gasLimit = toInt(_o["currentGasLimit"]); From 55df43ff4ba3f13c10c342b48ed07b22ecc70ff3 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 5 Apr 2015 22:09:02 +0200 Subject: [PATCH 09/10] Fast databases. --- libethcore/Common.cpp | 2 +- libethereum/BlockChain.cpp | 54 ++++++++++++++++++++++++++++++++++-- libethereum/BlockChain.h | 26 +++++++++++++++++ libethereum/Client.cpp | 2 +- libethereum/EthereumPeer.cpp | 7 ++++- 5 files changed, 85 insertions(+), 6 deletions(-) diff --git a/libethcore/Common.cpp b/libethcore/Common.cpp index 02e806905..572ade3e2 100644 --- a/libethcore/Common.cpp +++ b/libethcore/Common.cpp @@ -36,7 +36,7 @@ namespace eth const unsigned c_ethashVersion = c_ethashRevision; const unsigned c_protocolVersion = 60; const unsigned c_minorProtocolVersion = 0; -const unsigned c_databaseBaseVersion = 8; +const unsigned c_databaseBaseVersion = 9; #if ETH_FATDB const unsigned c_databaseVersionModifier = 1; #else diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index cad23171a..74aeb4fad 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -21,6 +21,9 @@ #include "BlockChain.h" +#if ETH_PROFILING_GPERF +#include +#endif #include #include #include @@ -65,7 +68,7 @@ std::ostream& dev::eth::operator<<(std::ostream& _out, BlockChain const& _bc) return _out; } -ldb::Slice dev::eth::toSlice(h256 const& _h, unsigned _sub) +ldb::Slice dev::eth::oldToSlice(h256 const& _h, unsigned _sub) { #if ALL_COMPILERS_ARE_CPP11_COMPLIANT static thread_local h256 h = _h ^ sha3(h256(u256(_sub))); @@ -79,6 +82,21 @@ ldb::Slice dev::eth::toSlice(h256 const& _h, unsigned _sub) #endif } +ldb::Slice dev::eth::toSlice(h256 const& _h, unsigned _sub) +{ +#if ALL_COMPILERS_ARE_CPP11_COMPLIANT + static thread_local h256 h = _h ^ sha3(h256(u256(_sub))); + return ldb::Slice((char const*)&h, 32); +#else + static boost::thread_specific_ptr> t_h; + if (!t_h.get()) + t_h.reset(new FixedHash<33>); + *t_h = FixedHash<33>(_h); + (*t_h)[32] = (uint8_t)_sub; + return (ldb::Slice)t_h->ref();//(char const*)t_h.get(), 32); +#endif +} + #if ETH_DEBUG static const chrono::system_clock::duration c_collectionDuration = chrono::seconds(15); static const unsigned c_collectionQueueSize = 2; @@ -148,7 +166,7 @@ void BlockChain::open(std::string const& _path, WithExisting _we) } } - if (!details(m_genesisHash)) + if (_we != WithExisting::Verify && !details(m_genesisHash)) { // Insert details of genesis block. m_details[m_genesisHash] = BlockDetails(0, c_genesisDifficulty, h256(), {}); @@ -178,7 +196,6 @@ void BlockChain::close() m_blocks.clear(); } -#include #define IGNORE_EXCEPTIONS(X) try { X; } catch (...) {} void BlockChain::rebuild(std::string const& _path, std::function const& _progress) @@ -187,6 +204,7 @@ void BlockChain::rebuild(std::string const& _path, std::functionsecond; + } + + string d; + m_blocksDB->Get(m_readOptions, oldToSlice(_hash), &d); + + if (!d.size()) + { + cwarn << "Couldn't find requested block:" << _hash.abridged(); + return bytes(); + } + + WriteGuard l(x_blocks); + m_blocks[_hash].resize(d.size()); + memcpy(m_blocks[_hash].data(), d.data(), d.size()); + + noteUsed(_hash); + + return m_blocks[_hash]; +} diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h index 2e2d8cf34..83b1926e8 100644 --- a/libethereum/BlockChain.h +++ b/libethereum/BlockChain.h @@ -64,6 +64,7 @@ struct BlockChainWarn: public LogChannel { static const char* name() { return "= std::map const& genesisState(); ldb::Slice toSlice(h256 const& _h, unsigned _sub = 0); +ldb::Slice oldToSlice(h256 const& _h, unsigned _sub = 0); using BlocksHash = std::map; using TransactionHashes = h256s; @@ -119,6 +120,7 @@ public: /// Get a block (RLP format) for the given hash (or the most recent mined if none given). Thread-safe. bytes block(h256 const& _hash) const; bytes block() const { return block(currentHash()); } + bytes oldBlock(h256 const& _hash) const; /// Get the familial details concerning a block (or the most recent mined if none given). Thread-safe. BlockDetails details(h256 const& _hash) const { return queryExtras(_hash, m_details, x_details, NullBlockDetails); } @@ -268,6 +270,30 @@ private: return ret.first->second; } + template T oldQueryExtras(h256 const& _h, std::map& _m, boost::shared_mutex& _x, T const& _n, ldb::DB* _extrasDB = nullptr) const + { + { + ReadGuard l(_x); + auto it = _m.find(_h); + if (it != _m.end()) + return it->second; + } + + std::string s; + (_extrasDB ? _extrasDB : m_extrasDB)->Get(m_readOptions, oldToSlice(_h, N), &s); + if (s.empty()) + { +// cout << "Not found in DB: " << _h << endl; + return _n; + } + + noteUsed(_h, N); + + WriteGuard l(_x); + auto ret = _m.insert(std::make_pair(_h, T(RLP(s)))); + return ret.first->second; + } + void checkConsistency(); /// The caches of the disk DB and their locks. diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index 95fd2364f..71a9a4dbe 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -123,7 +123,7 @@ Client::Client(p2p::Host* _extNet, std::string const& _dbPath, WithExisting _for m_bc(_dbPath, max(m_vc.action(), _forceAction), [](unsigned d, unsigned t){ cerr << "REVISING BLOCKCHAIN: Processed " << d << " of " << t << "...\r"; }), m_gp(new TrivialGasPricer), m_stateDB(State::openDB(_dbPath, max(m_vc.action(), _forceAction))), - m_preMine(m_stateDB), + m_preMine(m_stateDB, BaseState::CanonGenesis), m_postMine(m_stateDB) { m_gp->update(m_bc); diff --git a/libethereum/EthereumPeer.cpp b/libethereum/EthereumPeer.cpp index 15ce5c3f6..ca0195efe 100644 --- a/libethereum/EthereumPeer.cpp +++ b/libethereum/EthereumPeer.cpp @@ -478,7 +478,12 @@ bool EthereumPeer::interpret(unsigned _id, RLP const& _r) clogS(NetMessageSummary) << dec << success << "imported OK," << unknown << "with unknown parents," << future << "with future timestamps," << got << " already known," << repeated << " repeats received."; if (m_asking == Asking::Blocks) - transition(Asking::Blocks); + { + if (!got) + transition(Asking::Blocks); + else + transition(Asking::Nothing); + } break; } case NewBlockPacket: From 747927553c331beaf5a63ef857395bd624a0d0f6 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 5 Apr 2015 22:13:24 +0200 Subject: [PATCH 10/10] Typo "fix" fixed. --- test/TestHelper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/TestHelper.cpp b/test/TestHelper.cpp index 140efdb99..dd7c09eab 100644 --- a/test/TestHelper.cpp +++ b/test/TestHelper.cpp @@ -95,7 +95,7 @@ void ImportTest::importEnv(json_spirit::mObject& _o) assert(_o.count("currentCoinbase") > 0); assert(_o.count("currentNumber") > 0); - m_environment.currentBlock.parentHash = h256(_o["parentHash"].get_str()); + m_environment.currentBlock.parentHash = h256(_o["previousHash"].get_str()); m_environment.currentBlock.number = toInt(_o["currentNumber"]); m_environment.currentBlock.gasLimit = toInt(_o["currentGasLimit"]); m_environment.currentBlock.difficulty = toInt(_o["currentDifficulty"]);