From 346db17ba3e41e5489794f3d0c3aa021f8275571 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 24 Jun 2015 15:25:06 +0200 Subject: [PATCH] Parallelised large-file imports. --- eth/main.cpp | 34 ++++++++++++++++++++++++++++++++-- libethcore/Common.h | 6 ++++++ libethereum/BlockChain.cpp | 1 + libethereum/BlockChain.h | 6 ------ libethereum/Client.cpp | 12 ++++++++++++ libethereum/Client.h | 8 ++++++++ libethereum/State.h | 2 ++ 7 files changed, 61 insertions(+), 8 deletions(-) diff --git a/eth/main.cpp b/eth/main.cpp index c5a15e5fd..80dfc41af 100644 --- a/eth/main.cpp +++ b/eth/main.cpp @@ -187,6 +187,7 @@ void help() << " --from Export only from block n; n may be a decimal, a '0x' prefixed hash, or 'latest'." << endl << " --to Export only to block n (inclusive); n may be a decimal, a '0x' prefixed hash, or 'latest'." << endl << " --only Equivalent to --export-from n --export-to n." << endl + << " --dont-check Avoids checking some of the aspects of blocks. Faster importing, but only do if you know the data is valid." << endl << endl << "General Options:" << endl << " -d,--db-path Load database from path (default: " << getDataDir() << ")" << endl @@ -283,6 +284,7 @@ int main(int argc, char** argv) /// File name for import/export. string filename; + bool safeImport = false; /// Hashes/numbers for export range. string exportFrom = "1"; @@ -395,6 +397,8 @@ int main(int argc, char** argv) mode = OperationMode::Import; filename = argv[++i]; } + else if (arg == "--dont-check") + safeImport = true; else if ((arg == "-E" || arg == "--export") && i + 1 < argc) { mode = OperationMode::Export; @@ -753,13 +757,18 @@ int main(int argc, char** argv) unsigned futureTime = 0; unsigned unknownParent = 0; unsigned bad = 0; + chrono::steady_clock::time_point t = chrono::steady_clock::now(); + double last = 0; + unsigned lastImported = 0; + unsigned imported = 0; while (in.peek() != -1) { bytes block(8); in.read((char*)block.data(), 8); block.resize(RLP(block, RLP::LaisezFaire).actualSize()); in.read((char*)block.data() + 8, block.size() - 8); - switch (web3.ethereum()->injectBlock(block)) + + switch (web3.ethereum()->queueBlock(block, safeImport)) { case ImportResult::Success: good++; break; case ImportResult::AlreadyKnown: alreadyHave++; break; @@ -768,8 +777,29 @@ int main(int argc, char** argv) case ImportResult::FutureTimeKnown: futureTime++; break; default: bad++; break; } + + // sync chain with queue + tuple r = web3.ethereum()->syncQueue(10); + imported += get<2>(r); + + double e = chrono::duration_cast(chrono::steady_clock::now() - t).count() / 1000.0; + if ((unsigned)e >= last + 10) + { + auto i = imported - lastImported; + auto d = e - last; + cout << i << " more imported at " << (round(i * 10 / d) / 10) << " blocks/s. " << imported << " imported in " << e << " seconds at " << (round(imported * 10 / e) / 10) << " blocks/s (#" << web3.ethereum()->number() << ")" << endl; + last = (unsigned)e; + lastImported = imported; + } + } + + while (web3.ethereum()->blockQueue().items().first + web3.ethereum()->blockQueue().items().second > 0) + { + sleep(1); + web3.ethereum()->syncQueue(100000); } - cout << (good + bad + futureTime + unknownParent + alreadyHave) << " total: " << good << " ok, " << alreadyHave << " got, " << futureTime << " future, " << unknownParent << " unknown parent, " << bad << " malformed." << endl; + double e = chrono::duration_cast(chrono::steady_clock::now() - t).count() / 1000.0; + cout << imported << " imported in " << e << " seconds at " << (round(imported * 10 / e) / 10) << " blocks/s (#" << web3.ethereum()->number() << ")" << endl; return 0; } diff --git a/libethcore/Common.h b/libethcore/Common.h index 19ca600b9..25a6a8e1d 100644 --- a/libethcore/Common.h +++ b/libethcore/Common.h @@ -97,6 +97,12 @@ enum class RelativeBlock: BlockNumber Pending = PendingBlock }; +struct ImportRoute +{ + h256s deadBlocks; + h256s liveBlocks; +}; + enum class ImportResult { Success = 0, diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 640fd2df4..5f82d02e3 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -1116,6 +1116,7 @@ VerifiedBlockRef BlockChain::verifyBlock(bytes const& _block, function; using TransactionHashes = h256s; using UncleHashes = h256s; -struct ImportRoute -{ - h256s deadBlocks; - h256s liveBlocks; -}; - enum { ExtraDetails = 0, ExtraBlockHash, diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index f66ef8c3b..b7a870b0d 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -87,6 +87,18 @@ void VersionChecker::setOk() } } +ImportResult Client::queueBlock(bytes const& _block, bool _isSafe) +{ + if (m_bq.status().verified + m_bq.status().verifying + m_bq.status().unverified > 30000) + sleep(1); + return m_bq.import(&_block, bc(), _isSafe); +} + +tuple Client::syncQueue(unsigned _max) +{ + return m_bc.sync(m_bq, m_stateDB, _max); +} + void Client::onBadBlock(Exception& _ex) const { // BAD BLOCK!!! diff --git a/libethereum/Client.h b/libethereum/Client.h index 2b168f241..f38c7c099 100644 --- a/libethereum/Client.h +++ b/libethereum/Client.h @@ -138,6 +138,9 @@ public: /// Blocks until all pending transactions have been processed. virtual void flushTransactions() override; + /// Queues a block for import. + ImportResult queueBlock(bytes const& _block, bool _isSafe = false); + using Interface::call; // to remove warning about hiding virtual function /// Makes the given call. Nothing is recorded into the state. This cheats by creating a null address and endowing it with a lot of ETH. ExecutionResult call(Address _dest, bytes const& _data = bytes(), u256 _gas = 125000, u256 _value = 0, u256 _gasPrice = 1 * ether, Address const& _from = Address()); @@ -160,6 +163,11 @@ public: SyncStatus syncStatus() const; /// Get the block queue. BlockQueue const& blockQueue() const { return m_bq; } + /// Get the block queue. + OverlayDB const& stateDB() const { return m_stateDB; } + + /// Freeze worker thread and sync some of the block queue. + std::tuple syncQueue(unsigned _max = 1); // Mining stuff: diff --git a/libethereum/State.h b/libethereum/State.h index 93a4f4ade..6ad8fed09 100644 --- a/libethereum/State.h +++ b/libethereum/State.h @@ -58,6 +58,8 @@ using errinfo_transactionIndex = boost::error_info; using errinfo_receipts = boost::error_info>; +using errinfo_transaction = boost::error_info; +using errinfo_phase = boost::error_info; using errinfo_required_LogBloom = boost::error_info; using errinfo_got_LogBloom = boost::error_info; using LogBloomRequirementError = boost::tuple;