From 51f575e049d8a472c11047f4341f1b1d5e66906a Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 9 Apr 2015 22:36:16 +0200 Subject: [PATCH] Blockchain import/export. One or two minor additons to eth. --- eth/main.cpp | 134 +++++++++++++++++++++++++++++++------ libdevcore/RLP.cpp | 2 +- libdevcore/RLP.h | 5 +- libdevcrypto/Common.cpp | 6 +- libethereum/BlockChain.cpp | 8 +-- libethereum/BlockChain.h | 10 ++- libethereum/Client.h | 3 +- libethereum/ClientBase.cpp | 5 ++ libethereum/ClientBase.h | 4 +- libtestutils/FixedClient.h | 1 + mix/MixClient.h | 7 +- 11 files changed, 147 insertions(+), 38 deletions(-) diff --git a/eth/main.cpp b/eth/main.cpp index 1f5b6d574..4ced0272c 100644 --- a/eth/main.cpp +++ b/eth/main.cpp @@ -115,9 +115,14 @@ void help() << " /Etherum or Library/Application Support/Ethereum)." << endl << " -D,--create-dag Create the DAG in preparation for mining on given block and exit." << endl << " -e,--ether-price Set the ether price in the reference unit e.g. ยข (Default: 30.679)." << endl + << " -E,--export Export file as a concatenated series of blocks and exit." << endl + << " --export-from Export only from block n; n may be a decimal, a '0x' prefixed hash, or 'latest'." << endl + << " --export-to Export only to block n (inclusive); n may be a decimal, a '0x' prefixed hash, or 'latest'." << endl + << " --export-only Equivalent to --export-from n --export-to n." << endl << " -f,--force-mining Mine even when there are no transaction to mine (Default: off)" << endl << " -h,--help Show this help message and exit." << endl << " -i,--interactive Enter interactive mode (default: non-interactive)." << endl + << " -I,--import Import file as a concatenated series of blocks and exit." << endl #if ETH_JSONRPC << " -j,--json-rpc Enable JSON-RPC server (default: off)." << endl << " --json-rpc-port Specify JSON-RPC server port (implies '-j', default: " << SensibleHttpPort << ")." << endl @@ -209,33 +214,61 @@ void doInitDAG(unsigned _n) exit(0); } -static const unsigned NoDAGInit = (unsigned)-3; +enum class OperationMode +{ + Node, + Import, + Export, + DAGInit +}; int main(int argc, char** argv) { - unsigned initDAG = NoDAGInit; - string listenIP; - unsigned short listenPort = 30303; - string publicIP; - string remoteHost; - unsigned short remotePort = 30303; + /// Operating mode. + OperationMode mode = OperationMode::Node; string dbPath; - unsigned mining = ~(unsigned)0; - NodeMode mode = NodeMode::Full; - unsigned peers = 5; - int miners = -1; + + /// File name for import/export. + string filename; + + /// Hashes/numbers for export range. + string exportFrom = "1"; + string exportTo = "latest"; + + /// DAG initialisation param. + unsigned initDAG; + + /// General params for Node operation + NodeMode nodeMode = NodeMode::Full; bool interactive = false; #if ETH_JSONRPC int jsonrpc = -1; #endif - bool bootstrap = false; bool upnp = true; - bool forceMining = false; WithExisting killChain = WithExisting::Trust; bool jit = false; + + /// Networking params. + string clientName; + string listenIP; + unsigned short listenPort = 30303; + string publicIP; + string remoteHost; + unsigned short remotePort = 30303; + unsigned peers = 5; + bool bootstrap = false; + + /// Mining params + unsigned mining = ~(unsigned)0; + int miners = -1; + bool forceMining = false; + bool turboMining = false; + + /// Structured logging params bool structuredLogging = false; string structuredLoggingFormat = "%Y-%m-%dT%H:%M:%S"; - string clientName; + + /// Transaction params TransactionPriority priority = TransactionPriority::Medium; double etherPrice = 30.679; double blockFees = 15.0; @@ -275,6 +308,16 @@ int main(int argc, char** argv) remoteHost = argv[++i]; else if ((arg == "-p" || arg == "--port") && i + 1 < argc) remotePort = (short)atoi(argv[++i]); + else if ((arg == "-I" || arg == "--import") && i + 1 < argc) + { + mode = OperationMode::Import; + filename = argv[++i]; + } + else if ((arg == "-E" || arg == "--export") && i + 1 < argc) + { + mode = OperationMode::Export; + filename = argv[++i]; + } else if ((arg == "-n" || arg == "--upnp") && i + 1 < argc) { string m = argv[++i]; @@ -321,6 +364,7 @@ int main(int argc, char** argv) else if ((arg == "-D" || arg == "--create-dag") && i + 1 < argc) { string m = boost::to_lower_copy(string(argv[++i])); + mode = OperationMode::DAGInit; if (m == "next") initDAG = PendingBlock; else if (m == "this") @@ -402,6 +446,8 @@ int main(int argc, char** argv) bootstrap = true; else if (arg == "-f" || arg == "--force-mining") forceMining = true; + else if (arg == "-T" || arg == "--turbo-mining") + turboMining = true; else if (arg == "-i" || arg == "--interactive") interactive = true; #if ETH_JSONRPC @@ -420,9 +466,9 @@ int main(int argc, char** argv) { string m = argv[++i]; if (m == "full") - mode = NodeMode::Full; + nodeMode = NodeMode::Full; else if (m == "peer") - mode = NodeMode::PeerServer; + nodeMode = NodeMode::PeerServer; else { cerr << "Unknown mode: " << m << endl; @@ -452,7 +498,7 @@ int main(int argc, char** argv) // Two codepaths is necessary since named block require database, but numbered // blocks are superuseful to have when database is already open in another process. - if (initDAG < NoDAGInit) + if (mode == OperationMode::DAGInit && !(initDAG == LatestBlock || initDAG == PendingBlock)) doInitDAG(initDAG); if (!clientName.empty()) @@ -469,23 +515,69 @@ int main(int argc, char** argv) clientImplString, dbPath, killChain, - mode == NodeMode::Full ? set{"eth", "shh"} : set(), + nodeMode == NodeMode::Full ? set{"eth", "shh"} : set(), netPrefs, &nodesState, miners ); - if (initDAG == LatestBlock || initDAG == PendingBlock) + if (mode == OperationMode::DAGInit) doInitDAG(web3.ethereum()->blockChain().number() + (initDAG == PendingBlock ? 30000 : 0)); - + + auto toNumber = [&](string const& s) -> unsigned { + if (s == "latest") + return web3.ethereum()->number(); + if (s.size() == 64 || (s.size() == 66 && s.substr(0, 2) == "0x")) + return web3.ethereum()->blockChain().number(h256(s)); + try { + return stol(s); + } + catch (...) + { + cerr << "Bad block number/hash option: " << s << endl; + exit(-1); + } + }; + + if (mode == OperationMode::Export) + { + ofstream fout; + fout.open(filename); + unsigned last = toNumber(exportTo); + for (unsigned i = toNumber(exportFrom); i <= last; ++i) + { + bytes block = web3.ethereum()->blockChain().block(web3.ethereum()->blockChain().numberHash(i)); + fout.write((char const*)block.data(), block.size()); + } + return 0; + } + + if (mode == OperationMode::Import) + { + ifstream fin; + fin.open(filename); + + while (fin) + { + bytes block(8); + fin.read((char*)block.data(), 8); + unsigned remaining = RLP(block, RLP::LaisezFaire).actualSize() - 8; + block.resize(remaining); + fin.read((char*)block.data() + 8, remaining); + web3.ethereum()->injectBlock(block); + } + return 0; + } + web3.setIdealPeerCount(peers); std::shared_ptr gasPricer = make_shared(u256(double(ether / 1000) / etherPrice), u256(blockFees * 1000)); - eth::Client* c = mode == NodeMode::Full ? web3.ethereum() : nullptr; + eth::Client* c = nodeMode == NodeMode::Full ? web3.ethereum() : nullptr; StructuredLogger::starting(clientImplString, dev::Version); if (c) { c->setGasPricer(gasPricer); c->setForceMining(forceMining); + c->setTurboMining(turboMining); c->setAddress(coinbase); } diff --git a/libdevcore/RLP.cpp b/libdevcore/RLP.cpp index 26f10ecf5..994aac265 100644 --- a/libdevcore/RLP.cpp +++ b/libdevcore/RLP.cpp @@ -107,7 +107,7 @@ unsigned RLP::actualSize() const if (isSingleByte()) return 1; if (isData() || isList()) - return payload().data() - m_data.data() + length(); + return payloadOffset() + length(); return 0; } diff --git a/libdevcore/RLP.h b/libdevcore/RLP.h index a837f9221..caaf10b6a 100644 --- a/libdevcore/RLP.h +++ b/libdevcore/RLP.h @@ -290,7 +290,7 @@ public: RLPs toList() const; /// @returns the data payload. Valid for all types. - bytesConstRef payload() const { return isSingleByte() ? m_data.cropped(0, 1) : m_data.cropped(1 + lengthSize()); } + bytesConstRef payload() const { return m_data.cropped(payloadOffset()); } /// @returns the theoretical size of this item. /// @note Under normal circumstances, is equivalent to m_data.size() - use that unless you know it won't work. @@ -309,6 +309,9 @@ private: /// @returns the size in bytes of the payload, as given by the RLP as opposed to as inferred from m_data. unsigned length() const; + /// @returns the number of bytes into the data that the payload starts. + unsigned payloadOffset() const { return isSingleByte() ? 0 : (1 + lengthSize()); } + /// @returns the number of data items. unsigned items() const; diff --git a/libdevcrypto/Common.cpp b/libdevcrypto/Common.cpp index e108b230f..f9720fb8d 100644 --- a/libdevcrypto/Common.cpp +++ b/libdevcrypto/Common.cpp @@ -37,9 +37,9 @@ static Secp256k1 s_secp256k1; bool dev::SignatureStruct::isValid() const { - if (this->v > 1 || - this->r >= h256("0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141") || - this->s >= h256("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2f")) + if (v > 1 || + r >= h256("0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141") || + s >= h256("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2f")) return false; return true; } diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index f64d6557c..cb2d26eff 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -240,7 +240,7 @@ void BlockChain::rebuild(std::string const& _path, std::function BlockChain::sync(BlockQueue& _bq, OverlayDB const& _st return make_tuple(fresh, dead, _bq.doneDrain(badBlocks)); } -pair BlockChain::attemptImport(bytes const& _block, OverlayDB const& _stateDB, bool _force) noexcept +pair BlockChain::attemptImport(bytes const& _block, OverlayDB const& _stateDB, Aversion _force) noexcept { try { @@ -347,7 +347,7 @@ pair BlockChain::attemptImport(bytes const& _block, OverlayDB const } } -pair BlockChain::import(bytes const& _block, OverlayDB const& _db, bool _force) +pair BlockChain::import(bytes const& _block, OverlayDB const& _db, Aversion _force) { //@tidy This is a behemoth of a method - could do to be split into a few smaller ones. @@ -386,7 +386,7 @@ pair BlockChain::import(bytes const& _block, OverlayDB const& _db, #endif // Check block doesn't already exist first! - if (isKnown(bi.hash()) && !_force) + if (isKnown(bi.hash()) && _force == Aversion::AvoidOldBlocks) { clog(BlockChainNote) << bi.hash() << ": Not new."; BOOST_THROW_EXCEPTION(AlreadyHaveBlock()); diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h index b4e18a037..765e00b03 100644 --- a/libethereum/BlockChain.h +++ b/libethereum/BlockChain.h @@ -80,6 +80,12 @@ enum { using ProgressCallback = std::function; +enum class Aversion +{ + AvoidOldBlocks, + ImportOldBlocks +}; + /** * @brief Implements the blockchain database. All data this gives is disk-backed. * @threadsafe @@ -102,11 +108,11 @@ public: /// Attempt to import the given block directly into the CanonBlockChain and sync with the state DB. /// @returns the block hashes of any blocks that came into/went out of the canonical block chain. - std::pair attemptImport(bytes const& _block, OverlayDB const& _stateDB, bool _force = false) noexcept; + std::pair attemptImport(bytes const& _block, OverlayDB const& _stateDB, Aversion _force = Aversion::AvoidOldBlocks) noexcept; /// Import block into disk-backed DB /// @returns the block hashes of any blocks that came into/went out of the canonical block chain. - std::pair import(bytes const& _block, OverlayDB const& _stateDB, bool _force = false); + std::pair import(bytes const& _block, OverlayDB const& _stateDB, Aversion _force = Aversion::AvoidOldBlocks); /// Returns true if the given block is known (though not necessarily a part of the canon chain). bool isKnown(h256 const& _hash) const; diff --git a/libethereum/Client.h b/libethereum/Client.h index 25a087de0..57fe0d9de 100644 --- a/libethereum/Client.h +++ b/libethereum/Client.h @@ -229,8 +229,9 @@ public: protected: /// InterfaceStub methods + virtual BlockChain& bc() override { return m_bc; } virtual BlockChain const& bc() const override { return m_bc; } - + /// Returns the state object for the full block (i.e. the terminal state) for index _h. /// Works properly with LatestBlock and PendingBlock. using ClientBase::asOf; diff --git a/libethereum/ClientBase.cpp b/libethereum/ClientBase.cpp index 1d27bcfce..f57b2b174 100644 --- a/libethereum/ClientBase.cpp +++ b/libethereum/ClientBase.cpp @@ -107,6 +107,11 @@ ExecutionResult ClientBase::create(Secret _secret, u256 _value, bytes const& _da return ret; } +void ClientBase::injectBlock(bytes const& _block) +{ + bc().import(_block, preMine().db()); +} + u256 ClientBase::balanceAt(Address _a, BlockNumber _block) const { return asOf(_block).balance(_a); diff --git a/libethereum/ClientBase.h b/libethereum/ClientBase.h index d06904b5d..00bb02ed4 100644 --- a/libethereum/ClientBase.h +++ b/libethereum/ClientBase.h @@ -109,7 +109,6 @@ public: virtual LocalisedLogEntries peekWatch(unsigned _watchId) const override; virtual LocalisedLogEntries checkWatch(unsigned _watchId) override; - // TODO: switch all the _blockHash arguments to also accept _blockNumber virtual h256 hashFromNumber(BlockNumber _number) const override; virtual eth::BlockInfo blockInfo(h256 _hash) const override; virtual eth::BlockDetails blockDetails(h256 _hash) const override; @@ -125,6 +124,8 @@ public: virtual eth::Transactions pending() const override; virtual h256s pendingHashes() const override; + void injectBlock(bytes const& _block); + using Interface::diff; virtual StateDiff diff(unsigned _txi, h256 _block) const override; virtual StateDiff diff(unsigned _txi, BlockNumber _block) const override; @@ -155,6 +156,7 @@ public: protected: /// The interface that must be implemented in any class deriving this. /// { + virtual BlockChain& bc() = 0; virtual BlockChain const& bc() const = 0; virtual State asOf(h256 const& _h) const = 0; virtual State preMine() const = 0; diff --git a/libtestutils/FixedClient.h b/libtestutils/FixedClient.h index 95acc3edb..59da9075f 100644 --- a/libtestutils/FixedClient.h +++ b/libtestutils/FixedClient.h @@ -42,6 +42,7 @@ public: // stub virtual void flushTransactions() override {} + virtual eth::BlockChain& bc() override { BOOST_THROW_EXCEPTION(InterfaceNotSupported("FixedClient::bc()")); } virtual eth::BlockChain const& bc() const override { return m_bc; } using ClientBase::asOf; virtual eth::State asOf(h256 const& _h) const override; diff --git a/mix/MixClient.h b/mix/MixClient.h index 480509545..39d5bf081 100644 --- a/mix/MixClient.h +++ b/mix/MixClient.h @@ -79,11 +79,10 @@ public: std::vector userAccounts() { return m_userAccounts; } protected: - virtual dev::eth::BlockChain& bc() { return *m_bc; } - - /// InterfaceStub methods - virtual dev::eth::State asOf(h256 const& _block) const override; + /// ClientBase methods using ClientBase::asOf; + virtual dev::eth::State asOf(h256 const& _block) const override; + virtual dev::eth::BlockChain& bc() { return *m_bc; } virtual dev::eth::BlockChain const& bc() const override { return *m_bc; } virtual dev::eth::State preMine() const override { ReadGuard l(x_state); return m_startState; } virtual dev::eth::State postMine() const override { ReadGuard l(x_state); return m_state; }