Blockchain import/export.

One or two minor additons to eth.

eth/main.cpp

@@ -115,9 +115,14 @@ void help()
 		<< "                         <APPDATA>/Etherum or Library/Application Support/Ethereum)." << endl
 		<< "    -D,--create-dag <this/next/number>  Create the DAG in preparation for mining on given block and exit." << endl
 		<< "    -e,--ether-price <n>  Set the ether price in the reference unit e.g. ¢ (Default: 30.679)." << endl
+		<< "    -E,--export <file>  Export file as a concatenated series of blocks and exit." << endl
+		<< "    --export-from <n>  Export only from block n; n may be a decimal, a '0x' prefixed hash, or 'latest'." << endl
+		<< "    --export-to <n>  Export only to block n (inclusive); n may be a decimal, a '0x' prefixed hash, or 'latest'." << endl
+		<< "    --export-only <n>  Equivalent to --export-from n --export-to n." << endl
 		<< "    -f,--force-mining  Mine even when there are no transaction to mine (Default: off)" << endl
 		<< "    -h,--help  Show this help message and exit." << endl
 		<< "    -i,--interactive  Enter interactive mode (default: non-interactive)." << endl
+		<< "    -I,--import <file>  Import file as a concatenated series of blocks and exit." << endl
 #if ETH_JSONRPC
 		<< "    -j,--json-rpc  Enable JSON-RPC server (default: off)." << endl
 		<< "    --json-rpc-port	 Specify JSON-RPC server port (implies '-j', default: " << SensibleHttpPort << ")." << endl
@@ -209,33 +214,61 @@ void doInitDAG(unsigned _n)
 	exit(0);
 }
 
-static const unsigned NoDAGInit = (unsigned)-3;
+enum class OperationMode
+{
+	Node,
+	Import,
+	Export,
+	DAGInit
+};
 
 int main(int argc, char** argv)
 {
-	unsigned initDAG = NoDAGInit;
+	/// Operating mode.
-	string listenIP;
+	OperationMode mode = OperationMode::Node;
-	unsigned short listenPort = 30303;
-	string publicIP;
-	string remoteHost;
-	unsigned short remotePort = 30303;
 	string dbPath;
-	unsigned mining = ~(unsigned)0;
+
-	NodeMode mode = NodeMode::Full;
+	/// File name for import/export.
-	unsigned peers = 5;
+	string filename;
-	int miners = -1;
+
+	/// Hashes/numbers for export range.
+	string exportFrom = "1";
+	string exportTo = "latest";
+
+	/// DAG initialisation param.
+	unsigned initDAG;
+
+	/// General params for Node operation
+	NodeMode nodeMode = NodeMode::Full;
 	bool interactive = false;
 #if ETH_JSONRPC
 	int jsonrpc = -1;
 #endif
-	bool bootstrap = false;
 	bool upnp = true;
-	bool forceMining = false;
 	WithExisting killChain = WithExisting::Trust;
 	bool jit = false;
+
+	/// Networking params.
+	string clientName;
+	string listenIP;
+	unsigned short listenPort = 30303;
+	string publicIP;
+	string remoteHost;
+	unsigned short remotePort = 30303;
+	unsigned peers = 5;
+	bool bootstrap = false;
+
+	/// Mining params
+	unsigned mining = ~(unsigned)0;
+	int miners = -1;
+	bool forceMining = false;
+	bool turboMining = false;
+
+	/// Structured logging params
 	bool structuredLogging = false;
 	string structuredLoggingFormat = "%Y-%m-%dT%H:%M:%S";
-	string clientName;
+
+	/// Transaction params
 	TransactionPriority priority = TransactionPriority::Medium;
 	double etherPrice = 30.679;
 	double blockFees = 15.0;
@@ -275,6 +308,16 @@ int main(int argc, char** argv)
 			remoteHost = argv[++i];
 		else if ((arg == "-p" || arg == "--port") && i + 1 < argc)
 			remotePort = (short)atoi(argv[++i]);
+		else if ((arg == "-I" || arg == "--import") && i + 1 < argc)
+		{
+			mode = OperationMode::Import;
+			filename = argv[++i];
+		}
+		else if ((arg == "-E" || arg == "--export") && i + 1 < argc)
+		{
+			mode = OperationMode::Export;
+			filename = argv[++i];
+		}
 		else if ((arg == "-n" || arg == "--upnp") && i + 1 < argc)
 		{
 			string m = argv[++i];
@@ -321,6 +364,7 @@ int main(int argc, char** argv)
 		else if ((arg == "-D" || arg == "--create-dag") && i + 1 < argc)
 		{
 			string m = boost::to_lower_copy(string(argv[++i]));
+			mode = OperationMode::DAGInit;
 			if (m == "next")
 				initDAG = PendingBlock;
 			else if (m == "this")
@@ -402,6 +446,8 @@ int main(int argc, char** argv)
 			bootstrap = true;
 		else if (arg == "-f" || arg == "--force-mining")
 			forceMining = true;
+		else if (arg == "-T" || arg == "--turbo-mining")
+			turboMining = true;
 		else if (arg == "-i" || arg == "--interactive")
 			interactive = true;
 #if ETH_JSONRPC
@@ -420,9 +466,9 @@ int main(int argc, char** argv)
 		{
 			string m = argv[++i];
 			if (m == "full")
-				mode = NodeMode::Full;
+				nodeMode = NodeMode::Full;
 			else if (m == "peer")
-				mode = NodeMode::PeerServer;
+				nodeMode = NodeMode::PeerServer;
 			else
 			{
 				cerr << "Unknown mode: " << m << endl;
@@ -452,7 +498,7 @@ int main(int argc, char** argv)
 
 	// Two codepaths is necessary since named block require database, but numbered
 	// blocks are superuseful to have when database is already open in another process.
-	if (initDAG < NoDAGInit)
+	if (mode == OperationMode::DAGInit && !(initDAG == LatestBlock || initDAG == PendingBlock))
 		doInitDAG(initDAG);
 
 	if (!clientName.empty())
@@ -469,23 +515,69 @@ int main(int argc, char** argv)
 		clientImplString,
 		dbPath,
 		killChain,
-		mode == NodeMode::Full ? set<string>{"eth", "shh"} : set<string>(),
+		nodeMode == NodeMode::Full ? set<string>{"eth", "shh"} : set<string>(),
 		netPrefs,
 		&nodesState,
 		miners
 		);
 	
-	if (initDAG == LatestBlock || initDAG == PendingBlock)
+	if (mode == OperationMode::DAGInit)
 		doInitDAG(web3.ethereum()->blockChain().number() + (initDAG == PendingBlock ? 30000 : 0));
-	
+
+	auto toNumber = [&](string const& s) -> unsigned {
+		if (s == "latest")
+			return web3.ethereum()->number();
+		if (s.size() == 64 || (s.size() == 66 && s.substr(0, 2) == "0x"))
+			return web3.ethereum()->blockChain().number(h256(s));
+		try {
+			return stol(s);
+		}
+		catch (...)
+		{
+			cerr << "Bad block number/hash option: " << s << endl;
+			exit(-1);
+		}
+	};
+
+	if (mode == OperationMode::Export)
+	{
+		ofstream fout;
+		fout.open(filename);
+		unsigned last = toNumber(exportTo);
+		for (unsigned i = toNumber(exportFrom); i <= last; ++i)
+		{
+			bytes block = web3.ethereum()->blockChain().block(web3.ethereum()->blockChain().numberHash(i));
+			fout.write((char const*)block.data(), block.size());
+		}
+		return 0;
+	}
+
+	if (mode == OperationMode::Import)
+	{
+		ifstream fin;
+		fin.open(filename);
+
+		while (fin)
+		{
+			bytes block(8);
+			fin.read((char*)block.data(), 8);
+			unsigned remaining = RLP(block, RLP::LaisezFaire).actualSize() - 8;
+			block.resize(remaining);
+			fin.read((char*)block.data() + 8, remaining);
+			web3.ethereum()->injectBlock(block);
+		}
+		return 0;
+	}
+
 	web3.setIdealPeerCount(peers);
 	std::shared_ptr<eth::BasicGasPricer> gasPricer = make_shared<eth::BasicGasPricer>(u256(double(ether / 1000) / etherPrice), u256(blockFees * 1000));
-	eth::Client* c = mode == NodeMode::Full ? web3.ethereum() : nullptr;
+	eth::Client* c = nodeMode == NodeMode::Full ? web3.ethereum() : nullptr;
 	StructuredLogger::starting(clientImplString, dev::Version);
 	if (c)
 	{
 		c->setGasPricer(gasPricer);
 		c->setForceMining(forceMining);
+		c->setTurboMining(turboMining);
 		c->setAddress(coinbase);
 	}
 

libdevcore/RLP.cpp

@@ -107,7 +107,7 @@ unsigned RLP::actualSize() const
 	if (isSingleByte())
 		return 1;
 	if (isData() || isList())
-		return payload().data() - m_data.data() + length();
+		return payloadOffset() + length();
 	return 0;
 }
 

libdevcore/RLP.h

@@ -290,7 +290,7 @@ public:
 	RLPs toList() const;
 
 	/// @returns the data payload. Valid for all types.
-	bytesConstRef payload() const { return isSingleByte() ? m_data.cropped(0, 1) : m_data.cropped(1 + lengthSize()); }
+	bytesConstRef payload() const { return m_data.cropped(payloadOffset()); }
 
 	/// @returns the theoretical size of this item.
 	/// @note Under normal circumstances, is equivalent to m_data.size() - use that unless you know it won't work.
@@ -309,6 +309,9 @@ private:
 	/// @returns the size in bytes of the payload, as given by the RLP as opposed to as inferred from m_data.
 	unsigned length() const;
 
+	/// @returns the number of bytes into the data that the payload starts.
+	unsigned payloadOffset() const { return isSingleByte() ? 0 : (1 + lengthSize()); }
+
 	/// @returns the number of data items.
 	unsigned items() const;
 

libdevcrypto/Common.cpp

@@ -37,9 +37,9 @@ static Secp256k1 s_secp256k1;
 
 bool dev::SignatureStruct::isValid() const
 {
-	if (this->v > 1 ||
+	if (v > 1 ||
-			this->r >= h256("0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141") ||
+		r >= h256("0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141") ||
-			this->s >= h256("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2f"))
+		s >= h256("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2f"))
 		return false;
 	return true;
 }

libethereum/BlockChain.cpp

@@ -240,7 +240,7 @@ void BlockChain::rebuild(std::string const& _path, std::function<void(unsigned,
 				return;
 			}
 			lastHash = bi.hash();
-			import(b, s.db(), true);
+			import(b, s.db(), Aversion::ImportOldBlocks);
 		}
 		catch (...)
 		{
@@ -334,7 +334,7 @@ tuple<h256s, h256s, bool> BlockChain::sync(BlockQueue& _bq, OverlayDB const& _st
 	return make_tuple(fresh, dead, _bq.doneDrain(badBlocks));
 }
 
-pair<h256s, h256> BlockChain::attemptImport(bytes const& _block, OverlayDB const& _stateDB, bool _force) noexcept
+pair<h256s, h256> BlockChain::attemptImport(bytes const& _block, OverlayDB const& _stateDB, Aversion _force) noexcept
 {
 	try
 	{
@@ -347,7 +347,7 @@ pair<h256s, h256> BlockChain::attemptImport(bytes const& _block, OverlayDB const
 	}
 }
 
-pair<h256s, h256> BlockChain::import(bytes const& _block, OverlayDB const& _db, bool _force)
+pair<h256s, h256> BlockChain::import(bytes const& _block, OverlayDB const& _db, Aversion _force)
 {
 	//@tidy This is a behemoth of a method - could do to be split into a few smaller ones.
 
@@ -386,7 +386,7 @@ pair<h256s, h256> BlockChain::import(bytes const& _block, OverlayDB const& _db,
 #endif
 
 	// Check block doesn't already exist first!
-	if (isKnown(bi.hash()) && !_force)
+	if (isKnown(bi.hash()) && _force == Aversion::AvoidOldBlocks)
 	{
 		clog(BlockChainNote) << bi.hash() << ": Not new.";
 		BOOST_THROW_EXCEPTION(AlreadyHaveBlock());

libethereum/BlockChain.h

@@ -80,6 +80,12 @@ enum {
 
 using ProgressCallback = std::function<void(unsigned, unsigned)>;
 
+enum class Aversion
+{
+	AvoidOldBlocks,
+	ImportOldBlocks
+};
+
 /**
  * @brief Implements the blockchain database. All data this gives is disk-backed.
  * @threadsafe
@@ -102,11 +108,11 @@ public:
 
 	/// Attempt to import the given block directly into the CanonBlockChain and sync with the state DB.
 	/// @returns the block hashes of any blocks that came into/went out of the canonical block chain.
-	std::pair<h256s, h256> attemptImport(bytes const& _block, OverlayDB const& _stateDB, bool _force = false) noexcept;
+	std::pair<h256s, h256> attemptImport(bytes const& _block, OverlayDB const& _stateDB, Aversion _force = Aversion::AvoidOldBlocks) noexcept;
 
 	/// Import block into disk-backed DB
 	/// @returns the block hashes of any blocks that came into/went out of the canonical block chain.
-	std::pair<h256s, h256> import(bytes const& _block, OverlayDB const& _stateDB, bool _force = false);
+	std::pair<h256s, h256> import(bytes const& _block, OverlayDB const& _stateDB, Aversion _force = Aversion::AvoidOldBlocks);
 
 	/// Returns true if the given block is known (though not necessarily a part of the canon chain).
 	bool isKnown(h256 const& _hash) const;

libethereum/Client.h

@@ -229,8 +229,9 @@ public:
 
 protected:
 	/// InterfaceStub methods
+	virtual BlockChain& bc() override { return m_bc; }
 	virtual BlockChain const& bc() const override { return m_bc; }
-	
+
 	/// Returns the state object for the full block (i.e. the terminal state) for index _h.
 	/// Works properly with LatestBlock and PendingBlock.
 	using ClientBase::asOf;

libethereum/ClientBase.cpp

@@ -107,6 +107,11 @@ ExecutionResult ClientBase::create(Secret _secret, u256 _value, bytes const& _da
 	return ret;
 }
 
+void ClientBase::injectBlock(bytes const& _block)
+{
+	bc().import(_block, preMine().db());
+}
+
 u256 ClientBase::balanceAt(Address _a, BlockNumber _block) const
 {
 	return asOf(_block).balance(_a);

libethereum/ClientBase.h

@@ -109,7 +109,6 @@ public:
 	virtual LocalisedLogEntries peekWatch(unsigned _watchId) const override;
 	virtual LocalisedLogEntries checkWatch(unsigned _watchId) override;
 
-	// TODO: switch all the _blockHash arguments to also accept _blockNumber
 	virtual h256 hashFromNumber(BlockNumber _number) const override;
 	virtual eth::BlockInfo blockInfo(h256 _hash) const override;
 	virtual eth::BlockDetails blockDetails(h256 _hash) const override;
@@ -125,6 +124,8 @@ public:
 	virtual eth::Transactions pending() const override;
 	virtual h256s pendingHashes() const override;
 
+	void injectBlock(bytes const& _block);
+
 	using Interface::diff;
 	virtual StateDiff diff(unsigned _txi, h256 _block) const override;
 	virtual StateDiff diff(unsigned _txi, BlockNumber _block) const override;
@@ -155,6 +156,7 @@ public:
 protected:
 	/// The interface that must be implemented in any class deriving this.
 	/// {
+	virtual BlockChain& bc() = 0;
 	virtual BlockChain const& bc() const = 0;
 	virtual State asOf(h256 const& _h) const = 0;
 	virtual State preMine() const = 0;

libtestutils/FixedClient.h

@@ -42,6 +42,7 @@ public:
 	
 	// stub
 	virtual void flushTransactions() override {}
+	virtual eth::BlockChain& bc() override { BOOST_THROW_EXCEPTION(InterfaceNotSupported("FixedClient::bc()")); }
 	virtual eth::BlockChain const& bc() const override { return m_bc; }
 	using ClientBase::asOf;
 	virtual eth::State asOf(h256 const& _h) const override;

mix/MixClient.h

@@ -79,11 +79,10 @@ public:
 	std::vector<KeyPair> userAccounts() { return m_userAccounts; }
 
 protected:
-	virtual dev::eth::BlockChain& bc() { return *m_bc; }
+	/// ClientBase methods
-
-	/// InterfaceStub methods
-	virtual dev::eth::State asOf(h256 const& _block) const override;
 	using ClientBase::asOf;
+	virtual dev::eth::State asOf(h256 const& _block) const override;
+	virtual dev::eth::BlockChain& bc() { return *m_bc; }
 	virtual dev::eth::BlockChain const& bc() const override { return *m_bc; }
 	virtual dev::eth::State preMine() const override { ReadGuard l(x_state);  return m_startState; }
 	virtual dev::eth::State postMine() const override { ReadGuard l(x_state); return m_state; }