Parallelised large-file imports.

eth/main.cpp

@@ -187,6 +187,7 @@ void help()
 		<< "    --from <n>  Export only from block n; n may be a decimal, a '0x' prefixed hash, or 'latest'." << endl
 		<< "    --to <n>  Export only to block n (inclusive); n may be a decimal, a '0x' prefixed hash, or 'latest'." << endl
 		<< "    --only <n>  Equivalent to --export-from n --export-to n." << endl
+		<< "    --dont-check  Avoids checking some of the aspects of blocks. Faster importing, but only do if you know the data is valid." << endl
 		<< endl
 		<< "General Options:" << endl
 		<< "    -d,--db-path <path>  Load database from path (default: " << getDataDir() << ")" << endl
@@ -283,6 +284,7 @@ int main(int argc, char** argv)
 
 	/// File name for import/export.
 	string filename;
+	bool safeImport = false;
 
 	/// Hashes/numbers for export range.
 	string exportFrom = "1";
@@ -395,6 +397,8 @@ int main(int argc, char** argv)
 			mode = OperationMode::Import;
 			filename = argv[++i];
 		}
+		else if (arg == "--dont-check")
+			safeImport = true;
 		else if ((arg == "-E" || arg == "--export") && i + 1 < argc)
 		{
 			mode = OperationMode::Export;
@@ -753,13 +757,18 @@ int main(int argc, char** argv)
 		unsigned futureTime = 0;
 		unsigned unknownParent = 0;
 		unsigned bad = 0;
+		chrono::steady_clock::time_point t = chrono::steady_clock::now();
+		double last = 0;
+		unsigned lastImported = 0;
+		unsigned imported = 0;
 		while (in.peek() != -1)
 		{
 			bytes block(8);
 			in.read((char*)block.data(), 8);
 			block.resize(RLP(block, RLP::LaisezFaire).actualSize());
 			in.read((char*)block.data() + 8, block.size() - 8);
-			switch (web3.ethereum()->injectBlock(block))
+
+			switch (web3.ethereum()->queueBlock(block, safeImport))
 			{
 			case ImportResult::Success: good++; break;
 			case ImportResult::AlreadyKnown: alreadyHave++; break;
@@ -768,8 +777,29 @@ int main(int argc, char** argv)
 			case ImportResult::FutureTimeKnown: futureTime++; break;
 			default: bad++; break;
 			}
+
+			// sync chain with queue
+			tuple<ImportRoute, bool, unsigned> r = web3.ethereum()->syncQueue(10);
+			imported += get<2>(r);
+
+			double e = chrono::duration_cast<chrono::milliseconds>(chrono::steady_clock::now() - t).count() / 1000.0;
+			if ((unsigned)e >= last + 10)
+			{
+				auto i = imported - lastImported;
+				auto d = e - last;
+				cout << i << " more imported at " << (round(i * 10 / d) / 10) << " blocks/s. " << imported << " imported in " << e << " seconds at " << (round(imported * 10 / e) / 10) << " blocks/s (#" << web3.ethereum()->number() << ")" << endl;
+				last = (unsigned)e;
+				lastImported = imported;
+			}
+		}
+
+		while (web3.ethereum()->blockQueue().items().first + web3.ethereum()->blockQueue().items().second > 0)
+		{
+			sleep(1);
+			web3.ethereum()->syncQueue(100000);
 		}
-		cout << (good + bad + futureTime + unknownParent + alreadyHave) << " total: " << good << " ok, " << alreadyHave << " got, " << futureTime << " future, " << unknownParent << " unknown parent, " << bad << " malformed." << endl;
+		double e = chrono::duration_cast<chrono::milliseconds>(chrono::steady_clock::now() - t).count() / 1000.0;
+		cout << imported << " imported in " << e << " seconds at " << (round(imported * 10 / e) / 10) << " blocks/s (#" << web3.ethereum()->number() << ")" << endl;
 		return 0;
 	}
 

libethcore/Common.h

@@ -97,6 +97,12 @@ enum class RelativeBlock: BlockNumber
 	Pending = PendingBlock
 };
 
+struct ImportRoute
+{
+	h256s deadBlocks;
+	h256s liveBlocks;
+};
+
 enum class ImportResult
 {
 	Success = 0,

libethereum/BlockChain.cpp

@@ -1116,6 +1116,7 @@ VerifiedBlockRef BlockChain::verifyBlock(bytes const& _block, function<void(Exce
 		catch (Exception& ex)
 		{
 			ex << errinfo_transactionIndex(i);
+			ex << errinfo_transaction(tr.data().toBytes());
 			ex << errinfo_block(_block);
 			throw;
 		}

libethereum/BlockChain.h

@@ -76,12 +76,6 @@ using BlocksHash = std::unordered_map<h256, bytes>;
 using TransactionHashes = h256s;
 using UncleHashes = h256s;
 
-struct ImportRoute
-{
-	h256s deadBlocks;
-	h256s liveBlocks;
-};
-
 enum {
 	ExtraDetails = 0,
 	ExtraBlockHash,

libethereum/Client.cpp

@@ -87,6 +87,18 @@ void VersionChecker::setOk()
 	}
 }
 
+ImportResult Client::queueBlock(bytes const& _block, bool _isSafe)
+{
+	if (m_bq.status().verified + m_bq.status().verifying + m_bq.status().unverified > 30000)
+		sleep(1);
+	return m_bq.import(&_block, bc(), _isSafe);
+}
+
+tuple<ImportRoute, bool, unsigned> Client::syncQueue(unsigned _max)
+{
+	return m_bc.sync(m_bq, m_stateDB, _max);
+}
+
 void Client::onBadBlock(Exception& _ex) const
 {
 	// BAD BLOCK!!!

libethereum/Client.h

@@ -138,6 +138,9 @@ public:
 	/// Blocks until all pending transactions have been processed.
 	virtual void flushTransactions() override;
 
+	/// Queues a block for import.
+	ImportResult queueBlock(bytes const& _block, bool _isSafe = false);
+
 	using Interface::call; // to remove warning about hiding virtual function
 	/// Makes the given call. Nothing is recorded into the state. This cheats by creating a null address and endowing it with a lot of ETH.
 	ExecutionResult call(Address _dest, bytes const& _data = bytes(), u256 _gas = 125000, u256 _value = 0, u256 _gasPrice = 1 * ether, Address const& _from = Address());
@@ -160,6 +163,11 @@ public:
 	SyncStatus syncStatus() const;
 	/// Get the block queue.
 	BlockQueue const& blockQueue() const { return m_bq; }
+	/// Get the block queue.
+	OverlayDB const& stateDB() const { return m_stateDB; }
+
+	/// Freeze worker thread and sync some of the block queue.
+	std::tuple<ImportRoute, bool, unsigned> syncQueue(unsigned _max = 1);
 
 	// Mining stuff:
 

libethereum/State.h

@@ -58,6 +58,8 @@ using errinfo_transactionIndex = boost::error_info<struct tag_transactionIndex,
 
 using errinfo_vmtrace = boost::error_info<struct tag_vmtrace, std::string>;
 using errinfo_receipts = boost::error_info<struct tag_receipts, std::vector<bytes>>;
+using errinfo_transaction = boost::error_info<struct tag_transaction, bytes>;
+using errinfo_phase = boost::error_info<struct tag_phase, unsigned>;
 using errinfo_required_LogBloom = boost::error_info<struct tag_required_LogBloom, LogBloom>;
 using errinfo_got_LogBloom = boost::error_info<struct tag_get_LogBloom, LogBloom>;
 using LogBloomRequirementError = boost::tuple<errinfo_required_LogBloom, errinfo_got_LogBloom>;