diff --git a/libethcore/Common.cpp b/libethcore/Common.cpp index 65c0b8b92..9eb622fe3 100644 --- a/libethcore/Common.cpp +++ b/libethcore/Common.cpp @@ -33,13 +33,14 @@ namespace eth { const unsigned c_protocolVersion = 56; -const unsigned c_databaseVersion = 6 + +const unsigned c_databaseBaseVersion = 7; #if ETH_FATDB - 1000 +const unsigned c_databaseVersionModifier = 1000; #else - 0 +const unsigned c_databaseVersionModifier = 0; #endif -; + +const unsigned c_databaseVersion = c_databaseBaseVersion + c_databaseVersionModifier; vector> const& units() { diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index ac88e7cdf..10f44576b 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -46,7 +46,7 @@ namespace js = json_spirit; std::ostream& dev::eth::operator<<(std::ostream& _out, BlockChain const& _bc) { string cmp = toBigEndianString(_bc.currentHash()); - auto it = _bc.m_db->NewIterator(_bc.m_readOptions); + auto it = _bc.m_blocksDB->NewIterator(_bc.m_readOptions); for (it->SeekToFirst(); it->Valid(); it->Next()) if (it->key().ToString() != "best") { @@ -103,9 +103,9 @@ void BlockChain::open(std::string _path, bool _killExisting) ldb::Options o; o.create_if_missing = true; - ldb::DB::Open(o, _path + "/blocks", &m_db); + ldb::DB::Open(o, _path + "/blocks", &m_blocksDB); ldb::DB::Open(o, _path + "/details", &m_extrasDB); - if (!m_db) + if (!m_blocksDB) BOOST_THROW_EXCEPTION(DatabaseAlreadyOpen()); if (!m_extrasDB) BOOST_THROW_EXCEPTION(DatabaseAlreadyOpen()); @@ -133,10 +133,10 @@ void BlockChain::close() { cnote << "Closing blockchain DB"; delete m_extrasDB; - delete m_db; + delete m_blocksDB; m_lastBlockHash = m_genesisHash; m_details.clear(); - m_cache.clear(); + m_blocks.clear(); } template @@ -295,6 +295,23 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db) m_details[newHash] = BlockDetails((unsigned)pd.number + 1, td, bi.parentHash, {}); m_details[bi.parentHash].children.push_back(newHash); } + { + WriteGuard l(x_blockHashes); + m_blockHashes[h256(bi.number)].value = newHash; + } + // Collate transaction hashes and remember who they were. + h256s tas; + { + RLP blockRLP(_block); + TransactionAddress ta; + ta.blockHash = newHash; + WriteGuard l(x_transactionAddresses); + for (ta.index = 0; ta.index < blockRLP[1].itemCount(); ++ta.index) + { + tas.push_back(sha3(blockRLP[1][ta.index].data())); + m_transactionAddresses[tas.back()] = ta; + } + } { WriteGuard l(x_logBlooms); m_logBlooms[newHash] = blb; @@ -304,17 +321,14 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db) m_receipts[newHash] = br; } - m_extrasDB->Put(m_writeOptions, toSlice(newHash), (ldb::Slice)dev::ref(m_details[newHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(bi.parentHash), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(newHash, 3), (ldb::Slice)dev::ref(m_logBlooms[newHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(newHash, 4), (ldb::Slice)dev::ref(m_receipts[newHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(newHash, 4), (ldb::Slice)dev::ref(m_receipts[newHash].rlp())); - m_db->Put(m_writeOptions, toSlice(newHash), (ldb::Slice)ref(_block)); - RLP blockRLP(_block); - TransactionAddress ta; - ta.blockHash = newHash; - for (ta.index = 0; ta.index < blockRLP[1].itemCount(); ++ta.index) - m_extrasDB->Put(m_writeOptions, toSlice(sha3(blockRLP[1][ta.index].data()), 5), (ldb::Slice)dev::ref(ta.rlp())); + m_blocksDB->Put(m_writeOptions, toSlice(newHash), (ldb::Slice)ref(_block)); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[newHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(bi.parentHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(h256(bi.number), ExtraBlockHash), (ldb::Slice)dev::ref(m_blockHashes[h256(bi.number)].rlp())); + for (auto const& h: tas) + m_extrasDB->Put(m_writeOptions, toSlice(h, ExtraTransactionAddress), (ldb::Slice)dev::ref(m_transactionAddresses[h].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraLogBlooms), (ldb::Slice)dev::ref(m_logBlooms[newHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraReceipts), (ldb::Slice)dev::ref(m_receipts[newHash].rlp())); #if ETH_PARANOIA checkConsistency(); @@ -420,7 +434,7 @@ void BlockChain::checkConsistency() WriteGuard l(x_details); m_details.clear(); } - ldb::Iterator* it = m_db->NewIterator(m_readOptions); + ldb::Iterator* it = m_blocksDB->NewIterator(m_readOptions); for (it->SeekToFirst(); it->Valid(); it->Next()) if (it->key().size() == 32) { @@ -463,12 +477,12 @@ bool BlockChain::isKnown(h256 _hash) const if (_hash == m_genesisHash) return true; { - ReadGuard l(x_cache); - if (m_cache.count(_hash)) + ReadGuard l(x_blocks); + if (m_blocks.count(_hash)) return true; } string d; - m_db->Get(m_readOptions, ldb::Slice((char const*)&_hash, 32), &d); + m_blocksDB->Get(m_readOptions, ldb::Slice((char const*)&_hash, 32), &d); return !!d.size(); } @@ -478,14 +492,14 @@ bytes BlockChain::block(h256 _hash) const return m_genesisBlock; { - ReadGuard l(x_cache); - auto it = m_cache.find(_hash); - if (it != m_cache.end()) + ReadGuard l(x_blocks); + auto it = m_blocks.find(_hash); + if (it != m_blocks.end()) return it->second; } string d; - m_db->Get(m_readOptions, ldb::Slice((char const*)&_hash, 32), &d); + m_blocksDB->Get(m_readOptions, ldb::Slice((char const*)&_hash, 32), &d); if (!d.size()) { @@ -493,18 +507,9 @@ bytes BlockChain::block(h256 _hash) const return bytes(); } - WriteGuard l(x_cache); - m_cache[_hash].resize(d.size()); - memcpy(m_cache[_hash].data(), d.data(), d.size()); - - return m_cache[_hash]; -} + WriteGuard l(x_blocks); + m_blocks[_hash].resize(d.size()); + memcpy(m_blocks[_hash].data(), d.data(), d.size()); -h256 BlockChain::numberHash(unsigned _n) const -{ - if (!_n) - return genesisHash(); - h256 ret = currentHash(); - for (; _n < details().number; ++_n, ret = details(ret).parent) {} - return ret; + return m_blocks[_hash]; } diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h index cd3053c63..62a5bf792 100644 --- a/libethereum/BlockChain.h +++ b/libethereum/BlockChain.h @@ -62,8 +62,17 @@ std::map const& genesisState(); ldb::Slice toSlice(h256 _h, unsigned _sub = 0); +using BlocksHash = std::map; using TransactionHashes = h256s; +enum { + ExtraDetails = 0, + ExtraBlockHash, + ExtraTransactionAddress, + ExtraLogBlooms, + ExtraReceipts +}; + /** * @brief Implements the blockchain database. All data this gives is disk-backed. * @threadsafe @@ -99,28 +108,31 @@ public: BlockInfo info(h256 _hash) const { return BlockInfo(block(_hash)); } BlockInfo info() const { return BlockInfo(block()); } - /// Get the familiar details concerning a block (or the most recent mined if none given). Thread-safe. - BlockDetails details(h256 _hash) const { return queryExtras(_hash, m_details, x_details, NullBlockDetails); } + /// Get a block (RLP format) for the given hash (or the most recent mined if none given). Thread-safe. + bytes block(h256 _hash) const; + bytes block() const { return block(currentHash()); } + + /// Get the familial details concerning a block (or the most recent mined if none given). Thread-safe. + BlockDetails details(h256 _hash) const { return queryExtras(_hash, m_details, x_details, NullBlockDetails); } BlockDetails details() const { return details(currentHash()); } /// Get the transactions' log blooms of a block (or the most recent mined if none given). Thread-safe. - BlockLogBlooms logBlooms(h256 _hash) const { return queryExtras(_hash, m_logBlooms, x_logBlooms, NullBlockLogBlooms); } + BlockLogBlooms logBlooms(h256 _hash) const { return queryExtras(_hash, m_logBlooms, x_logBlooms, NullBlockLogBlooms); } BlockLogBlooms logBlooms() const { return logBlooms(currentHash()); } /// Get the transactions' receipts of a block (or the most recent mined if none given). Thread-safe. - BlockReceipts receipts(h256 _hash) const { return queryExtras(_hash, m_receipts, x_receipts, NullBlockReceipts); } + BlockReceipts receipts(h256 _hash) const { return queryExtras(_hash, m_receipts, x_receipts, NullBlockReceipts); } BlockReceipts receipts() const { return receipts(currentHash()); } /// Get a list of transaction hashes for a given block. Thread-safe. TransactionHashes transactionHashes(h256 _hash) const { auto b = block(_hash); RLP rlp(b); h256s ret; for (auto t: rlp[1]) ret.push_back(sha3(t.data())); return ret; } TransactionHashes transactionHashes() const { return transactionHashes(currentHash()); } - /// Get a block (RLP format) for the given hash (or the most recent mined if none given). Thread-safe. - bytes block(h256 _hash) const; - bytes block() const { return block(currentHash()); } + /// Get a list of transaction hashes for a given block. Thread-safe. + h256 numberHash(u256 _index) const { if (!_index) return genesisHash(); return queryExtras(h256(_index), m_blockHashes, x_blockHashes, NullBlockHash).value; } /// Get a transaction from its hash. Thread-safe. - bytes transaction(h256 _transactionHash) const { TransactionAddress ta = queryExtras(_transactionHash, m_transactionAddresses, x_transactionAddresses, NullTransactionAddress); if (!ta) return bytes(); return transaction(ta.blockHash, ta.index); } + bytes transaction(h256 _transactionHash) const { TransactionAddress ta = queryExtras(_transactionHash, m_transactionAddresses, x_transactionAddresses, NullTransactionAddress); if (!ta) return bytes(); return transaction(ta.blockHash, ta.index); } /// Get a block's transaction (RLP format) for the given block hash (or the most recent mined if none given) & index. Thread-safe. bytes transaction(h256 _blockHash, unsigned _i) const { bytes b = block(_blockHash); return RLP(b)[1][_i].data().toBytes(); } @@ -136,9 +148,6 @@ public: /// Get the hash of the genesis block. Thread-safe. h256 genesisHash() const { return m_genesisHash; } - /// Get the hash of a block of a given number. Slow; try not to use it too much. - h256 numberHash(unsigned _n) const; - /// Get all blocks not allowed as uncles given a parent (i.e. featured as uncles/main in parent, parent + 1, ... parent + 5). /// @returns set including the header-hash of every parent (including @a _parent) up to and including generation +5 /// togther with all their quoted uncles. @@ -160,6 +169,21 @@ public: */ h256s treeRoute(h256 _from, h256 _to, h256* o_common = nullptr, bool _pre = true, bool _post = true) const; + struct Statistics + { + unsigned memDetails; + unsigned memLogBlooms; + unsigned memReceipts; + unsigned memTransactionAddresses; + unsigned memCache; + }; + + /// @returns statistics about memory usage. + Statistics usage() const; + + /// Deallocate unused data. + void garbageCollect(); + private: void open(std::string _path, bool _killExisting = false); void close(); @@ -189,6 +213,8 @@ private: void checkConsistency(); /// The caches of the disk DB and their locks. + mutable boost::shared_mutex x_blocks; + mutable BlocksHash m_blocks; mutable boost::shared_mutex x_details; mutable BlockDetailsHash m_details; mutable boost::shared_mutex x_logBlooms; @@ -197,11 +223,11 @@ private: mutable BlockReceiptsHash m_receipts; mutable boost::shared_mutex x_transactionAddresses; mutable TransactionAddressHash m_transactionAddresses; - mutable boost::shared_mutex x_cache; - mutable std::map m_cache; + mutable boost::shared_mutex x_blockHashes; + mutable BlockHashHash m_blockHashes; /// The disk DBs. Thread-safe, so no need for locks. - ldb::DB* m_db; + ldb::DB* m_blocksDB; ldb::DB* m_extrasDB; /// Hash of the last (valid) block on the longest chain. diff --git a/libethereum/BlockDetails.h b/libethereum/BlockDetails.h index b59d165ab..f9bccea1e 100644 --- a/libethereum/BlockDetails.h +++ b/libethereum/BlockDetails.h @@ -70,6 +70,15 @@ struct BlockReceipts TransactionReceipts receipts; }; +struct BlockHash +{ + BlockHash() {} + BlockHash(RLP const& _r) { value = _r.toHash(); } + bytes rlp() const { return dev::rlp(value); } + + h256 value; +}; + struct TransactionAddress { TransactionAddress() {} @@ -86,11 +95,13 @@ using BlockDetailsHash = std::map; using BlockLogBloomsHash = std::map; using BlockReceiptsHash = std::map; using TransactionAddressHash = std::map; +using BlockHashHash = std::map; static const BlockDetails NullBlockDetails; static const BlockLogBlooms NullBlockLogBlooms; static const BlockReceipts NullBlockReceipts; static const TransactionAddress NullTransactionAddress; +static const BlockHash NullBlockHash; } } diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index 2db673d90..827346c95 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -521,9 +521,18 @@ void Client::doWork() { if (m.isComplete()) { - cwork << "CHAIN <== postSTATE"; + // TODO: enable a short-circuit option since we mined it. will need to get the end state from the miner. + auto lm = dynamic_cast(&m); h256s hs; + if (false && lm && !m_verifyOwnBlocks) { + // TODO: implement + //m_bc.attemptImport(m_blockData(), m_stateDB, lm->state()); + // TODO: derive hs from lm->state() + } + else + { + cwork << "CHAIN <== postSTATE"; WriteGuard l(x_stateDB); hs = m_bc.attemptImport(m.blockData(), m_stateDB); } diff --git a/libethereum/Client.h b/libethereum/Client.h index 900d339ad..9fad6dc22 100644 --- a/libethereum/Client.h +++ b/libethereum/Client.h @@ -368,6 +368,7 @@ private: bool m_paranoia = false; ///< Should we be paranoid about our state? bool m_turboMining = false; ///< Don't squander all of our time mining actually just sleeping. bool m_forceMining = false; ///< Mine even when there are no transactions pending? + bool m_verifyOwnBlocks = true; ///< Shoudl be verify blocks that we mined? mutable Mutex m_filterLock; std::map m_filters; diff --git a/libethereum/Miner.h b/libethereum/Miner.h index 0c8eeb684..e472c6f64 100644 --- a/libethereum/Miner.h +++ b/libethereum/Miner.h @@ -131,6 +131,9 @@ public: /// Get and clear the mining history. std::list miningHistory() { Guard l(x_mineInfo); auto ret = m_mineHistory; m_mineHistory.clear(); return ret; } + /// @returns the state on which we mined. + State const& state() const { return m_mineState; } + private: /// Do some work on the mining. virtual void doWork();