diff --git a/libethcore/BlockInfo.cpp b/libethcore/BlockInfo.cpp index 6b4d5f01f..e441b749b 100644 --- a/libethcore/BlockInfo.cpp +++ b/libethcore/BlockInfo.cpp @@ -119,8 +119,8 @@ void BlockInfo::verifyInternals(bytesConstRef _block) const u256 mgp = (u256)-1; - Overlay db; - GenericTrieDB t(&db); + OverlayDB db; + GenericTrieDB t(&db); t.init(); unsigned i = 0; for (auto const& tr: root[1]) diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index b7db36910..27f1b337e 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -97,8 +97,8 @@ bytes BlockChain::createGenesisBlock() h256 stateRoot; { - BasicMap db; - TrieDB state(&db); + MemoryDB db; + TrieDB state(&db); state.init(); eth::commit(genesisState(), db, state); stateRoot = state.root(); @@ -167,7 +167,7 @@ bool contains(T const& _t, V const& _v) return false; } -bool BlockChain::attemptImport(bytes const& _block, Overlay const& _stateDB) +bool BlockChain::attemptImport(bytes const& _block, OverlayDB const& _stateDB) { #if ETH_CATCH try @@ -185,7 +185,7 @@ bool BlockChain::attemptImport(bytes const& _block, Overlay const& _stateDB) } -void BlockChain::import(bytes const& _block, Overlay const& _db) +void BlockChain::import(bytes const& _block, OverlayDB const& _db) { // VERIFY: populates from the block and checks the block is internally coherent. BlockInfo bi; diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h index cd3c54807..c76fbb663 100644 --- a/libethereum/BlockChain.h +++ b/libethereum/BlockChain.h @@ -55,7 +55,7 @@ typedef std::map BlockDetailsHash; static const BlockDetails NullBlockDetails; static const h256s NullH256s; -class Overlay; +class OverlayDB; class AlreadyHaveBlock: public std::exception {}; class UnknownParent: public std::exception {}; @@ -82,10 +82,10 @@ public: void process(); /// Attempt to import the given block. - bool attemptImport(bytes const& _block, Overlay const& _stateDB); + bool attemptImport(bytes const& _block, OverlayDB const& _stateDB); /// Import block into disk-backed DB - void import(bytes const& _block, Overlay const& _stateDB); + void import(bytes const& _block, OverlayDB const& _stateDB); /// Get the number of the last block of the longest chain. BlockDetails const& details(h256 _hash) const; diff --git a/libethereum/Client.h b/libethereum/Client.h index 949c1378d..4d1b7f178 100644 --- a/libethereum/Client.h +++ b/libethereum/Client.h @@ -175,7 +175,7 @@ private: VersionChecker m_vc; ///< Dummy object to check & update the protocol version. BlockChain m_bc; ///< Maintains block database. TransactionQueue m_tq; ///< Maintains list of incoming transactions not yet on the block chain. - Overlay m_stateDB; ///< Acts as the central point for the state database, so multiple States can share it. + OverlayDB m_stateDB; ///< Acts as the central point for the state database, so multiple States can share it. State m_preMine; ///< The present state of the client. State m_postMine; ///< The state of the client which we're mining (i.e. it'll have all the rewards added). std::unique_ptr m_net; ///< Should run in background and send us events when blocks found and allow us to send blocks as required. diff --git a/libethereum/PeerNetwork.h b/libethereum/PeerNetwork.h index 3b901999b..8aeff9980 100644 --- a/libethereum/PeerNetwork.h +++ b/libethereum/PeerNetwork.h @@ -37,7 +37,7 @@ namespace eth bool isPrivateAddress(bi::address _addressToCheck); -class Overlay; +class OverlayDB; class BlockChain; class TransactionQueue; class PeerServer; diff --git a/libethereum/PeerServer.cpp b/libethereum/PeerServer.cpp index 19db81c42..58656a05d 100644 --- a/libethereum/PeerServer.cpp +++ b/libethereum/PeerServer.cpp @@ -352,7 +352,7 @@ bool PeerServer::ensureInitialised(BlockChain& _bc, TransactionQueue& _tq) return false; } -bool PeerServer::sync(BlockChain& _bc, TransactionQueue& _tq, Overlay& _o) +bool PeerServer::sync(BlockChain& _bc, TransactionQueue& _tq, OverlayDB& _o) { bool ret = ensureInitialised(_bc, _tq); diff --git a/libethereum/PeerServer.h b/libethereum/PeerServer.h index 05051e82a..47030641b 100644 --- a/libethereum/PeerServer.h +++ b/libethereum/PeerServer.h @@ -57,7 +57,7 @@ public: void connect(bi::tcp::endpoint const& _ep); /// Sync with the BlockChain. It might contain one of our mined blocks, we might have new candidates from the network. - bool sync(BlockChain& _bc, TransactionQueue&, Overlay& _o); + bool sync(BlockChain& _bc, TransactionQueue&, OverlayDB& _o); bool sync(); /// Conduct I/O, polling, syncing, whatever. diff --git a/libethereum/State.cpp b/libethereum/State.cpp index e1f57066e..678cf8d1b 100644 --- a/libethereum/State.cpp +++ b/libethereum/State.cpp @@ -37,7 +37,7 @@ using namespace eth; #define ctrace clog(StateTrace) -Overlay State::openDB(std::string _path, bool _killExisting) +OverlayDB State::openDB(std::string _path, bool _killExisting) { if (_path.empty()) _path = Defaults::get()->m_dbPath; @@ -51,10 +51,10 @@ Overlay State::openDB(std::string _path, bool _killExisting) ldb::DB* db = nullptr; ldb::DB::Open(o, _path + "/state", &db); cnote << "Opened state DB."; - return Overlay(db); + return OverlayDB(db); } -State::State(Address _coinbaseAddress, Overlay const& _db): +State::State(Address _coinbaseAddress, OverlayDB const& _db): m_db(_db), m_state(&m_db), m_ourAddress(_coinbaseAddress) @@ -127,7 +127,7 @@ State& State::operator=(State const& _s) struct CachedAddressState { - CachedAddressState(std::string const& _rlp, AddressState const* _s, Overlay const* _o): rS(_rlp), r(rS), s(_s), o(_o) {} + CachedAddressState(std::string const& _rlp, AddressState const* _s, OverlayDB const* _o): rS(_rlp), r(rS), s(_s), o(_o) {} bool exists() const { @@ -157,7 +157,7 @@ struct CachedAddressState std::map ret; if (r) { - TrieDB memdb(const_cast(o), r[2].toHash()); // promise we won't alter the overlay! :) + TrieDB memdb(const_cast(o), r[2].toHash()); // promise we won't alter the overlay! :) for (auto const& j: memdb) ret[j.first] = RLP(j.second).toInt(); } @@ -204,7 +204,7 @@ struct CachedAddressState std::string rS; RLP r; AddressState const* s; - Overlay const* o; + OverlayDB const* o; }; StateDiff State::diff(State const& _c) const @@ -215,8 +215,8 @@ StateDiff State::diff(State const& _c) const std::set
trieAds; std::set
trieAdsD; - auto trie = TrieDB(const_cast(&m_db), rootHash()); - auto trieD = TrieDB(const_cast(&_c.m_db), _c.rootHash()); + auto trie = TrieDB(const_cast(&m_db), rootHash()); + auto trieD = TrieDB(const_cast(&_c.m_db), _c.rootHash()); for (auto i: trie) ads.insert(i.first), trieAds.insert(i.first); @@ -481,8 +481,8 @@ u256 State::playbackRaw(bytesConstRef _block, BlockInfo const& _grandParent, boo // cnote << "playback begins:" << m_state.root(); // cnote << m_state; - BasicMap tm; - GenericTrieDB transactionManifest(&tm); + MemoryDB tm; + GenericTrieDB transactionManifest(&tm); transactionManifest.init(); // All ok with the block generally. Play back the transactions now... @@ -548,7 +548,7 @@ u256 State::playbackRaw(bytesConstRef _block, BlockInfo const& _grandParent, boo { cwarn << "Bad state root!"; cnote << "Given to be:" << m_currentBlock.stateRoot; - cnote << TrieDB(&m_db, m_currentBlock.stateRoot); + cnote << TrieDB(&m_db, m_currentBlock.stateRoot); cnote << "Calculated to be:" << rootHash(); cnote << m_state; cnote << *this; @@ -663,8 +663,8 @@ void State::commitToMine(BlockChain const& _bc) else uncles.appendList(0); - BasicMap tm; - GenericTrieDB transactionManifest(&tm); + MemoryDB tm; + GenericTrieDB transactionManifest(&tm); transactionManifest.init(); RLPStream txs; @@ -812,7 +812,7 @@ u256 State::storage(Address _id, u256 _memory) const return mit->second; // Not in the storage cache - go to the DB. - TrieDB memdb(const_cast(&m_db), it->second.oldRoot()); // promise we won't change the overlay! :) + TrieDB memdb(const_cast(&m_db), it->second.oldRoot()); // promise we won't change the overlay! :) string payload = memdb.at(_memory); u256 ret = payload.size() ? RLP(payload).toInt() : 0; it->second.setStorage(_memory, ret); @@ -830,7 +830,7 @@ map State::storage(Address _id) const // Pull out all values from trie storage. if (it->second.oldRoot()) { - TrieDB memdb(const_cast(&m_db), it->second.oldRoot()); // promise we won't alter the overlay! :) + TrieDB memdb(const_cast(&m_db), it->second.oldRoot()); // promise we won't alter the overlay! :) for (auto const& i: memdb) ret[i.first] = RLP(i.second).toInt(); } @@ -883,7 +883,7 @@ bool State::isTrieGood(bool _enforceRefs, bool _requireNoLeftOvers) const for (auto const& i: m_state) { RLP r(i.second); - TrieDB storageDB(const_cast(&m_db), r[2].toHash()); // promise not to alter Overlay. + TrieDB storageDB(const_cast(&m_db), r[2].toHash()); // promise not to alter OverlayDB. for (auto const& j: storageDB) { (void)j; } if (!e && r[3].toHash() != EmptySHA3 && m_db.lookup(r[3].toHash()).empty()) return false; @@ -900,8 +900,8 @@ bool State::isTrieGood(bool _enforceRefs, bool _requireNoLeftOvers) const } // TODO: kill temp nodes automatically in TrieDB -// TODO: maintain node overlay revisions for stateroots -> each commit gives a stateroot + Overlay; allow overlay copying for rewind operations. -// TODO: TransactionReceipt trie should be BasicMap and built as necessary +// TODO: maintain node overlay revisions for stateroots -> each commit gives a stateroot + OverlayDB; allow overlay copying for rewind operations. +// TODO: TransactionReceipt trie should be MemoryDB and built as necessary u256 State::execute(bytesConstRef _rlp) { @@ -1095,7 +1095,7 @@ std::ostream& eth::operator<<(std::ostream& _out, State const& _s) _out << "--- " << _s.rootHash() << std::endl; std::set
d; std::set
dtr; - auto trie = TrieDB(const_cast(&_s.m_db), _s.rootHash()); + auto trie = TrieDB(const_cast(&_s.m_db), _s.rootHash()); for (auto i: trie) d.insert(i.first), dtr.insert(i.first); for (auto i: _s.m_cache) @@ -1127,7 +1127,7 @@ std::ostream& eth::operator<<(std::ostream& _out, State const& _s) std::set cached; if (r) { - TrieDB memdb(const_cast(&_s.m_db), r[2].toHash()); // promise we won't alter the overlay! :) + TrieDB memdb(const_cast(&_s.m_db), r[2].toHash()); // promise we won't alter the overlay! :) for (auto const& j: memdb) mem[j.first] = RLP(j.second).toInt(), back.insert(j.first); } diff --git a/libethereum/State.h b/libethereum/State.h index 71703716e..ab3c8b53c 100644 --- a/libethereum/State.h +++ b/libethereum/State.h @@ -112,7 +112,7 @@ class State public: /// Construct state object. - State(Address _coinbaseAddress = Address(), Overlay const& _db = Overlay()); + State(Address _coinbaseAddress = Address(), OverlayDB const& _db = OverlayDB()); /// Copy state object. State(State const& _s); @@ -126,8 +126,8 @@ public: Address address() const { return m_ourAddress; } /// Open a DB - useful for passing into the constructor & keeping for other states that are necessary. - static Overlay openDB(std::string _path, bool _killExisting = false); - static Overlay openDB(bool _killExisting = false) { return openDB(std::string(), _killExisting); } + static OverlayDB openDB(std::string _path, bool _killExisting = false); + static OverlayDB openDB(bool _killExisting = false) { return openDB(std::string(), _killExisting); } /// @returns the set containing all addresses currently in use in Ethereum. std::map addresses() const; @@ -300,12 +300,12 @@ private: bool isTrieGood(bool _enforceRefs, bool _requireNoLeftOvers) const; void paranoia(std::string const& _when, bool _enforceRefs = false) const; - Overlay m_db; ///< Our overlay for the state tree. - TrieDB m_state; ///< Our state tree, as an Overlay DB. + OverlayDB m_db; ///< Our overlay for the state tree. + TrieDB m_state; ///< Our state tree, as an OverlayDB DB. std::vector m_transactions; ///< The current list of transactions that we've included in the state. std::set m_transactionSet; ///< The set of transaction hashes that we've included in the state. -// GenericTrieDB m_transactionManifest; ///< The transactions trie; saved from the last commitToMine, or invalid/empty if commitToMine was never called. - Overlay m_lastTx; +// GenericTrieDB m_transactionManifest; ///< The transactions trie; saved from the last commitToMine, or invalid/empty if commitToMine was never called. + OverlayDB m_lastTx; mutable std::map m_cache; ///< Our address cache. This stores the states of each address that has (or at least might have) been changed. diff --git a/libethsupport/MemoryDB.cpp b/libethsupport/MemoryDB.cpp new file mode 100644 index 000000000..44da8457b --- /dev/null +++ b/libethsupport/MemoryDB.cpp @@ -0,0 +1,115 @@ +/* + This file is part of cpp-ethereum. + + cpp-ethereum is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + cpp-ethereum is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with cpp-ethereum. If not, see . +*/ +/** @file MemoryDB.cpp + * @author Gav Wood + * @date 2014 + */ + +#include "Common.h" +#include "MemoryDB.h" +using namespace std; +using namespace eth; + +namespace eth +{ + +std::map MemoryDB::get() const +{ + if (!m_enforceRefs) + return m_over; + std::map ret; + for (auto const& i: m_refCount) + if (i.second) + ret.insert(*m_over.find(i.first)); + return ret; +} + +std::string MemoryDB::lookup(h256 _h) const +{ + auto it = m_over.find(_h); + if (it != m_over.end()) + { + if (!m_enforceRefs || (m_refCount.count(it->first) && m_refCount.at(it->first))) + return it->second; +// else if (m_enforceRefs && m_refCount.count(it->first) && !m_refCount.at(it->first)) +// cnote << "Lookup required for value with no refs. Let's hope it's in the DB." << _h.abridged(); + } + return std::string(); +} + +bool MemoryDB::exists(h256 _h) const +{ + auto it = m_over.find(_h); + if (it != m_over.end() && (!m_enforceRefs || (m_refCount.count(it->first) && m_refCount.at(it->first)))) + return true; + return false; +} + +void MemoryDB::insert(h256 _h, bytesConstRef _v) +{ + m_over[_h] = _v.toString(); + m_refCount[_h]++; +#if ETH_PARANOIA + dbdebug << "INST" << _h.abridged() << "=>" << m_refCount[_h]; +#endif +} + +bool MemoryDB::kill(h256 _h) +{ + if (m_refCount.count(_h)) + { + if (m_refCount[_h] > 0) + --m_refCount[_h]; +#if ETH_PARANOIA + else + { + // If we get to this point, then there was probably a node in the level DB which we need to remove and which we have previously + // used as part of the memory-based MemoryDB. Nothing to be worried about *as long as the node exists in the DB*. + dbdebug << "NOKILL-WAS" << _h.abridged(); + return false; + } + dbdebug << "KILL" << _h.abridged() << "=>" << m_refCount[_h]; + return true; + } + else + { + dbdebug << "NOKILL" << _h.abridged(); + return false; + } +#else + return true; + } +#endif +} + +void MemoryDB::purge() +{ + for (auto const& i: m_refCount) + if (!i.second) + m_over.erase(i.first); +} + +set MemoryDB::keys() const +{ + set ret; + for (auto const& i: m_refCount) + if (i.second) + ret.insert(i.first); + return ret; +} + +} diff --git a/libethsupport/MemoryDB.h b/libethsupport/MemoryDB.h new file mode 100644 index 000000000..c5627b4d7 --- /dev/null +++ b/libethsupport/MemoryDB.h @@ -0,0 +1,85 @@ +/* + This file is part of cpp-ethereum. + + cpp-ethereum is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + cpp-ethereum is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with cpp-ethereum. If not, see . +*/ +/** @file MemoryDB.h + * @author Gav Wood + * @date 2014 + */ + +#pragma once + +#include +#include "Common.h" +#include "FixedHash.h" +#include "RLP.h" +#include "Log.h" + +namespace eth +{ + +struct DBChannel: public LogChannel { static const char* name() { return "TDB"; } static const int verbosity = 12; }; + +#define dbdebug clog(DBChannel) + +class MemoryDB +{ + friend class EnforceRefs; + +public: + MemoryDB() {} + + void clear() { m_over.clear(); } + std::map get() const; + + std::string lookup(h256 _h) const; + bool exists(h256 _h) const; + void insert(h256 _h, bytesConstRef _v); + bool kill(h256 _h); + void purge(); + + std::set keys() const; + +protected: + std::map m_over; + std::map m_refCount; + + mutable bool m_enforceRefs = false; +}; + +class EnforceRefs +{ +public: + EnforceRefs(MemoryDB const& _o, bool _r): m_o(_o), m_r(_o.m_enforceRefs) { _o.m_enforceRefs = _r; } + ~EnforceRefs() { m_o.m_enforceRefs = m_r; } + +private: + MemoryDB const& m_o; + bool m_r; +}; + +inline std::ostream& operator<<(std::ostream& _out, MemoryDB const& _m) +{ + for (auto i: _m.get()) + { + _out << i.first << ": "; + _out << RLP(i.second); + _out << " " << toHex(i.second); + _out << std::endl; + } + return _out; +} + +} diff --git a/libethsupport/OverlayDB.cpp b/libethsupport/OverlayDB.cpp new file mode 100644 index 000000000..04e576ae1 --- /dev/null +++ b/libethsupport/OverlayDB.cpp @@ -0,0 +1,99 @@ +/* + This file is part of cpp-ethereum. + + cpp-ethereum is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + cpp-ethereum is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with cpp-ethereum. If not, see . +*/ +/** @file TrieDB.cpp + * @author Gav Wood + * @date 2014 + */ + +#include "Common.h" +#include "OverlayDB.h" +using namespace std; +using namespace eth; + +namespace eth +{ + +OverlayDB::~OverlayDB() +{ + if (m_db.use_count() == 1 && m_db.get()) + cnote << "Closing state DB"; +} + +void OverlayDB::setDB(ldb::DB* _db, bool _clearOverlay) +{ + m_db = std::shared_ptr(_db); + if (_clearOverlay) + m_over.clear(); +} + +void OverlayDB::commit() +{ + if (m_db) + { +// cnote << "Committing nodes to disk DB:"; + for (auto const& i: m_over) + { +// cnote << i.first << "#" << m_refCount[i.first]; + if (m_refCount[i.first]) + m_db->Put(m_writeOptions, ldb::Slice((char const*)i.first.data(), i.first.size), ldb::Slice(i.second.data(), i.second.size())); + } + m_over.clear(); + m_refCount.clear(); + } +} + +void OverlayDB::rollback() +{ + m_over.clear(); + m_refCount.clear(); +} + +std::string OverlayDB::lookup(h256 _h) const +{ + std::string ret = MemoryDB::lookup(_h); + if (ret.empty() && m_db) + m_db->Get(m_readOptions, ldb::Slice((char const*)_h.data(), 32), &ret); + return ret; +} + +bool OverlayDB::exists(h256 _h) const +{ + if (MemoryDB::exists(_h)) + return true; + std::string ret; + if (m_db) + m_db->Get(m_readOptions, ldb::Slice((char const*)_h.data(), 32), &ret); + return !ret.empty(); +} + +void OverlayDB::kill(h256 _h) +{ +#if ETH_PARANOIA + if (!MemoryDB::kill(_h)) + { + std::string ret; + if (m_db) + m_db->Get(m_readOptions, ldb::Slice((char const*)_h.data(), 32), &ret); + if (ret.empty()) + cnote << "Decreasing DB node ref count below zero with no DB node. Probably have a corrupt Trie." << _h.abridged(); + } +#else + MemoryDB::kill(_h); +#endif +} + +} diff --git a/libethsupport/OverlayDB.h b/libethsupport/OverlayDB.h new file mode 100644 index 000000000..9a5653360 --- /dev/null +++ b/libethsupport/OverlayDB.h @@ -0,0 +1,58 @@ +/* + This file is part of cpp-ethereum. + + cpp-ethereum is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + cpp-ethereum is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with cpp-ethereum. If not, see . +*/ +/** @file MemoryDB.h + * @author Gav Wood + * @date 2014 + */ + +#pragma once + +#include +#include "Common.h" +#include "MemoryDB.h" +#include "Log.h" +namespace ldb = leveldb; + +namespace eth +{ + +class OverlayDB: public MemoryDB +{ +public: + OverlayDB(ldb::DB* _db = nullptr): m_db(_db) {} + ~OverlayDB(); + + ldb::DB* db() const { return m_db.get(); } + void setDB(ldb::DB* _db, bool _clearOverlay = true); + + void commit(); + void rollback(); + + std::string lookup(h256 _h) const; + bool exists(h256 _h) const; + void kill(h256 _h); + +private: + using MemoryDB::clear; + + std::shared_ptr m_db; + + ldb::ReadOptions m_readOptions; + ldb::WriteOptions m_writeOptions; +}; + +} diff --git a/libethsupport/TrieDB.cpp b/libethsupport/TrieDB.cpp index 0c9ca6206..7160fe5b5 100644 --- a/libethsupport/TrieDB.cpp +++ b/libethsupport/TrieDB.cpp @@ -29,158 +29,4 @@ namespace eth const h256 c_shaNull = sha3(rlp("")); -std::map BasicMap::get() const -{ - if (!m_enforceRefs) - return m_over; - std::map ret; - for (auto const& i: m_refCount) - if (i.second) - ret.insert(*m_over.find(i.first)); - return ret; -} - -std::string BasicMap::lookup(h256 _h) const -{ - auto it = m_over.find(_h); - if (it != m_over.end()) - { - if (!m_enforceRefs || (m_refCount.count(it->first) && m_refCount.at(it->first))) - return it->second; -// else if (m_enforceRefs && m_refCount.count(it->first) && !m_refCount.at(it->first)) -// cnote << "Lookup required for value with no refs. Let's hope it's in the DB." << _h.abridged(); - } - return std::string(); -} - -bool BasicMap::exists(h256 _h) const -{ - auto it = m_over.find(_h); - if (it != m_over.end() && (!m_enforceRefs || (m_refCount.count(it->first) && m_refCount.at(it->first)))) - return true; - return false; -} - -void BasicMap::insert(h256 _h, bytesConstRef _v) -{ - m_over[_h] = _v.toString(); - m_refCount[_h]++; -#if ETH_PARANOIA - tdebug << "INST" << _h.abridged() << "=>" << m_refCount[_h]; -#endif -} - -bool BasicMap::kill(h256 _h) -{ - if (m_refCount.count(_h)) - { - if (m_refCount[_h] > 0) - --m_refCount[_h]; -#if ETH_PARANOIA - else - { - // If we get to this point, then there was probably a node in the level DB which we need to remove and which we have previously - // used as part of the memory-based BasicMap. Nothing to be worried about *as long as the node exists in the DB*. - tdebug << "NOKILL-WAS" << _h.abridged(); - return false; - } - tdebug << "KILL" << _h.abridged() << "=>" << m_refCount[_h]; - return true; - } - else - { - tdebug << "NOKILL" << _h.abridged(); - return false; - } -#else - return true; - } -#endif -} - -void BasicMap::purge() -{ - for (auto const& i: m_refCount) - if (!i.second) - m_over.erase(i.first); -} - -set BasicMap::keys() const -{ - set ret; - for (auto const& i: m_refCount) - if (i.second) - ret.insert(i.first); - return ret; -} - -Overlay::~Overlay() -{ - if (m_db.use_count() == 1 && m_db.get()) - cnote << "Closing state DB"; -} - -void Overlay::setDB(ldb::DB* _db, bool _clearOverlay) -{ - m_db = std::shared_ptr(_db); - if (_clearOverlay) - m_over.clear(); -} - -void Overlay::commit() -{ - if (m_db) - { -// cnote << "Committing nodes to disk DB:"; - for (auto const& i: m_over) - { -// cnote << i.first << "#" << m_refCount[i.first]; - if (m_refCount[i.first]) - m_db->Put(m_writeOptions, ldb::Slice((char const*)i.first.data(), i.first.size), ldb::Slice(i.second.data(), i.second.size())); - } - m_over.clear(); - m_refCount.clear(); - } -} - -void Overlay::rollback() -{ - m_over.clear(); - m_refCount.clear(); -} - -std::string Overlay::lookup(h256 _h) const -{ - std::string ret = BasicMap::lookup(_h); - if (ret.empty() && m_db) - m_db->Get(m_readOptions, ldb::Slice((char const*)_h.data(), 32), &ret); - return ret; -} - -bool Overlay::exists(h256 _h) const -{ - if (BasicMap::exists(_h)) - return true; - std::string ret; - if (m_db) - m_db->Get(m_readOptions, ldb::Slice((char const*)_h.data(), 32), &ret); - return !ret.empty(); -} - -void Overlay::kill(h256 _h) -{ -#if ETH_PARANOIA - if (!BasicMap::kill(_h)) - { - std::string ret; - if (m_db) - m_db->Get(m_readOptions, ldb::Slice((char const*)_h.data(), 32), &ret); - if (ret.empty()) - cnote << "Decreasing DB node ref count below zero with no DB node. Probably have a corrupt Trie." << _h.abridged(); - } -#else - BasicMap::kill(_h); -#endif -} - } diff --git a/libethsupport/TrieDB.h b/libethsupport/TrieDB.h index 8b430ea31..58cbcf039 100644 --- a/libethsupport/TrieDB.h +++ b/libethsupport/TrieDB.h @@ -24,6 +24,8 @@ #include #include #include "Common.h" +#include "MemoryDB.h" +#include "OverlayDB.h" #include "Log.h" #include "TrieCommon.h" namespace ldb = leveldb; @@ -32,85 +34,9 @@ namespace eth { struct TrieDBChannel: public LogChannel { static const char* name() { return "-T-"; } static const int verbosity = 6; }; - #define tdebug clog(TrieDBChannel) -class BasicMap -{ - friend class EnforceRefs; - -public: - BasicMap() {} - - void clear() { m_over.clear(); } - std::map get() const; - - std::string lookup(h256 _h) const; - bool exists(h256 _h) const; - void insert(h256 _h, bytesConstRef _v); - bool kill(h256 _h); - void purge(); - - std::set keys() const; - -protected: - std::map m_over; - std::map m_refCount; - - mutable bool m_enforceRefs = false; -}; - -inline std::ostream& operator<<(std::ostream& _out, BasicMap const& _m) -{ - for (auto i: _m.get()) - { - _out << i.first << ": "; - _out << RLP(i.second); - _out << " " << toHex(i.second); - _out << std::endl; - } - return _out; -} - class InvalidTrie: public std::exception {}; -class EnforceRefs; - -class Overlay: public BasicMap -{ -public: - Overlay(ldb::DB* _db = nullptr): m_db(_db) {} - ~Overlay(); - - ldb::DB* db() const { return m_db.get(); } - void setDB(ldb::DB* _db, bool _clearOverlay = true); - - void commit(); - void rollback(); - - std::string lookup(h256 _h) const; - bool exists(h256 _h) const; - void kill(h256 _h); - -private: - using BasicMap::clear; - - std::shared_ptr m_db; - - ldb::ReadOptions m_readOptions; - ldb::WriteOptions m_writeOptions; -}; - -class EnforceRefs -{ -public: - EnforceRefs(BasicMap const& _o, bool _r): m_o(_o), m_r(_o.m_enforceRefs) { _o.m_enforceRefs = _r; } - ~EnforceRefs() { m_o.m_enforceRefs = m_r; } - -private: - BasicMap const& m_o; - bool m_r; -}; - extern const h256 c_shaNull; /** @@ -233,18 +159,9 @@ public: using value_type = std::pair; iterator() {} - iterator(GenericTrieDB const* _db) - { - m_that = _db; - m_trail.push_back({_db->node(_db->m_root), std::string(1, '\0'), 255}); // one null byte is the HPE for the empty key. - next(); - } + iterator(GenericTrieDB const* _db); - iterator& operator++() - { - next(); - return *this; - } + iterator& operator++() { next(); return *this; } value_type operator*() const { return at(); } value_type operator->() const { return at(); } @@ -252,119 +169,10 @@ public: bool operator==(iterator const& _c) const { return _c.m_trail == m_trail; } bool operator!=(iterator const& _c) const { return _c.m_trail != m_trail; } - value_type at() const - { - assert(m_trail.size()); - Node const& b = m_trail.back(); - assert(b.key.size()); - assert(!(b.key[0] & 0x10)); // should be an integer number of bytes (i.e. not an odd number of nibbles). - - RLP rlp(b.rlp); - if (rlp.itemCount() == 2) - return std::make_pair(bytesConstRef(b.key).cropped(1), rlp[1].payload()); - else - return std::make_pair(bytesConstRef(b.key).cropped(1), rlp[16].payload()); - } + value_type at() const; private: - void next() - { - while (true) - { - if (m_trail.empty()) - { - m_that = nullptr; - return; - } - - Node const& b = m_trail.back(); - RLP rlp(b.rlp); - - if (m_trail.back().child == 255) - { - // Entering. Look for first... - if (rlp.isEmpty()) - { - m_trail.pop_back(); - continue; - } - if (!(rlp.isList() && (rlp.itemCount() == 2 || rlp.itemCount() == 17))) - { -#if ETH_PARANOIA - cwarn << "BIG FAT ERROR. STATE TRIE CORRUPTED!!!!!"; - cwarn << b.rlp.size() << toHex(b.rlp); - cwarn << rlp; - auto c = rlp.itemCount(); - cwarn << c; - throw InvalidTrie(); -#else - m_that = nullptr; - return; -#endif - } - if (rlp.itemCount() == 2) - { - // Just turn it into a valid Branch - m_trail.back().key = hexPrefixEncode(keyOf(m_trail.back().key), keyOf(rlp), false); - if (isLeaf(rlp)) - { - // leaf - exit now. - m_trail.back().child = 0; - return; - } - - // enter child. - m_trail.back().rlp = m_that->deref(rlp[1]); - // no need to set .child as 255 - it's already done. - continue; - } - else - { - // Already a branch - look for first valid. - m_trail.back().setFirstChild(); - // run through to... - } - } - else - { - // Continuing/exiting. Look for next... - if (!(rlp.isList() && rlp.itemCount() == 17)) - { - m_trail.pop_back(); - continue; - } - // else run through to... - m_trail.back().incrementChild(); - } - - // ...here. should only get here if we're a list. - assert(rlp.isList() && rlp.itemCount() == 17); - for (;; m_trail.back().incrementChild()) - if (m_trail.back().child == 17) - { - // finished here. - m_trail.pop_back(); - break; - } - else if (!rlp[m_trail.back().child].isEmpty()) - { - if (m_trail.back().child == 16) - return; // have a value at this node - exit now. - else - { - // lead-on to another node - enter child. - // fixed so that Node passed into push_back is constructed *before* m_trail is potentially resized (which invalidates back and rlp) - Node const& back = m_trail.back(); - m_trail.push_back(Node{ - m_that->deref(rlp[back.child]), - hexPrefixEncode(keyOf(back.key), NibbleSlice(bytesConstRef(&back.child, 1), 1), false), - 255 - }); - break; - } - } - } - } + void next(); struct Node { @@ -482,15 +290,7 @@ public: value_type operator*() const { return at(); } value_type operator->() const { return at(); } - value_type at() const - { - auto p = Super::at(); - value_type ret; - assert(p.first.size() == sizeof(KeyType)); - memcpy(&ret.first, p.first.data(), sizeof(KeyType)); - ret.second = p.second; - return ret; - } + value_type at() const; }; iterator begin() const { return this; } @@ -511,6 +311,136 @@ std::ostream& operator<<(std::ostream& _out, TrieDB const& _db) namespace eth { +template GenericTrieDB::iterator::iterator(GenericTrieDB const* _db) +{ + m_that = _db; + m_trail.push_back({_db->node(_db->m_root), std::string(1, '\0'), 255}); // one null byte is the HPE for the empty key. + next(); +} + +template typename GenericTrieDB::iterator::value_type GenericTrieDB::iterator::at() const +{ + assert(m_trail.size()); + Node const& b = m_trail.back(); + assert(b.key.size()); + assert(!(b.key[0] & 0x10)); // should be an integer number of bytes (i.e. not an odd number of nibbles). + + RLP rlp(b.rlp); + if (rlp.itemCount() == 2) + return std::make_pair(bytesConstRef(b.key).cropped(1), rlp[1].payload()); + else + return std::make_pair(bytesConstRef(b.key).cropped(1), rlp[16].payload()); +} + +template void GenericTrieDB::iterator::next() +{ + while (true) + { + if (m_trail.empty()) + { + m_that = nullptr; + return; + } + + Node const& b = m_trail.back(); + RLP rlp(b.rlp); + + if (m_trail.back().child == 255) + { + // Entering. Look for first... + if (rlp.isEmpty()) + { + m_trail.pop_back(); + continue; + } + if (!(rlp.isList() && (rlp.itemCount() == 2 || rlp.itemCount() == 17))) + { +#if ETH_PARANOIA + cwarn << "BIG FAT ERROR. STATE TRIE CORRUPTED!!!!!"; + cwarn << b.rlp.size() << toHex(b.rlp); + cwarn << rlp; + auto c = rlp.itemCount(); + cwarn << c; + throw InvalidTrie(); +#else + m_that = nullptr; + return; +#endif + } + if (rlp.itemCount() == 2) + { + // Just turn it into a valid Branch + m_trail.back().key = hexPrefixEncode(keyOf(m_trail.back().key), keyOf(rlp), false); + if (isLeaf(rlp)) + { + // leaf - exit now. + m_trail.back().child = 0; + return; + } + + // enter child. + m_trail.back().rlp = m_that->deref(rlp[1]); + // no need to set .child as 255 - it's already done. + continue; + } + else + { + // Already a branch - look for first valid. + m_trail.back().setFirstChild(); + // run through to... + } + } + else + { + // Continuing/exiting. Look for next... + if (!(rlp.isList() && rlp.itemCount() == 17)) + { + m_trail.pop_back(); + continue; + } + // else run through to... + m_trail.back().incrementChild(); + } + + // ...here. should only get here if we're a list. + assert(rlp.isList() && rlp.itemCount() == 17); + for (;; m_trail.back().incrementChild()) + if (m_trail.back().child == 17) + { + // finished here. + m_trail.pop_back(); + break; + } + else if (!rlp[m_trail.back().child].isEmpty()) + { + if (m_trail.back().child == 16) + return; // have a value at this node - exit now. + else + { + // lead-on to another node - enter child. + // fixed so that Node passed into push_back is constructed *before* m_trail is potentially resized (which invalidates back and rlp) + Node const& back = m_trail.back(); + m_trail.push_back(Node{ + m_that->deref(rlp[back.child]), + hexPrefixEncode(keyOf(back.key), NibbleSlice(bytesConstRef(&back.child, 1), 1), false), + 255 + }); + break; + } + } + } +} + +template typename TrieDB::iterator::value_type TrieDB::iterator::at() const +{ + auto p = Super::at(); + value_type ret; + assert(p.first.size() == sizeof(KeyType)); + memcpy(&ret.first, p.first.data(), sizeof(KeyType)); + ret.second = p.second; + return ret; +} + template void GenericTrieDB::init() { m_root = insertNode(&RLPNull); diff --git a/test/state.cpp b/test/state.cpp index 262bf6762..92b274ecc 100644 --- a/test/state.cpp +++ b/test/state.cpp @@ -38,7 +38,7 @@ int stateTest() Defaults::setDBPath(boost::filesystem::temp_directory_path().string()); - Overlay stateDB = State::openDB(); + OverlayDB stateDB = State::openDB(); BlockChain bc; State s(myMiner.address(), stateDB); diff --git a/test/trie.cpp b/test/trie.cpp index 59823f20b..defa87879 100644 --- a/test/trie.cpp +++ b/test/trie.cpp @@ -63,8 +63,8 @@ BOOST_AUTO_TEST_CASE(trie_tests) for (unsigned j = 0; j < eth::test::fac((unsigned)ss.size()); ++j) { next_permutation(ss.begin(), ss.end()); - BasicMap m; - GenericTrieDB t(&m); + MemoryDB m; + GenericTrieDB t(&m); t.init(); BOOST_REQUIRE(t.check(true)); for (auto const& k: ss) @@ -91,8 +91,8 @@ BOOST_AUTO_TEST_CASE(moreTrieTests) #if 0 // More tests... { - BasicMap m; - GenericTrieDB t(&m); + MemoryDB m; + GenericTrieDB t(&m); t.init(); // initialise as empty tree. cout << t; cout << m; @@ -125,8 +125,8 @@ BOOST_AUTO_TEST_CASE(moreTrieTests) cout << hash256(StringMap()) << endl; } { - BasicMap m; - GenericTrieDB t(&m); + MemoryDB m; + GenericTrieDB t(&m); t.init(); // initialise as empty tree. t.insert(string("a"), string("A")); t.insert(string("b"), string("B")); @@ -160,8 +160,8 @@ BOOST_AUTO_TEST_CASE(moreTrieTests) } #endif { - BasicMap m; - GenericTrieDB d(&m); + MemoryDB m; + GenericTrieDB d(&m); d.init(); // initialise as empty tree. MemTrie t; StringMap s; @@ -233,10 +233,10 @@ BOOST_AUTO_TEST_CASE(trieStess) { cnote << "Stress-testing Trie..."; { - BasicMap m; - BasicMap dm; + MemoryDB m; + MemoryDB dm; EnforceRefs e(dm, true); - GenericTrieDB d(&dm); + GenericTrieDB d(&dm); d.init(); // initialise as empty tree. MemTrie t; BOOST_REQUIRE(d.check(true)); @@ -267,9 +267,9 @@ BOOST_AUTO_TEST_CASE(trieStess) for (auto i: d) cwarn << i.first.toString() << i.second.toString(); - BasicMap dm2; + MemoryDB dm2; EnforceRefs e2(dm2, true); - GenericTrieDB d2(&dm2); + GenericTrieDB d2(&dm2); d2.init(); // initialise as empty tree. for (auto i: d) d2.insert(i.first, i.second);