diff --git a/libdevcore/FixedHash.h b/libdevcore/FixedHash.h index 2b4e6bc08..24fac9c0a 100644 --- a/libdevcore/FixedHash.h +++ b/libdevcore/FixedHash.h @@ -53,7 +53,7 @@ public: enum ConstructFromStringType { FromHex, FromBinary }; /// Method to convert from a string. - enum ConstructFromHashType { AlignLeft, AlignRight }; + enum ConstructFromHashType { AlignLeft, AlignRight, FailIfDifferent }; /// Construct an empty hash. FixedHash() { m_data.fill(0); } @@ -65,10 +65,10 @@ public: FixedHash(Arith const& _arith) { toBigEndian(_arith, m_data); } /// Explicitly construct, copying from a byte array. - explicit FixedHash(bytes const& _b) { if (_b.size() == N) memcpy(m_data.data(), _b.data(), std::min(_b.size(), N)); } + explicit FixedHash(bytes const& _b, ConstructFromHashType _t = FailIfDifferent) { if (_b.size() == N) memcpy(m_data.data(), _b.data(), std::min(_b.size(), N)); else if (_t != FailIfDifferent) { m_data.fill(0); auto c = std::min(_b.size(), N); for (unsigned i = 0; i < c; ++i) m_data[_t == AlignRight ? N - 1 - i : i] = _b[_t == AlignRight ? _b.size() - 1 - i : i]; } } /// Explicitly construct, copying from a byte array. - explicit FixedHash(bytesConstRef _b) { if (_b.size() == N) memcpy(m_data.data(), _b.data(), std::min(_b.size(), N)); } + explicit FixedHash(bytesConstRef _b, ConstructFromHashType _t = FailIfDifferent) { if (_b.size() == N) memcpy(m_data.data(), _b.data(), std::min(_b.size(), N)); else if (_t != FailIfDifferent) { m_data.fill(0); auto c = std::min(_b.size(), N); for (unsigned i = 0; i < c; ++i) m_data[_t == AlignRight ? N - 1 - i : i] = _b[_t == AlignRight ? _b.size() - 1 - i : i]; } } /// Explicitly construct, copying from a bytes in memory with given pointer. explicit FixedHash(byte const* _bs, ConstructFromPointerType) { memcpy(m_data.data(), _bs, N); } diff --git a/libdevcrypto/MemoryDB.cpp b/libdevcrypto/MemoryDB.cpp index 4fc4ed9ad..56b77ddee 100644 --- a/libdevcrypto/MemoryDB.cpp +++ b/libdevcrypto/MemoryDB.cpp @@ -107,7 +107,7 @@ set MemoryDB::keys() const { set ret; for (auto const& i: m_refCount) - if (i.second) + if (i.second && h128(i.first.ref().cropped(0, 16))) ret.insert(i.first); return ret; } diff --git a/libdevcrypto/MemoryDB.h b/libdevcrypto/MemoryDB.h index 4b8d3b3a2..816e393a3 100644 --- a/libdevcrypto/MemoryDB.h +++ b/libdevcrypto/MemoryDB.h @@ -26,6 +26,7 @@ #include #include #include +#include "SHA3.h" namespace dev { @@ -50,6 +51,9 @@ public: bool kill(h256 _h); void purge(); + bytes lookupAux(h256 _h) const { return asBytes(lookup(h256(sha3(_h).ref().cropped(16), h256::AlignRight))); } + void insertAux(h256 _h, bytesConstRef _v) { return insert(h256(sha3(_h).ref().cropped(16), h256::AlignRight), _v); } + std::set keys() const; protected: diff --git a/libdevcrypto/TrieDB.h b/libdevcrypto/TrieDB.h index c08199e06..676193112 100644 --- a/libdevcrypto/TrieDB.h +++ b/libdevcrypto/TrieDB.h @@ -62,17 +62,21 @@ extern const h256 EmptyTrie; * assert(t.isEmpty()); * @endcode */ -template +template class GenericTrieDB { public: + using DB = _DB; + GenericTrieDB(DB* _db): m_db(_db) {} GenericTrieDB(DB* _db, h256 _root) { open(_db, _root); } ~GenericTrieDB() {} + void open(DB* _db) { m_db = _db; } void open(DB* _db, h256 _root) { m_db = _db; setRoot(_root); } - void init(); + void init() { setRoot(insertNode(&RLPNull)); assert(node(m_root).size()); } + void setRoot(h256 _root) { m_root = _root; @@ -83,7 +87,6 @@ public: if (!node(m_root).size()) BOOST_THROW_EXCEPTION(RootNotFound()); } - bool haveRoot(h256 _root, bool _enforceRefs = true) { return _root == c_shaNull ? true : m_db->lookup(_root, _enforceRefs).size(); } /// True if the trie is uninitialised (i.e. that the DB doesn't contain the root node). bool isNull() const { return !node(m_root).size(); } @@ -211,6 +214,9 @@ public: iterator lower_bound(bytesConstRef _key) const { return iterator(this, _key); } +protected: + DB* db() const { return m_db; } + private: RLPStream& streamNode(RLPStream& _s, bytes const& _b); @@ -281,30 +287,33 @@ std::ostream& operator<<(std::ostream& _out, GenericTrieDB const& _db) return _out; } -template -class TrieDB: public GenericTrieDB +template +class SpecificTrieDB: public Generic { public: - TrieDB(DB* _db): GenericTrieDB(_db) {} - TrieDB(DB* _db, h256 _root): GenericTrieDB(_db, _root) {} + using DB = typename Generic::DB; + using KeyType = _KeyType; + + SpecificTrieDB(DB* _db): Generic(_db) {} + SpecificTrieDB(DB* _db, h256 _root): Generic(_db, _root) {} std::string operator[](KeyType _k) const { return at(_k); } - bool contains(KeyType _k) const { return GenericTrieDB::contains(bytesConstRef((byte const*)&_k, sizeof(KeyType))); } - std::string at(KeyType _k) const { return GenericTrieDB::at(bytesConstRef((byte const*)&_k, sizeof(KeyType))); } - void insert(KeyType _k, bytesConstRef _value) { GenericTrieDB::insert(bytesConstRef((byte const*)&_k, sizeof(KeyType)), _value); } + bool contains(KeyType _k) const { return Generic::contains(bytesConstRef((byte const*)&_k, sizeof(KeyType))); } + std::string at(KeyType _k) const { return Generic::at(bytesConstRef((byte const*)&_k, sizeof(KeyType))); } + void insert(KeyType _k, bytesConstRef _value) { Generic::insert(bytesConstRef((byte const*)&_k, sizeof(KeyType)), _value); } void insert(KeyType _k, bytes const& _value) { insert(_k, bytesConstRef(&_value)); } - void remove(KeyType _k) { GenericTrieDB::remove(bytesConstRef((byte const*)&_k, sizeof(KeyType))); } + void remove(KeyType _k) { Generic::remove(bytesConstRef((byte const*)&_k, sizeof(KeyType))); } - class iterator: public GenericTrieDB::iterator + class iterator: public Generic::iterator { public: - using Super = typename GenericTrieDB::iterator; + using Super = typename Generic::iterator; using value_type = std::pair; iterator() {} - iterator(TrieDB const* _db): Super(_db) {} - iterator(TrieDB const* _db, bytesConstRef _k): Super(_db, _k) {} + iterator(Generic const* _db): Super(_db) {} + iterator(Generic const* _db, bytesConstRef _k): Super(_db, _k) {} value_type operator*() const { return at(); } value_type operator->() const { return at(); } @@ -317,27 +326,28 @@ public: iterator lower_bound(KeyType _k) const { return iterator(this, bytesConstRef((byte const*)&_k, sizeof(KeyType))); } }; -template -std::ostream& operator<<(std::ostream& _out, TrieDB const& _db) +template +std::ostream& operator<<(std::ostream& _out, SpecificTrieDB const& _db) { for (auto const& i: _db) _out << i.first << ": " << escaped(i.second.toString(), false) << std::endl; return _out; } -template -class SecureGenericTrieDB: private TrieDB +template +class HashedGenericTrieDB: private SpecificTrieDB, h256> { - using Super = TrieDB; + using Super = SpecificTrieDB, h256>; public: - SecureGenericTrieDB(DB* _db): Super(_db) {} - SecureGenericTrieDB(DB* _db, h256 _root): Super(_db, _root) {} + using DB = _DB; + + HashedGenericTrieDB(DB* _db): Super(_db) {} + HashedGenericTrieDB(DB* _db, h256 _root): Super(_db, _root) {} using Super::open; using Super::init; using Super::setRoot; - using Super::haveRoot; /// True if the trie is uninitialised (i.e. that the DB doesn't contain the root node). using Super::isNull; @@ -350,29 +360,63 @@ public: using Super::check; std::string at(bytesConstRef _key) const { return Super::at(sha3(_key)); } + bool contains(bytesConstRef _key) { return Super::contains(sha3(_key)); } void insert(bytesConstRef _key, bytesConstRef _value) { Super::insert(sha3(_key), _value); } void remove(bytesConstRef _key) { Super::remove(sha3(_key)); } - bool contains(bytesConstRef _key) { return Super::contains(sha3(_key)); } + + // empty from the PoV of the iterator interface. + using iterator = void*; + iterator begin() const { return nullptr; } + iterator end() const { return nullptr; } + iterator lower_bound(bytesConstRef) const { return end(); } }; -template -class SecureTrieDB: public SecureGenericTrieDB +// Hashed & Basic +template +class FatGenericTrieDB: public GenericTrieDB { - using Super = SecureGenericTrieDB; + using Super = GenericTrieDB; public: - SecureTrieDB(DB* _db): Super(_db) {} - SecureTrieDB(DB* _db, h256 _root): Super(_db, _root) {} + FatGenericTrieDB(DB* _db): Super(_db), m_secure(_db) {} + FatGenericTrieDB(DB* _db, h256 _root) { open(_db, _root); } - std::string operator[](KeyType _k) const { return at(_k); } + void open(DB* _db, h256 _root) { Super::open(_db); m_secure.open(_db); setRoot(_root); } - bool contains(KeyType _k) const { return Super::contains(bytesConstRef((byte const*)&_k, sizeof(KeyType))); } - std::string at(KeyType _k) const { return Super::at(bytesConstRef((byte const*)&_k, sizeof(KeyType))); } - void insert(KeyType _k, bytesConstRef _value) { Super::insert(bytesConstRef((byte const*)&_k, sizeof(KeyType)), _value); } - void insert(KeyType _k, bytes const& _value) { insert(_k, bytesConstRef(&_value)); } - void remove(KeyType _k) { Super::remove(bytesConstRef((byte const*)&_k, sizeof(KeyType))); } + void init() { Super::init(); m_secure.init(); syncRoot(); } + + void setRoot(h256 _root) + { + m_secure.setRoot(_root); + Super::setRoot(h256(Super::db()->lookupAux(m_secure.root()))); + } + + h256 root() const { return m_secure.root(); } + + void insert(bytesConstRef _key, bytesConstRef _value) { Super::insert(_key, _value); m_secure.insert(_key, _value); syncRoot(); } + void remove(bytesConstRef _key) { Super::remove(_key); m_secure.remove(_key); syncRoot(); } + + std::set leftOvers(std::ostream* = nullptr) const { return {}; } + bool check(bool) const { return m_secure.check(false) && Super::check(false); } + +private: + void syncRoot() + { + // Root changed. Need to record the mapping so we can determine on setRoot. + Super::db()->insertAux(m_secure.root(), Super::root().ref()); + } + + HashedGenericTrieDB m_secure; }; +template using TrieDB = SpecificTrieDB, KeyType>; + +#if ETH_FAT_DB +template using SecureTrieDB = SpecificTrieDB, KeyType>; +#else +template using SecureTrieDB = SpecificTrieDB, KeyType>; +#endif + } // Template implementations... @@ -641,7 +685,7 @@ template void GenericTrieDB::iterator::next() } } -template typename TrieDB::iterator::value_type TrieDB::iterator::at() const +template typename SpecificTrieDB::iterator::value_type SpecificTrieDB::iterator::at() const { auto p = Super::at(); value_type ret; @@ -651,13 +695,6 @@ template typename TrieDB::iterator::value return ret; } -template void GenericTrieDB::init() -{ - m_root = insertNode(&RLPNull); -// std::cout << "Initialised root to " << m_root << std::endl; - assert(node(m_root).size()); -} - template void GenericTrieDB::insert(bytesConstRef _key, bytesConstRef _value) { #if ETH_PARANOIA diff --git a/test/trie.cpp b/test/trie.cpp index 39a3a59a5..4b676fb9a 100644 --- a/test/trie.cpp +++ b/test/trie.cpp @@ -48,6 +48,8 @@ static unsigned fac(unsigned _i) } } +using dev::operator <<; + BOOST_AUTO_TEST_SUITE(TrieTests) BOOST_AUTO_TEST_CASE(trie_test_anyorder) @@ -79,15 +81,35 @@ BOOST_AUTO_TEST_CASE(trie_test_anyorder) next_permutation(ss.begin(), ss.end()); MemoryDB m; GenericTrieDB t(&m); + MemoryDB hm; + HashedGenericTrieDB ht(&hm); + MemoryDB fm; + FatGenericTrieDB ft(&fm); t.init(); + ht.init(); + ft.init(); BOOST_REQUIRE(t.check(true)); + BOOST_REQUIRE(ht.check(true)); + BOOST_REQUIRE(ft.check(true)); for (auto const& k: ss) { t.insert(k.first, k.second); + ht.insert(k.first, k.second); + ft.insert(k.first, k.second); BOOST_REQUIRE(t.check(true)); + BOOST_REQUIRE(ht.check(true)); + BOOST_REQUIRE(ft.check(true)); + for (auto i = ft.begin(), j = t.begin(); i != ft.end() && j != t.end(); ++i, ++j) + { + BOOST_CHECK_EQUAL(i == ft.end(), j == t.end()); + BOOST_REQUIRE((*i).first.toBytes() == (*j).first.toBytes()); + BOOST_REQUIRE((*i).second.toBytes() == (*j).second.toBytes()); + } + BOOST_CHECK_EQUAL(ht.root(), ft.root()); } BOOST_REQUIRE(!o["root"].is_null()); BOOST_CHECK_EQUAL(o["root"].get_str(), "0x" + toHex(t.root().asArray())); + BOOST_CHECK_EQUAL(ht.root(), ft.root()); } } } @@ -139,15 +161,33 @@ BOOST_AUTO_TEST_CASE(trie_tests_ordered) MemoryDB m; GenericTrieDB t(&m); + MemoryDB hm; + HashedGenericTrieDB ht(&hm); + MemoryDB fm; + FatGenericTrieDB ft(&fm); t.init(); + ht.init(); + ft.init(); BOOST_REQUIRE(t.check(true)); + BOOST_REQUIRE(ht.check(true)); + BOOST_REQUIRE(ft.check(true)); + for (auto const& k: ss) { if (find(keysToBeDeleted.begin(), keysToBeDeleted.end(), k.first) != keysToBeDeleted.end() && k.second.empty()) - t.remove(k.first); + t.remove(k.first), ht.remove(k.first), ft.remove(k.first); else - t.insert(k.first, k.second); + t.insert(k.first, k.second), ht.insert(k.first, k.second), ft.insert(k.first, k.second); BOOST_REQUIRE(t.check(true)); + BOOST_REQUIRE(ht.check(true)); + BOOST_REQUIRE(ft.check(true)); + for (auto i = ft.begin(), j = t.begin(); i != ft.end() && j != t.end(); ++i, ++j) + { + BOOST_CHECK_EQUAL(i == ft.end(), j == t.end()); + BOOST_REQUIRE((*i).first.toBytes() == (*j).first.toBytes()); + BOOST_REQUIRE((*i).second.toBytes() == (*j).second.toBytes()); + } + BOOST_CHECK_EQUAL(ht.root(), ft.root()); } BOOST_REQUIRE(!o["root"].is_null());