lukechilds
/
ethminer
mirror of https://github.com/lukechilds/ethminer.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Fat Trie and tests for it.

cl-refactor
Gav Wood 10 years ago
parent
bb0f5358bf
commit
c88590f0b8
5 changed files with 130 additions and 49 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      libdevcore/FixedHash.h
  2. 2
      libdevcrypto/MemoryDB.cpp
  3. 4
      libdevcrypto/MemoryDB.h
  4. 123
      libdevcrypto/TrieDB.h
  5. 44
      test/trie.cpp

6
libdevcore/FixedHash.h
View File

@ -53,7 +53,7 @@ public:
enum ConstructFromStringType { FromHex, FromBinary }; enum ConstructFromStringType { FromHex, FromBinary };
/// Method to convert from a string. /// Method to convert from a string.
enum ConstructFromHashType { AlignLeft, AlignRight }; enum ConstructFromHashType { AlignLeft, AlignRight, FailIfDifferent };
/// Construct an empty hash. /// Construct an empty hash.
FixedHash() { m_data.fill(0); } FixedHash() { m_data.fill(0); }
@ -65,10 +65,10 @@ public:
FixedHash(Arith const& _arith) { toBigEndian(_arith, m_data); } FixedHash(Arith const& _arith) { toBigEndian(_arith, m_data); }
/// Explicitly construct, copying from a byte array. /// Explicitly construct, copying from a byte array.
explicit FixedHash(bytes const& _b) { if (_b.size() == N) memcpy(m_data.data(), _b.data(), std::min<unsigned>(_b.size(), N)); } explicit FixedHash(bytes const& _b, ConstructFromHashType _t = FailIfDifferent) { if (_b.size() == N) memcpy(m_data.data(), _b.data(), std::min<unsigned>(_b.size(), N)); else if (_t != FailIfDifferent) { m_data.fill(0); auto c = std::min<unsigned>(_b.size(), N); for (unsigned i = 0; i < c; ++i) m_data[_t == AlignRight ? N - 1 - i : i] = _b[_t == AlignRight ? _b.size() - 1 - i : i]; } }
/// Explicitly construct, copying from a byte array. /// Explicitly construct, copying from a byte array.
explicit FixedHash(bytesConstRef _b) { if (_b.size() == N) memcpy(m_data.data(), _b.data(), std::min<unsigned>(_b.size(), N)); } explicit FixedHash(bytesConstRef _b, ConstructFromHashType _t = FailIfDifferent) { if (_b.size() == N) memcpy(m_data.data(), _b.data(), std::min<unsigned>(_b.size(), N)); else if (_t != FailIfDifferent) { m_data.fill(0); auto c = std::min<unsigned>(_b.size(), N); for (unsigned i = 0; i < c; ++i) m_data[_t == AlignRight ? N - 1 - i : i] = _b[_t == AlignRight ? _b.size() - 1 - i : i]; } }
/// Explicitly construct, copying from a bytes in memory with given pointer. /// Explicitly construct, copying from a bytes in memory with given pointer.
explicit FixedHash(byte const* _bs, ConstructFromPointerType) { memcpy(m_data.data(), _bs, N); } explicit FixedHash(byte const* _bs, ConstructFromPointerType) { memcpy(m_data.data(), _bs, N); }

2
libdevcrypto/MemoryDB.cpp
View File

@ -107,7 +107,7 @@ set<h256> MemoryDB::keys() const
{ {
set<h256> ret; set<h256> ret;
for (auto const& i: m_refCount) for (auto const& i: m_refCount)
if (i.second) if (i.second && h128(i.first.ref().cropped(0, 16)))
ret.insert(i.first); ret.insert(i.first);
return ret; return ret;
} }

4
libdevcrypto/MemoryDB.h
View File

@ -26,6 +26,7 @@
#include <libdevcore/FixedHash.h> #include <libdevcore/FixedHash.h>
#include <libdevcore/Log.h> #include <libdevcore/Log.h>
#include <libdevcore/RLP.h> #include <libdevcore/RLP.h>
#include "SHA3.h"
namespace dev namespace dev
{ {
@ -50,6 +51,9 @@ public:
bool kill(h256 _h); bool kill(h256 _h);
void purge(); void purge();
bytes lookupAux(h256 _h) const { return asBytes(lookup(h256(sha3(_h).ref().cropped(16), h256::AlignRight))); }
void insertAux(h256 _h, bytesConstRef _v) { return insert(h256(sha3(_h).ref().cropped(16), h256::AlignRight), _v); }
std::set<h256> keys() const; std::set<h256> keys() const;
protected: protected:

123
libdevcrypto/TrieDB.h
View File

@ -62,17 +62,21 @@ extern const h256 EmptyTrie;
* assert(t.isEmpty()); * assert(t.isEmpty());
* @endcode * @endcode
*/ */
template <class DB> template <class _DB>
class GenericTrieDB class GenericTrieDB
{ {
public: public:
using DB = _DB;
GenericTrieDB(DB* _db): m_db(_db) {} GenericTrieDB(DB* _db): m_db(_db) {}
GenericTrieDB(DB* _db, h256 _root) { open(_db, _root); } GenericTrieDB(DB* _db, h256 _root) { open(_db, _root); }
~GenericTrieDB() {} ~GenericTrieDB() {}
void open(DB* _db) { m_db = _db; }
void open(DB* _db, h256 _root) { m_db = _db; setRoot(_root); } void open(DB* _db, h256 _root) { m_db = _db; setRoot(_root); }
void init(); void init() { setRoot(insertNode(&RLPNull)); assert(node(m_root).size()); }
void setRoot(h256 _root) void setRoot(h256 _root)
{ {
m_root = _root; m_root = _root;
@ -83,7 +87,6 @@ public:
if (!node(m_root).size()) if (!node(m_root).size())
BOOST_THROW_EXCEPTION(RootNotFound()); BOOST_THROW_EXCEPTION(RootNotFound());
} }
bool haveRoot(h256 _root, bool _enforceRefs = true) { return _root == c_shaNull ? true : m_db->lookup(_root, _enforceRefs).size(); }
/// True if the trie is uninitialised (i.e. that the DB doesn't contain the root node). /// True if the trie is uninitialised (i.e. that the DB doesn't contain the root node).
bool isNull() const { return !node(m_root).size(); } bool isNull() const { return !node(m_root).size(); }
@ -211,6 +214,9 @@ public:
iterator lower_bound(bytesConstRef _key) const { return iterator(this, _key); } iterator lower_bound(bytesConstRef _key) const { return iterator(this, _key); }
protected:
DB* db() const { return m_db; }
private: private:
RLPStream& streamNode(RLPStream& _s, bytes const& _b); RLPStream& streamNode(RLPStream& _s, bytes const& _b);
@ -281,30 +287,33 @@ std::ostream& operator<<(std::ostream& _out, GenericTrieDB<DB> const& _db)
return _out; return _out;
} }
template <class KeyType, class DB> template <class Generic, class _KeyType>
class TrieDB: public GenericTrieDB<DB> class SpecificTrieDB: public Generic
{ {
public: public:
TrieDB(DB* _db): GenericTrieDB<DB>(_db) {} using DB = typename Generic::DB;
TrieDB(DB* _db, h256 _root): GenericTrieDB<DB>(_db, _root) {} using KeyType = _KeyType;
SpecificTrieDB(DB* _db): Generic(_db) {}
SpecificTrieDB(DB* _db, h256 _root): Generic(_db, _root) {}
std::string operator[](KeyType _k) const { return at(_k); } std::string operator[](KeyType _k) const { return at(_k); }
bool contains(KeyType _k) const { return GenericTrieDB<DB>::contains(bytesConstRef((byte const*)&_k, sizeof(KeyType))); } bool contains(KeyType _k) const { return Generic::contains(bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
std::string at(KeyType _k) const { return GenericTrieDB<DB>::at(bytesConstRef((byte const*)&_k, sizeof(KeyType))); } std::string at(KeyType _k) const { return Generic::at(bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
void insert(KeyType _k, bytesConstRef _value) { GenericTrieDB<DB>::insert(bytesConstRef((byte const*)&_k, sizeof(KeyType)), _value); } void insert(KeyType _k, bytesConstRef _value) { Generic::insert(bytesConstRef((byte const*)&_k, sizeof(KeyType)), _value); }
void insert(KeyType _k, bytes const& _value) { insert(_k, bytesConstRef(&_value)); } void insert(KeyType _k, bytes const& _value) { insert(_k, bytesConstRef(&_value)); }
void remove(KeyType _k) { GenericTrieDB<DB>::remove(bytesConstRef((byte const*)&_k, sizeof(KeyType))); } void remove(KeyType _k) { Generic::remove(bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
class iterator: public GenericTrieDB<DB>::iterator class iterator: public Generic::iterator
{ {
public: public:
using Super = typename GenericTrieDB<DB>::iterator; using Super = typename Generic::iterator;
using value_type = std::pair<KeyType, bytesConstRef>; using value_type = std::pair<KeyType, bytesConstRef>;
iterator() {} iterator() {}
iterator(TrieDB const* _db): Super(_db) {} iterator(Generic const* _db): Super(_db) {}
iterator(TrieDB const* _db, bytesConstRef _k): Super(_db, _k) {} iterator(Generic const* _db, bytesConstRef _k): Super(_db, _k) {}
value_type operator*() const { return at(); } value_type operator*() const { return at(); }
value_type operator->() const { return at(); } value_type operator->() const { return at(); }
@ -317,27 +326,28 @@ public:
iterator lower_bound(KeyType _k) const { return iterator(this, bytesConstRef((byte const*)&_k, sizeof(KeyType))); } iterator lower_bound(KeyType _k) const { return iterator(this, bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
}; };
template <class KeyType, class DB> template <class Generic, class KeyType>
std::ostream& operator<<(std::ostream& _out, TrieDB<KeyType, DB> const& _db) std::ostream& operator<<(std::ostream& _out, SpecificTrieDB<Generic, KeyType> const& _db)
{ {
for (auto const& i: _db) for (auto const& i: _db)
_out << i.first << ": " << escaped(i.second.toString(), false) << std::endl; _out << i.first << ": " << escaped(i.second.toString(), false) << std::endl;
return _out; return _out;
} }
template <class DB> template <class _DB>
class SecureGenericTrieDB: private TrieDB<h256, DB> class HashedGenericTrieDB: private SpecificTrieDB<GenericTrieDB<_DB>, h256>
{ {
using Super = TrieDB<h256, DB>; using Super = SpecificTrieDB<GenericTrieDB<_DB>, h256>;
public: public:
SecureGenericTrieDB(DB* _db): Super(_db) {} using DB = _DB;
SecureGenericTrieDB(DB* _db, h256 _root): Super(_db, _root) {}
HashedGenericTrieDB(DB* _db): Super(_db) {}
HashedGenericTrieDB(DB* _db, h256 _root): Super(_db, _root) {}
using Super::open; using Super::open;
using Super::init; using Super::init;
using Super::setRoot; using Super::setRoot;
using Super::haveRoot;
/// True if the trie is uninitialised (i.e. that the DB doesn't contain the root node). /// True if the trie is uninitialised (i.e. that the DB doesn't contain the root node).
using Super::isNull; using Super::isNull;
@ -350,29 +360,63 @@ public:
using Super::check; using Super::check;
std::string at(bytesConstRef _key) const { return Super::at(sha3(_key)); } std::string at(bytesConstRef _key) const { return Super::at(sha3(_key)); }
bool contains(bytesConstRef _key) { return Super::contains(sha3(_key)); }
void insert(bytesConstRef _key, bytesConstRef _value) { Super::insert(sha3(_key), _value); } void insert(bytesConstRef _key, bytesConstRef _value) { Super::insert(sha3(_key), _value); }
void remove(bytesConstRef _key) { Super::remove(sha3(_key)); } void remove(bytesConstRef _key) { Super::remove(sha3(_key)); }
bool contains(bytesConstRef _key) { return Super::contains(sha3(_key)); }
// empty from the PoV of the iterator interface.
using iterator = void*;
iterator begin() const { return nullptr; }
iterator end() const { return nullptr; }
iterator lower_bound(bytesConstRef) const { return end(); }
}; };
template <class KeyType, class DB> // Hashed & Basic
class SecureTrieDB: public SecureGenericTrieDB<DB> template <class DB>
class FatGenericTrieDB: public GenericTrieDB<DB>
{ {
using Super = SecureGenericTrieDB<DB>; using Super = GenericTrieDB<DB>;
public: public:
SecureTrieDB(DB* _db): Super(_db) {} FatGenericTrieDB(DB* _db): Super(_db), m_secure(_db) {}
SecureTrieDB(DB* _db, h256 _root): Super(_db, _root) {} FatGenericTrieDB(DB* _db, h256 _root) { open(_db, _root); }
std::string operator[](KeyType _k) const { return at(_k); } void open(DB* _db, h256 _root) { Super::open(_db); m_secure.open(_db); setRoot(_root); }
bool contains(KeyType _k) const { return Super::contains(bytesConstRef((byte const*)&_k, sizeof(KeyType))); } void init() { Super::init(); m_secure.init(); syncRoot(); }
std::string at(KeyType _k) const { return Super::at(bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
void insert(KeyType _k, bytesConstRef _value) { Super::insert(bytesConstRef((byte const*)&_k, sizeof(KeyType)), _value); } void setRoot(h256 _root)
void insert(KeyType _k, bytes const& _value) { insert(_k, bytesConstRef(&_value)); } {
void remove(KeyType _k) { Super::remove(bytesConstRef((byte const*)&_k, sizeof(KeyType))); } m_secure.setRoot(_root);
Super::setRoot(h256(Super::db()->lookupAux(m_secure.root())));
}
h256 root() const { return m_secure.root(); }
void insert(bytesConstRef _key, bytesConstRef _value) { Super::insert(_key, _value); m_secure.insert(_key, _value); syncRoot(); }
void remove(bytesConstRef _key) { Super::remove(_key); m_secure.remove(_key); syncRoot(); }
std::set<h256> leftOvers(std::ostream* = nullptr) const { return {}; }
bool check(bool) const { return m_secure.check(false) && Super::check(false); }
private:
void syncRoot()
{
// Root changed. Need to record the mapping so we can determine on setRoot.
Super::db()->insertAux(m_secure.root(), Super::root().ref());
}
HashedGenericTrieDB<DB> m_secure;
}; };
template <class KeyType, class DB> using TrieDB = SpecificTrieDB<GenericTrieDB<DB>, KeyType>;
#if ETH_FAT_DB
template <class KeyType, class DB> using SecureTrieDB = SpecificTrieDB<FatGenericTrieDB<DB>, KeyType>;
#else
template <class KeyType, class DB> using SecureTrieDB = SpecificTrieDB<HashedGenericTrieDB<DB>, KeyType>;
#endif
} }
// Template implementations... // Template implementations...
@ -641,7 +685,7 @@ template <class DB> void GenericTrieDB<DB>::iterator::next()
} }
} }
template <class KeyType, class DB> typename TrieDB<KeyType, DB>::iterator::value_type TrieDB<KeyType, DB>::iterator::at() const template <class KeyType, class DB> typename SpecificTrieDB<KeyType, DB>::iterator::value_type SpecificTrieDB<KeyType, DB>::iterator::at() const
{ {
auto p = Super::at(); auto p = Super::at();
value_type ret; value_type ret;
@ -651,13 +695,6 @@ template <class KeyType, class DB> typename TrieDB<KeyType, DB>::iterator::value
return ret; return ret;
} }
template <class DB> void GenericTrieDB<DB>::init()
{
m_root = insertNode(&RLPNull);
// std::cout << "Initialised root to " << m_root << std::endl;
assert(node(m_root).size());
}
template <class DB> void GenericTrieDB<DB>::insert(bytesConstRef _key, bytesConstRef _value) template <class DB> void GenericTrieDB<DB>::insert(bytesConstRef _key, bytesConstRef _value)
{ {
#if ETH_PARANOIA #if ETH_PARANOIA

44
test/trie.cpp
View File

@ -50,6 +50,8 @@ static unsigned fac(unsigned _i)
} }
} }
using dev::operator <<;
BOOST_AUTO_TEST_SUITE(TrieTests) BOOST_AUTO_TEST_SUITE(TrieTests)
BOOST_AUTO_TEST_CASE(trie_test_anyorder) BOOST_AUTO_TEST_CASE(trie_test_anyorder)
@ -81,15 +83,35 @@ BOOST_AUTO_TEST_CASE(trie_test_anyorder)
next_permutation(ss.begin(), ss.end()); next_permutation(ss.begin(), ss.end());
MemoryDB m; MemoryDB m;
GenericTrieDB<MemoryDB> t(&m); GenericTrieDB<MemoryDB> t(&m);
MemoryDB hm;
HashedGenericTrieDB<MemoryDB> ht(&hm);
MemoryDB fm;
FatGenericTrieDB<MemoryDB> ft(&fm);
t.init(); t.init();
ht.init();
ft.init();
BOOST_REQUIRE(t.check(true)); BOOST_REQUIRE(t.check(true));
BOOST_REQUIRE(ht.check(true));
BOOST_REQUIRE(ft.check(true));
for (auto const& k: ss) for (auto const& k: ss)
{ {
t.insert(k.first, k.second); t.insert(k.first, k.second);
ht.insert(k.first, k.second);
ft.insert(k.first, k.second);
BOOST_REQUIRE(t.check(true)); BOOST_REQUIRE(t.check(true));
BOOST_REQUIRE(ht.check(true));
BOOST_REQUIRE(ft.check(true));
for (auto i = ft.begin(), j = t.begin(); i != ft.end() && j != t.end(); ++i, ++j)
{
BOOST_CHECK_EQUAL(i == ft.end(), j == t.end());
BOOST_REQUIRE((*i).first.toBytes() == (*j).first.toBytes());
BOOST_REQUIRE((*i).second.toBytes() == (*j).second.toBytes());
}
BOOST_CHECK_EQUAL(ht.root(), ft.root());
} }
BOOST_REQUIRE(!o["root"].is_null()); BOOST_REQUIRE(!o["root"].is_null());
BOOST_CHECK_EQUAL(o["root"].get_str(), "0x" + toHex(t.root().asArray())); BOOST_CHECK_EQUAL(o["root"].get_str(), "0x" + toHex(t.root().asArray()));
BOOST_CHECK_EQUAL(ht.root(), ft.root());
} }
} }
} }
@ -141,15 +163,33 @@ BOOST_AUTO_TEST_CASE(trie_tests_ordered)
MemoryDB m; MemoryDB m;
GenericTrieDB<MemoryDB> t(&m); GenericTrieDB<MemoryDB> t(&m);
MemoryDB hm;
HashedGenericTrieDB<MemoryDB> ht(&hm);
MemoryDB fm;
FatGenericTrieDB<MemoryDB> ft(&fm);
t.init(); t.init();
ht.init();
ft.init();
BOOST_REQUIRE(t.check(true)); BOOST_REQUIRE(t.check(true));
BOOST_REQUIRE(ht.check(true));
BOOST_REQUIRE(ft.check(true));
for (auto const& k: ss) for (auto const& k: ss)
{ {
if (find(keysToBeDeleted.begin(), keysToBeDeleted.end(), k.first) != keysToBeDeleted.end() && k.second.empty()) if (find(keysToBeDeleted.begin(), keysToBeDeleted.end(), k.first) != keysToBeDeleted.end() && k.second.empty())
t.remove(k.first); t.remove(k.first), ht.remove(k.first), ft.remove(k.first);
else else
t.insert(k.first, k.second); t.insert(k.first, k.second), ht.insert(k.first, k.second), ft.insert(k.first, k.second);
BOOST_REQUIRE(t.check(true)); BOOST_REQUIRE(t.check(true));
BOOST_REQUIRE(ht.check(true));
BOOST_REQUIRE(ft.check(true));
for (auto i = ft.begin(), j = t.begin(); i != ft.end() && j != t.end(); ++i, ++j)
{
BOOST_CHECK_EQUAL(i == ft.end(), j == t.end());
BOOST_REQUIRE((*i).first.toBytes() == (*j).first.toBytes());
BOOST_REQUIRE((*i).second.toBytes() == (*j).second.toBytes());
}
BOOST_CHECK_EQUAL(ht.root(), ft.root());
} }
BOOST_REQUIRE(!o["root"].is_null()); BOOST_REQUIRE(!o["root"].is_null());

Write Preview
Loading…
Cancel
Save