lukechilds
/
ethminer
mirror of https://github.com/lukechilds/ethminer.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Fat Trie and tests for it.

cl-refactor
Gav Wood 10 years ago
parent
92bfa4a7cf
commit
0f01f4799f
5 changed files with 130 additions and 49 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      libdevcore/FixedHash.h
  2. 2
      libdevcrypto/MemoryDB.cpp
  3. 4
      libdevcrypto/MemoryDB.h
  4. 123
      libdevcrypto/TrieDB.h
  5. 44
      test/trie.cpp

6
libdevcore/FixedHash.h
View File

@ -53,7 +53,7 @@ public:
enum ConstructFromStringType { FromHex, FromBinary };
/// Method to convert from a string.
enum ConstructFromHashType { AlignLeft, AlignRight };
enum ConstructFromHashType { AlignLeft, AlignRight, FailIfDifferent };
/// Construct an empty hash.
FixedHash() { m_data.fill(0); }
@ -65,10 +65,10 @@ public:
FixedHash(Arith const& _arith) { toBigEndian(_arith, m_data); }
/// Explicitly construct, copying from a byte array.
explicit FixedHash(bytes const& _b) { if (_b.size() == N) memcpy(m_data.data(), _b.data(), std::min<unsigned>(_b.size(), N)); }
explicit FixedHash(bytes const& _b, ConstructFromHashType _t = FailIfDifferent) { if (_b.size() == N) memcpy(m_data.data(), _b.data(), std::min<unsigned>(_b.size(), N)); else if (_t != FailIfDifferent) { m_data.fill(0); auto c = std::min<unsigned>(_b.size(), N); for (unsigned i = 0; i < c; ++i) m_data[_t == AlignRight ? N - 1 - i : i] = _b[_t == AlignRight ? _b.size() - 1 - i : i]; } }
/// Explicitly construct, copying from a byte array.
explicit FixedHash(bytesConstRef _b) { if (_b.size() == N) memcpy(m_data.data(), _b.data(), std::min<unsigned>(_b.size(), N)); }
explicit FixedHash(bytesConstRef _b, ConstructFromHashType _t = FailIfDifferent) { if (_b.size() == N) memcpy(m_data.data(), _b.data(), std::min<unsigned>(_b.size(), N)); else if (_t != FailIfDifferent) { m_data.fill(0); auto c = std::min<unsigned>(_b.size(), N); for (unsigned i = 0; i < c; ++i) m_data[_t == AlignRight ? N - 1 - i : i] = _b[_t == AlignRight ? _b.size() - 1 - i : i]; } }
/// Explicitly construct, copying from a bytes in memory with given pointer.
explicit FixedHash(byte const* _bs, ConstructFromPointerType) { memcpy(m_data.data(), _bs, N); }

2
libdevcrypto/MemoryDB.cpp
View File

@ -107,7 +107,7 @@ set<h256> MemoryDB::keys() const
{
set<h256> ret;
for (auto const& i: m_refCount)
if (i.second)
if (i.second && h128(i.first.ref().cropped(0, 16)))
ret.insert(i.first);
return ret;
}

4
libdevcrypto/MemoryDB.h
View File

@ -26,6 +26,7 @@
#include <libdevcore/FixedHash.h>
#include <libdevcore/Log.h>
#include <libdevcore/RLP.h>
#include "SHA3.h"
namespace dev
{
@ -50,6 +51,9 @@ public:
bool kill(h256 _h);
void purge();
bytes lookupAux(h256 _h) const { return asBytes(lookup(h256(sha3(_h).ref().cropped(16), h256::AlignRight))); }
void insertAux(h256 _h, bytesConstRef _v) { return insert(h256(sha3(_h).ref().cropped(16), h256::AlignRight), _v); }
std::set<h256> keys() const;
protected:

123
libdevcrypto/TrieDB.h
View File

@ -62,17 +62,21 @@ extern const h256 EmptyTrie;
* assert(t.isEmpty());
* @endcode
*/
template <class DB>
template <class _DB>
class GenericTrieDB
{
public:
using DB = _DB;
GenericTrieDB(DB* _db): m_db(_db) {}
GenericTrieDB(DB* _db, h256 _root) { open(_db, _root); }
~GenericTrieDB() {}
void open(DB* _db) { m_db = _db; }
void open(DB* _db, h256 _root) { m_db = _db; setRoot(_root); }
void init();
void init() { setRoot(insertNode(&RLPNull)); assert(node(m_root).size()); }
void setRoot(h256 _root)
{
m_root = _root;
@ -83,7 +87,6 @@ public:
if (!node(m_root).size())
BOOST_THROW_EXCEPTION(RootNotFound());
}
bool haveRoot(h256 _root, bool _enforceRefs = true) { return _root == c_shaNull ? true : m_db->lookup(_root, _enforceRefs).size(); }
/// True if the trie is uninitialised (i.e. that the DB doesn't contain the root node).
bool isNull() const { return !node(m_root).size(); }
@ -211,6 +214,9 @@ public:
iterator lower_bound(bytesConstRef _key) const { return iterator(this, _key); }
protected:
DB* db() const { return m_db; }
private:
RLPStream& streamNode(RLPStream& _s, bytes const& _b);
@ -281,30 +287,33 @@ std::ostream& operator<<(std::ostream& _out, GenericTrieDB<DB> const& _db)
return _out;
}
template <class KeyType, class DB>
class TrieDB: public GenericTrieDB<DB>
template <class Generic, class _KeyType>
class SpecificTrieDB: public Generic
{
public:
TrieDB(DB* _db): GenericTrieDB<DB>(_db) {}
TrieDB(DB* _db, h256 _root): GenericTrieDB<DB>(_db, _root) {}
using DB = typename Generic::DB;
using KeyType = _KeyType;
SpecificTrieDB(DB* _db): Generic(_db) {}
SpecificTrieDB(DB* _db, h256 _root): Generic(_db, _root) {}
std::string operator[](KeyType _k) const { return at(_k); }
bool contains(KeyType _k) const { return GenericTrieDB<DB>::contains(bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
std::string at(KeyType _k) const { return GenericTrieDB<DB>::at(bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
void insert(KeyType _k, bytesConstRef _value) { GenericTrieDB<DB>::insert(bytesConstRef((byte const*)&_k, sizeof(KeyType)), _value); }
bool contains(KeyType _k) const { return Generic::contains(bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
std::string at(KeyType _k) const { return Generic::at(bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
void insert(KeyType _k, bytesConstRef _value) { Generic::insert(bytesConstRef((byte const*)&_k, sizeof(KeyType)), _value); }
void insert(KeyType _k, bytes const& _value) { insert(_k, bytesConstRef(&_value)); }
void remove(KeyType _k) { GenericTrieDB<DB>::remove(bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
void remove(KeyType _k) { Generic::remove(bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
class iterator: public GenericTrieDB<DB>::iterator
class iterator: public Generic::iterator
{
public:
using Super = typename GenericTrieDB<DB>::iterator;
using Super = typename Generic::iterator;
using value_type = std::pair<KeyType, bytesConstRef>;
iterator() {}
iterator(TrieDB const* _db): Super(_db) {}
iterator(TrieDB const* _db, bytesConstRef _k): Super(_db, _k) {}
iterator(Generic const* _db): Super(_db) {}
iterator(Generic const* _db, bytesConstRef _k): Super(_db, _k) {}
value_type operator*() const { return at(); }
value_type operator->() const { return at(); }
@ -317,27 +326,28 @@ public:
iterator lower_bound(KeyType _k) const { return iterator(this, bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
};
template <class KeyType, class DB>
std::ostream& operator<<(std::ostream& _out, TrieDB<KeyType, DB> const& _db)
template <class Generic, class KeyType>
std::ostream& operator<<(std::ostream& _out, SpecificTrieDB<Generic, KeyType> const& _db)
{
for (auto const& i: _db)
_out << i.first << ": " << escaped(i.second.toString(), false) << std::endl;
return _out;
}
template <class DB>
class SecureGenericTrieDB: private TrieDB<h256, DB>
template <class _DB>
class HashedGenericTrieDB: private SpecificTrieDB<GenericTrieDB<_DB>, h256>
{
using Super = TrieDB<h256, DB>;
using Super = SpecificTrieDB<GenericTrieDB<_DB>, h256>;
public:
SecureGenericTrieDB(DB* _db): Super(_db) {}
SecureGenericTrieDB(DB* _db, h256 _root): Super(_db, _root) {}
using DB = _DB;
HashedGenericTrieDB(DB* _db): Super(_db) {}
HashedGenericTrieDB(DB* _db, h256 _root): Super(_db, _root) {}
using Super::open;
using Super::init;
using Super::setRoot;
using Super::haveRoot;
/// True if the trie is uninitialised (i.e. that the DB doesn't contain the root node).
using Super::isNull;
@ -350,29 +360,63 @@ public:
using Super::check;
std::string at(bytesConstRef _key) const { return Super::at(sha3(_key)); }
bool contains(bytesConstRef _key) { return Super::contains(sha3(_key)); }
void insert(bytesConstRef _key, bytesConstRef _value) { Super::insert(sha3(_key), _value); }
void remove(bytesConstRef _key) { Super::remove(sha3(_key)); }
bool contains(bytesConstRef _key) { return Super::contains(sha3(_key)); }
// empty from the PoV of the iterator interface.
using iterator = void*;
iterator begin() const { return nullptr; }
iterator end() const { return nullptr; }
iterator lower_bound(bytesConstRef) const { return end(); }
};
template <class KeyType, class DB>
class SecureTrieDB: public SecureGenericTrieDB<DB>
// Hashed & Basic
template <class DB>
class FatGenericTrieDB: public GenericTrieDB<DB>
{
using Super = SecureGenericTrieDB<DB>;
using Super = GenericTrieDB<DB>;
public:
SecureTrieDB(DB* _db): Super(_db) {}
SecureTrieDB(DB* _db, h256 _root): Super(_db, _root) {}
FatGenericTrieDB(DB* _db): Super(_db), m_secure(_db) {}
FatGenericTrieDB(DB* _db, h256 _root) { open(_db, _root); }
std::string operator[](KeyType _k) const { return at(_k); }
void open(DB* _db, h256 _root) { Super::open(_db); m_secure.open(_db); setRoot(_root); }
bool contains(KeyType _k) const { return Super::contains(bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
std::string at(KeyType _k) const { return Super::at(bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
void insert(KeyType _k, bytesConstRef _value) { Super::insert(bytesConstRef((byte const*)&_k, sizeof(KeyType)), _value); }
void insert(KeyType _k, bytes const& _value) { insert(_k, bytesConstRef(&_value)); }
void remove(KeyType _k) { Super::remove(bytesConstRef((byte const*)&_k, sizeof(KeyType))); }
void init() { Super::init(); m_secure.init(); syncRoot(); }
void setRoot(h256 _root)
{
m_secure.setRoot(_root);
Super::setRoot(h256(Super::db()->lookupAux(m_secure.root())));
}
h256 root() const { return m_secure.root(); }
void insert(bytesConstRef _key, bytesConstRef _value) { Super::insert(_key, _value); m_secure.insert(_key, _value); syncRoot(); }
void remove(bytesConstRef _key) { Super::remove(_key); m_secure.remove(_key); syncRoot(); }
std::set<h256> leftOvers(std::ostream* = nullptr) const { return {}; }
bool check(bool) const { return m_secure.check(false) && Super::check(false); }
private:
void syncRoot()
{
// Root changed. Need to record the mapping so we can determine on setRoot.
Super::db()->insertAux(m_secure.root(), Super::root().ref());
}
HashedGenericTrieDB<DB> m_secure;
};
template <class KeyType, class DB> using TrieDB = SpecificTrieDB<GenericTrieDB<DB>, KeyType>;
#if ETH_FAT_DB
template <class KeyType, class DB> using SecureTrieDB = SpecificTrieDB<FatGenericTrieDB<DB>, KeyType>;
#else
template <class KeyType, class DB> using SecureTrieDB = SpecificTrieDB<HashedGenericTrieDB<DB>, KeyType>;
#endif
}
// Template implementations...
@ -641,7 +685,7 @@ template <class DB> void GenericTrieDB<DB>::iterator::next()
}
}
template <class KeyType, class DB> typename TrieDB<KeyType, DB>::iterator::value_type TrieDB<KeyType, DB>::iterator::at() const
template <class KeyType, class DB> typename SpecificTrieDB<KeyType, DB>::iterator::value_type SpecificTrieDB<KeyType, DB>::iterator::at() const
{
auto p = Super::at();
value_type ret;
@ -651,13 +695,6 @@ template <class KeyType, class DB> typename TrieDB<KeyType, DB>::iterator::value
return ret;
}
template <class DB> void GenericTrieDB<DB>::init()
{
m_root = insertNode(&RLPNull);
// std::cout << "Initialised root to " << m_root << std::endl;
assert(node(m_root).size());
}
template <class DB> void GenericTrieDB<DB>::insert(bytesConstRef _key, bytesConstRef _value)
{
#if ETH_PARANOIA

44
test/trie.cpp
View File

@ -48,6 +48,8 @@ static unsigned fac(unsigned _i)
}
}
using dev::operator <<;
BOOST_AUTO_TEST_SUITE(TrieTests)
BOOST_AUTO_TEST_CASE(trie_test_anyorder)
@ -79,15 +81,35 @@ BOOST_AUTO_TEST_CASE(trie_test_anyorder)
next_permutation(ss.begin(), ss.end());
MemoryDB m;
GenericTrieDB<MemoryDB> t(&m);
MemoryDB hm;
HashedGenericTrieDB<MemoryDB> ht(&hm);
MemoryDB fm;
FatGenericTrieDB<MemoryDB> ft(&fm);
t.init();
ht.init();
ft.init();
BOOST_REQUIRE(t.check(true));
BOOST_REQUIRE(ht.check(true));
BOOST_REQUIRE(ft.check(true));
for (auto const& k: ss)
{
t.insert(k.first, k.second);
ht.insert(k.first, k.second);
ft.insert(k.first, k.second);
BOOST_REQUIRE(t.check(true));
BOOST_REQUIRE(ht.check(true));
BOOST_REQUIRE(ft.check(true));
for (auto i = ft.begin(), j = t.begin(); i != ft.end() && j != t.end(); ++i, ++j)
{
BOOST_CHECK_EQUAL(i == ft.end(), j == t.end());
BOOST_REQUIRE((*i).first.toBytes() == (*j).first.toBytes());
BOOST_REQUIRE((*i).second.toBytes() == (*j).second.toBytes());
}
BOOST_CHECK_EQUAL(ht.root(), ft.root());
}
BOOST_REQUIRE(!o["root"].is_null());
BOOST_CHECK_EQUAL(o["root"].get_str(), "0x" + toHex(t.root().asArray()));
BOOST_CHECK_EQUAL(ht.root(), ft.root());
}
}
}
@ -139,15 +161,33 @@ BOOST_AUTO_TEST_CASE(trie_tests_ordered)
MemoryDB m;
GenericTrieDB<MemoryDB> t(&m);
MemoryDB hm;
HashedGenericTrieDB<MemoryDB> ht(&hm);
MemoryDB fm;
FatGenericTrieDB<MemoryDB> ft(&fm);
t.init();
ht.init();
ft.init();
BOOST_REQUIRE(t.check(true));
BOOST_REQUIRE(ht.check(true));
BOOST_REQUIRE(ft.check(true));
for (auto const& k: ss)
{
if (find(keysToBeDeleted.begin(), keysToBeDeleted.end(), k.first) != keysToBeDeleted.end() && k.second.empty())
t.remove(k.first);
t.remove(k.first), ht.remove(k.first), ft.remove(k.first);
else
t.insert(k.first, k.second);
t.insert(k.first, k.second), ht.insert(k.first, k.second), ft.insert(k.first, k.second);
BOOST_REQUIRE(t.check(true));
BOOST_REQUIRE(ht.check(true));
BOOST_REQUIRE(ft.check(true));
for (auto i = ft.begin(), j = t.begin(); i != ft.end() && j != t.end(); ++i, ++j)
{
BOOST_CHECK_EQUAL(i == ft.end(), j == t.end());
BOOST_REQUIRE((*i).first.toBytes() == (*j).first.toBytes());
BOOST_REQUIRE((*i).second.toBytes() == (*j).second.toBytes());
}
BOOST_CHECK_EQUAL(ht.root(), ft.root());
}
BOOST_REQUIRE(!o["root"].is_null());

Write Preview
Loading…
Cancel
Save