lukechilds
/
ethminer
mirror of https://github.com/lukechilds/ethminer.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Trie optimizations

cl-refactor
arkpar 10 years ago
parent
ebe06b37de
commit
b50d44b8ac
4 changed files with 119 additions and 40 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      libdevcrypto/OverlayDB.cpp
  2. 89
      libdevcrypto/TrieDB.h
  3. 3
      libethcore/Common.cpp
  4. 61
      test/libdevcrypto/trie.cpp

2
libdevcrypto/OverlayDB.cpp
View File

@ -94,7 +94,7 @@ void OverlayDB::commit()
bytes OverlayDB::lookupAux(h256 const& _h) const bytes OverlayDB::lookupAux(h256 const& _h) const
{ {
bytes ret = MemoryDB::lookupAux(_h); bytes ret = MemoryDB::lookupAux(_h);
if (!ret.empty()) if (!ret.empty() || !m_db)
return move(ret); return move(ret);
std::string v; std::string v;
bytes b = _h.asBytes(); bytes b = _h.asBytes();

89
libdevcrypto/TrieDB.h
View File

@ -220,6 +220,7 @@ public:
bool operator!=(Node const& _c) const { return !operator==(_c); } bool operator!=(Node const& _c) const { return !operator==(_c); }
}; };
protected:
std::vector<Node> m_trail; std::vector<Node> m_trail;
GenericTrieDB<DB> const* m_that; GenericTrieDB<DB> const* m_that;
}; };
@ -239,6 +240,7 @@ private:
void mergeAtAux(RLPStream& _out, RLP const& _replace, NibbleSlice _key, bytesConstRef _value); void mergeAtAux(RLPStream& _out, RLP const& _replace, NibbleSlice _key, bytesConstRef _value);
bytes mergeAt(RLP const& _replace, NibbleSlice _k, bytesConstRef _v, bool _inLine = false); bytes mergeAt(RLP const& _replace, NibbleSlice _k, bytesConstRef _v, bool _inLine = false);
bytes mergeAt(RLP const& _replace, h256 const& _replaceHash, NibbleSlice _k, bytesConstRef _v, bool _inLine = false);
bool deleteAtAux(RLPStream& _out, RLP const& _replace, NibbleSlice _key); bool deleteAtAux(RLPStream& _out, RLP const& _replace, NibbleSlice _key);
bytes deleteAt(RLP const& _replace, NibbleSlice _k); bytes deleteAt(RLP const& _replace, NibbleSlice _k);
@ -295,6 +297,7 @@ private:
// for the special case of the root (which is always looked up via a hash). In that case, // for the special case of the root (which is always looked up via a hash). In that case,
// use forceKillNode(). // use forceKillNode().
void killNode(RLP const& _d) { if (_d.data().size() >= 32) forceKillNode(sha3(_d.data())); } void killNode(RLP const& _d) { if (_d.data().size() >= 32) forceKillNode(sha3(_d.data())); }
void killNode(RLP const& _d, h256 const& _h) { if (_d.data().size() >= 32) forceKillNode(_h); }
h256 m_root; h256 m_root;
DB* m_db = nullptr; DB* m_db = nullptr;
@ -409,46 +412,59 @@ public:
iterator lower_bound(bytesConstRef) const { return iterator(); } iterator lower_bound(bytesConstRef) const { return iterator(); }
}; };
// Hashed & Basic // Hashed & Hash-key mapping
template <class DB> template <class _DB>
class FatGenericTrieDB: public GenericTrieDB<DB> class FatGenericTrieDB: private SpecificTrieDB<GenericTrieDB<_DB>, h256>
{ {
using Super = GenericTrieDB<DB>; using Super = SpecificTrieDB<GenericTrieDB<_DB>, h256>;
public: public:
FatGenericTrieDB(DB* _db): Super(_db), m_secure(_db) {} using DB = _DB;
FatGenericTrieDB(DB* _db, h256 _root, Verification _v = Verification::Normal) { open(_db, _root, _v); } FatGenericTrieDB(DB* _db = nullptr): Super(_db) {}
FatGenericTrieDB(DB* _db, h256 _root, Verification _v = Verification::Normal): Super(_db, _root, _v) {}
void open(DB* _db, h256 _root, Verification _v = Verification::Normal) { Super::open(_db); m_secure.open(_db); setRoot(_root, _v); }
void init() { Super::init(); m_secure.init(); syncRoot(); } using Super::init;
using Super::isNull;
using Super::isEmpty;
using Super::root;
using Super::leftOvers;
using Super::check;
using Super::open;
using Super::setRoot;
void setRoot(h256 _root, Verification _v = Verification::Normal) std::string at(bytesConstRef _key) const { return Super::at(sha3(_key)); }
bool contains(bytesConstRef _key) { return Super::contains(sha3(_key)); }
void insert(bytesConstRef _key, bytesConstRef _value)
{ {
if (!m_secure.isNull()) h256 hash = sha3(_key);
Super::db()->removeAux(m_secure.root()); Super::insert(hash, _value);
m_secure.setRoot(_root, _v); Super::db()->insertAux(hash, _key);
auto rb = Super::db()->lookupAux(m_secure.root());
auto r = h256(rb);
Super::setRoot(r, _v);
} }
h256 root() const { return m_secure.root(); } void remove(bytesConstRef _key) { Super::remove(sha3(_key)); }
void insert(bytesConstRef _key, bytesConstRef _value) { Super::insert(_key, _value); m_secure.insert(_key, _value); syncRoot(); } //friend class iterator;
void remove(bytesConstRef _key) { Super::remove(_key); m_secure.remove(_key); syncRoot(); }
h256Hash leftOvers(std::ostream* = nullptr) const { return h256Hash{}; } class iterator : public GenericTrieDB<_DB>::iterator
bool check(bool) const { return m_secure.check(false) && Super::check(false); } {
public:
using Super = typename GenericTrieDB<_DB>::iterator;
private: iterator() { }
void syncRoot() iterator(FatGenericTrieDB const* _trie): Super(_trie) { }
typename Super::value_type at() const
{ {
// Root changed. Need to record the mapping so we can determine on setRoot. auto hashed = Super::at();
Super::db()->insertAux(m_secure.root(), Super::root().ref()); m_key = static_cast<FatGenericTrieDB const*>(Super::m_that)->db()->lookupAux(h256(hashed.first));
return std::make_pair(&m_key, std::move(hashed.second));
} }
HashedGenericTrieDB<DB> m_secure; private:
mutable bytes m_key;
};
iterator begin() const { return iterator(); }
iterator end() const { return iterator(); }
}; };
template <class KeyType, class DB> using TrieDB = SpecificTrieDB<GenericTrieDB<DB>, KeyType>; template <class KeyType, class DB> using TrieDB = SpecificTrieDB<GenericTrieDB<DB>, KeyType>;
@ -745,7 +761,7 @@ template <class DB> void GenericTrieDB<DB>::insert(bytesConstRef _key, bytesCons
std::string rv = node(m_root); std::string rv = node(m_root);
assert(rv.size()); assert(rv.size());
bytes b = mergeAt(RLP(rv), NibbleSlice(_key), _value); bytes b = mergeAt(RLP(rv), m_root, NibbleSlice(_key), _value);
// mergeAt won't attempt to delete the node if it's less than 32 bytes // mergeAt won't attempt to delete the node if it's less than 32 bytes
// However, we know it's the root node and thus always hashed. // However, we know it's the root node and thus always hashed.
@ -765,8 +781,9 @@ template <class DB> std::string GenericTrieDB<DB>::atAux(RLP const& _here, Nibbl
if (_here.isEmpty() || _here.isNull()) if (_here.isEmpty() || _here.isNull())
// not found. // not found.
return std::string(); return std::string();
assert(_here.isList() && (_here.itemCount() == 2 || _here.itemCount() == 17)); unsigned itemCount = _here.itemCount();
if (_here.itemCount() == 2) assert(_here.isList() && (itemCount == 2 || itemCount == 17));
if (itemCount == 2)
{ {
auto k = keyOf(_here); auto k = keyOf(_here);
if (_key == k && isLeaf(_here)) if (_key == k && isLeaf(_here))
@ -792,6 +809,11 @@ template <class DB> std::string GenericTrieDB<DB>::atAux(RLP const& _here, Nibbl
} }
template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSlice _k, bytesConstRef _v, bool _inLine) template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSlice _k, bytesConstRef _v, bool _inLine)
{
return mergeAt(_orig, sha3(_orig.data()), _k, _v, _inLine);
}
template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, h256 const& _origHash, NibbleSlice _k, bytesConstRef _v, bool _inLine)
{ {
#if ETH_PARANOIA #if ETH_PARANOIA
tdebug << "mergeAt " << _orig << _k << sha3(_orig.data()); tdebug << "mergeAt " << _orig << _k << sha3(_orig.data());
@ -805,8 +827,9 @@ template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSli
if (_orig.isEmpty()) if (_orig.isEmpty())
return place(_orig, _k, _v); return place(_orig, _k, _v);
assert(_orig.isList() && (_orig.itemCount() == 2 || _orig.itemCount() == 17)); unsigned itemCount = _orig.itemCount();
if (_orig.itemCount() == 2) assert(_orig.isList() && (itemCount == 2 || itemCount == 17));
if (itemCount == 2)
{ {
// pair... // pair...
NibbleSlice k = keyOf(_orig); NibbleSlice k = keyOf(_orig);
@ -819,7 +842,7 @@ template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSli
if (_k.contains(k) && !isLeaf(_orig)) if (_k.contains(k) && !isLeaf(_orig))
{ {
if (!_inLine) if (!_inLine)
killNode(_orig); killNode(_orig, _origHash);
RLPStream s(2); RLPStream s(2);
s.append(_orig[0]); s.append(_orig[0]);
mergeAtAux(s, _orig[1], _k.mid(k.size()), _v); mergeAtAux(s, _orig[1], _k.mid(k.size()), _v);
@ -851,7 +874,7 @@ template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSli
// Kill the node. // Kill the node.
if (!_inLine) if (!_inLine)
killNode(_orig); killNode(_orig, _origHash);
// not exactly our node - delve to next level at the correct index. // not exactly our node - delve to next level at the correct index.
byte n = _k[0]; byte n = _k[0];

3
libethcore/Common.cpp
View File

@ -37,10 +37,11 @@ namespace eth
const unsigned c_protocolVersion = 60; const unsigned c_protocolVersion = 60;
const unsigned c_minorProtocolVersion = 2; const unsigned c_minorProtocolVersion = 2;
const unsigned c_databaseBaseVersion = 9;
#if ETH_FATDB #if ETH_FATDB
const unsigned c_databaseBaseVersion = 10;
const unsigned c_databaseVersionModifier = 1; const unsigned c_databaseVersionModifier = 1;
#else #else
const unsigned c_databaseBaseVersion = 9;
const unsigned c_databaseVersionModifier = 0; const unsigned c_databaseVersionModifier = 0;
#endif #endif

61
test/libdevcrypto/trie.cpp
View File

@ -124,7 +124,9 @@ BOOST_AUTO_TEST_CASE(hex_encoded_securetrie_test)
BOOST_REQUIRE(t.check(true)); BOOST_REQUIRE(t.check(true));
BOOST_REQUIRE(ht.check(true)); BOOST_REQUIRE(ht.check(true));
BOOST_REQUIRE(ft.check(true)); BOOST_REQUIRE(ft.check(true));
for (auto i = ft.begin(), j = t.begin(); i != ft.end() && j != t.end(); ++i, ++j) auto i = ft.begin();
auto j = t.begin();
for (; i != ft.end() && j != t.end(); ++i, ++j)
{ {
BOOST_CHECK_EQUAL(i == ft.end(), j == t.end()); BOOST_CHECK_EQUAL(i == ft.end(), j == t.end());
BOOST_REQUIRE((*i).first.toBytes() == (*j).first.toBytes()); BOOST_REQUIRE((*i).first.toBytes() == (*j).first.toBytes());
@ -189,7 +191,9 @@ BOOST_AUTO_TEST_CASE(trie_test_anyorder)
BOOST_REQUIRE(t.check(true)); BOOST_REQUIRE(t.check(true));
BOOST_REQUIRE(ht.check(true)); BOOST_REQUIRE(ht.check(true));
BOOST_REQUIRE(ft.check(true)); BOOST_REQUIRE(ft.check(true));
for (auto i = ft.begin(), j = t.begin(); i != ft.end() && j != t.end(); ++i, ++j) auto i = ft.begin();
auto j = t.begin();
for (; i != ft.end() && j != t.end(); ++i, ++j)
{ {
BOOST_CHECK_EQUAL(i == ft.end(), j == t.end()); BOOST_CHECK_EQUAL(i == ft.end(), j == t.end());
BOOST_REQUIRE((*i).first.toBytes() == (*j).first.toBytes()); BOOST_REQUIRE((*i).first.toBytes() == (*j).first.toBytes());
@ -274,7 +278,9 @@ BOOST_AUTO_TEST_CASE(trie_tests_ordered)
BOOST_REQUIRE(t.check(true)); BOOST_REQUIRE(t.check(true));
BOOST_REQUIRE(ht.check(true)); BOOST_REQUIRE(ht.check(true));
BOOST_REQUIRE(ft.check(true)); BOOST_REQUIRE(ft.check(true));
for (auto i = ft.begin(), j = t.begin(); i != ft.end() && j != t.end(); ++i, ++j) auto i = ft.begin();
auto j = t.begin();
for (; i != ft.end() && j != t.end(); ++i, ++j)
{ {
BOOST_CHECK_EQUAL(i == ft.end(), j == t.end()); BOOST_CHECK_EQUAL(i == ft.end(), j == t.end());
BOOST_REQUIRE((*i).first.toBytes() == (*j).first.toBytes()); BOOST_REQUIRE((*i).first.toBytes() == (*j).first.toBytes());
@ -558,6 +564,55 @@ BOOST_AUTO_TEST_CASE(trieStess)
} }
} }
template<typename Trie> void perfTestTrie(char const* _name)
{
for (size_t p = 1000; p != 1000000; p*=10)
{
MemoryDB dm;
Trie d(&dm);
d.init();
cnote << "TriePerf " << _name << p;
std::vector<h256> keys(1000);
boost::timer t;
size_t ki = 0;
for (size_t i = 0; i < p; ++i)
{
auto k = h256::random();
auto v = toString(i);
d.insert(k, v);
if (i % (p / 1000) == 0)
keys[ki++] = k;
}
cnote << "Insert " << p << "values: " << t.elapsed();
t.restart();
for (auto k: keys)
d.at(k);
cnote << "Query 1000 values: " << t.elapsed();
t.restart();
size_t i = 0;
for (auto it = d.begin(); i < 1000 && it != d.end(); ++it, ++i)
*it;
cnote << "Iterate 1000 values: " << t.elapsed();
t.restart();
for (auto k: keys)
d.remove(k);
cnote << "Remove 1000 values:" << t.elapsed() << "\n";
}
}
BOOST_AUTO_TEST_CASE(triePerf)
{
if (test::Options::get().performance)
{
perfTestTrie<SpecificTrieDB<GenericTrieDB<MemoryDB>, h256>>("GenericTrieDB");
perfTestTrie<SpecificTrieDB<HashedGenericTrieDB<MemoryDB>, h256>>("HashedGenericTrieDB");
perfTestTrie<SpecificTrieDB<FatGenericTrieDB<MemoryDB>, h256>>("FatGenericTrieDB");
}
}
BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END()

Write Preview
Loading…
Cancel
Save