lukechilds
/
ethminer
mirror of https://github.com/lukechilds/ethminer.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

TrieDB remove nodes and tests.

cl-refactor
Gav Wood 11 years ago
parent
04743c50f7
commit
0b29820d98
4 changed files with 303 additions and 87 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      libethereum/RLP.h
  2. 18
      libethereum/Trie.cpp
  3. 240
      libethereum/Trie.h
  4. 130
      test/main.cpp

2
libethereum/RLP.h
View File

@ -215,7 +215,7 @@ public:
template <class _N> _N toHash(int _flags = Strict) const
{
if (!isString() || (items() >= _N::size && (_flags & FailIfTooBig)))
if (!isString() || (items() > _N::size && (_flags & FailIfTooBig)))
if (_flags & ThrowOnFail)
throw BadCast();
else

18
libethereum/Trie.cpp
View File

@ -713,9 +713,9 @@ std::string hexPrefixEncode(bytesConstRef _data, bool _terminated, int _beginNib
std::string hexPrefixEncode(bytesConstRef _d1, uint _o1, bytesConstRef _d2, uint _o2, bool _terminated)
{
uint begin1 = _o1;
uint end1 = _d1.size() * 2 - _o1;
uint end1 = _d1.size() * 2;
uint begin2 = _o2;
uint end2 = _d2.size() * 2 - _o2;
uint end2 = _d2.size() * 2;
bool odd = (end1 - begin1 + end2 - begin2) & 1;
@ -767,4 +767,18 @@ uint NibbleSlice::shared(NibbleSlice _k) const
return sharedNibbles(data, offset, offset + size(), _k.data, _k.offset, _k.offset + _k.size());
}
byte uniqueInUse(RLP const& _orig, byte _except)
{
byte used = 255;
for (unsigned i = 0; i < 17; ++i)
if (i != _except && !_orig[i].isEmpty())
{
if (used == 255)
used = i;
else
return 255;
}
return used;
}
}

240
libethereum/Trie.h
View File

@ -117,6 +117,7 @@ class BasicMap
public:
BasicMap() {}
void clear() { m_over.clear(); }
std::map<h256, std::string> const& get() const { return m_over; }
std::string lookup(h256 _h) const { auto it = m_over.find(_h); if (it != m_over.end()) return it->second; return std::string(); }
@ -154,6 +155,8 @@ public:
std::string lookup(h256 _h) const { std::string ret = BasicMap::lookup(_h); if (ret.empty()) m_db->Get(m_readOptions, ldb::Slice((char const*)_h.data(), 32), &ret); return ret; }
private:
using BasicMap::clear;
ldb::DB* m_db = nullptr;
ldb::ReadOptions m_readOptions;
@ -197,42 +200,53 @@ private:
void mergeAtAux(RLPStream& _out, RLP const& _replace, NibbleSlice _key, bytesConstRef _value);
bytes mergeAt(RLP const& _replace, NibbleSlice _k, bytesConstRef _v);
// in1: null (DEL) -- OR -- [_k, V] (DEL)
// out1: [_k, _s]
// in2: [V0, ..., V15, S16] (DEL) AND _k == {}
// out2: [V0, ..., V15, _s]
bool deleteAtAux(RLPStream& _out, RLP const& _replace, NibbleSlice _key);
bytes deleteAt(RLP const& _replace, NibbleSlice _k);
// in: null (DEL) -- OR -- [_k, V] (DEL)
// out: [_k, _s]
// -- OR --
// in: [V0, ..., V15, S16] (DEL) AND _k == {}
// out: [V0, ..., V15, _s]
bytes place(RLP const& _orig, NibbleSlice _k, bytesConstRef _s);
// in1: [K, S] (DEL)
// out1: null
// in2: [V0, ..., V15, S] (DEL)
// out2: [V0, ..., V15, null]
// in: [K, S] (DEL)
// out: null
// -- OR --
// in: [V0, ..., V15, S] (DEL)
// out: [V0, ..., V15, null]
bytes remove(RLP const& _orig);
// in: [K1 & K2, V] (DEL) : nibbles(K1) == _s, 0 < _s < nibbles(K1 & K2)
// out: [K1, H] ; [K2, V] => H (INS) (being [K1, [K2, V]] (INS) if necessary)
// in: [K1 & K2, V] (DEL) : nibbles(K1) == _s, 0 < _s <= nibbles(K1 & K2)
// out: [K1, H] ; [K2, V] => H (INS) (being [K1, [K2, V]] if necessary)
bytes cleve(RLP const& _orig, uint _s);
// in: [K1, H] (DEL) ; H <= [K2, V] (DEL) (being [K1, [K2, V]] (DEL) if necessary)
// out: [K1 & K2, V]
bytes graft(RLP const& _orig);
// in: [V0, ... V15, S] (DEL) : (exists unique i: !!Vi AND !S "out1") OR (all i: !Vi AND !!S "out2")
// out1: [k{i}, Vi]
// out2: [k{}, S]
bytes merge(RLP const& _orig);
// in: [V0, ... V15, S] (DEL)
// out1: [k{i}, Vi] where i < 16
// out2: [k{}, S] where i == 16
bytes merge(RLP const& _orig, byte _i);
// in: [k{}, S] (DEL)
// out: [null ** 16, S]
// -- OR --
// in: [k{i}, V] (DEL)
// out: [null ** i, V, null ** (16 - i)]
// in: [k{i}, N] (DEL)
// out: [null ** i, N, null ** (16 - i)]
// -- OR --
// in: [k{i}K, V] (DEL)
// out: [null ** i, H, null ** (16 - i)] ; [K, V] => H (INS) (being [null ** i, [K, V], null ** (16 - i)] if necessary)
bytes branch(RLP const& _orig);
bool isTwoItemNode(RLP const& _n) const;
std::string node(h256 _h) const { return m_db->lookup(_h); }
void insertNode(h256 _h, bytesConstRef _v) { m_db->insert(_h, _v); }
void killNode(h256 _h) { m_db->kill(_h); }
h256 insertNode(bytesConstRef _v) { auto h = sha3(_v); insertNode(h, _v); return h; }
void killNode(RLP const& _d) { if (_d.data().size() >= 32) killNode(sha3(_d.data())); }
@ -263,6 +277,7 @@ namespace eth
uint sharedNibbles(bytesConstRef _a, uint _ab, uint _ae, bytesConstRef _b, uint _bb, uint _be);
bool isLeaf(RLP const& _twoItem);
byte uniqueInUse(RLP const& _orig, byte _except);
NibbleSlice keyOf(RLP const& _twoItem);
std::string hexPrefixEncode(bytesConstRef _data, bool _terminated, int _beginNibble, int _endNibble, uint _offset);
std::string hexPrefixEncode(bytesConstRef _d1, uint _o1, bytesConstRef _d2, uint _o2, bool _terminated);
@ -277,14 +292,14 @@ template <class DB> void GenericTrieDB<DB>::init()
template <class DB> void GenericTrieDB<DB>::insert(bytesConstRef _key, bytesConstRef _value)
{
std::string rv = node(m_root);
killNode(m_root);
bytes b = mergeAt(RLP(rv), NibbleSlice(_key), _value);
m_root = insertNode(&b);
}
template <class DB> void GenericTrieDB<DB>::remove(bytesConstRef _key)
{
// mergeAt won't attempt to delete the node is it's less than 32 bytes
// However, we know it's the root node and thus always hashed.
// So, if it's less than 32 (and thus should have been deleted but wasn't) then we delete it here.
if (rv.size() < 32)
killNode(m_root);
m_root = insertNode(&b);
}
template <class DB> std::string GenericTrieDB<DB>::at(bytesConstRef _key) const
@ -325,6 +340,8 @@ template <class DB> std::string GenericTrieDB<DB>::atAux(RLP const& _here, Nibbl
template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSlice _k, bytesConstRef _v)
{
// ::operator<<(std::cout << "mergeAt ", _orig) << _k << _v.toString() << std::endl;
// The caller will make sure that the bytes are inserted properly.
// - This might mean inserting an entry into m_over
// We will take care to ensure that (our reference to) _orig is killed.
@ -344,7 +361,7 @@ template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSli
return place(_orig, _k, _v);
// partial key is our key - move down.
if (_k.contains(k))
if (_k.contains(k) && !isLeaf(_orig))
{
killNode(sha3(_orig.data()));
RLPStream s(2);
@ -354,12 +371,12 @@ template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSli
}
auto sh = _k.shared(k);
// std::cout << _k << " sh " << k << " = " << sh << std::endl;
// 5 << _k << " sh " << k << " = " << sh << std::endl;
if (sh)
// shared stuff - cleve at disagreement.
return mergeAt(RLP(cleve(_orig, sh)), _k, _v);
else
// nothing shared - if we can branch, branch, otherwise cleve again.
// nothing shared - branch
return mergeAt(RLP(branch(_orig)), _k, _v);
}
else
@ -388,17 +405,155 @@ template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSli
template <class DB> void GenericTrieDB<DB>::mergeAtAux(RLPStream& _out, RLP const& _orig, NibbleSlice _k, bytesConstRef _v)
{
bytes b = mergeAt(_orig, _k, _v);
killNode(_orig);
RLP r = _orig;
std::string s;
if (!r.isList() && !r.isEmpty())
{
s = node(_orig.toHash<h256>());
r = RLP(s);
assert(!r.isNull());
killNode(_orig.toHash<h256>());
}
else
killNode(_orig);
bytes b = mergeAt(r, _k, _v);
// ::operator<<(std::cout, RLP(b)) << std::endl;
streamNode(_out, b);
}
template <class DB> void GenericTrieDB<DB>::remove(bytesConstRef _key)
{
std::string rv = node(m_root);
bytes b = deleteAt(RLP(rv), NibbleSlice(_key));
if (b.size())
{
if (rv.size() < 32)
killNode(m_root);
m_root = insertNode(&b);
}
}
template <class DB> bool GenericTrieDB<DB>::isTwoItemNode(RLP const& _n) const
{
return (_n.isString() && RLP(node(_n.toHash<h256>())).itemCount() == 2)
|| (_n.isList() && _n.itemCount() == 2);
}
template <class DB> bytes GenericTrieDB<DB>::deleteAt(RLP const& _orig, NibbleSlice _k)
{
// The caller will make sure that the bytes are inserted properly.
// - This might mean inserting an entry into m_over
// We will take care to ensure that (our reference to) _orig is killed.
// Empty - not found - no change.
if (_orig.isEmpty())
return bytes();
assert(_orig.isList() && (_orig.itemCount() == 2 || _orig.itemCount() == 17));
if (_orig.itemCount() == 2)
{
// pair...
NibbleSlice k = keyOf(_orig);
// exactly our node - return null.
if (k == _k && isLeaf(_orig))
return RLPNull;
// partial key is our key - move down.
if (_k.contains(k))
{
RLPStream s;
s.appendList(2) << _orig[0];
if (!deleteAtAux(s, _orig[1], _k.mid(k.size())))
return bytes();
killNode(sha3(_orig.data()));
RLP r(s.out());
if (isTwoItemNode(r[1]))
return graft(r);
return s.out();
}
else
// not found - no change.
return bytes();
}
else
{
// branch...
// exactly our node - remove and rejig.
if (_k.size() == 0 && !_orig[16].isEmpty())
{
// Kill the node.
killNode(sha3(_orig.data()));
byte used = uniqueInUse(_orig, 16);
if (used != 255)
if (_orig[used].isList() && _orig[used].itemCount() == 2)
return graft(RLP(merge(_orig, used)));
else
return merge(_orig, used);
else
{
RLPStream r(17);
for (byte i = 0; i < 16; ++i)
r << _orig[i];
r << "";
return r.out();
}
}
else
{
// not exactly our node - delve to next level at the correct index.
RLPStream r(17);
byte n = _k[0];
for (byte i = 0; i < 17; ++i)
if (i == n)
if (!deleteAtAux(r, _orig[i], _k.mid(1))) // bomb out if the key didn't turn up.
return bytes();
else {}
else
r << _orig[i];
// check if we ended up leaving the node invalid.
RLP rlp(r.out());
byte used = uniqueInUse(rlp, 255);
if (used == 255) // no - all ok.
return r.out();
// yes; merge
if (isTwoItemNode(rlp[used]))
return graft(RLP(merge(rlp, used)));
else
return merge(rlp, used);
}
}
}
template <class DB> bool GenericTrieDB<DB>::deleteAtAux(RLPStream& _out, RLP const& _orig, NibbleSlice _k)
{
bytes b = deleteAt(_orig.isList() ? _orig : RLP(node(_orig.toHash<h256>())), _k);
if (!b.size()) // not found - no change.
return false;
if (_orig.isList())
killNode(_orig);
else
killNode(_orig.toHash<h256>());
streamNode(_out, b);
return true;
}
// in1: null -- OR -- [_k, V]
// out1: [_k, _s]
// in2: [V0, ..., V15, S16] AND _k == {}
// out2: [V0, ..., V15, _s]
template <class DB> bytes GenericTrieDB<DB>::place(RLP const& _orig, NibbleSlice _k, bytesConstRef _s)
{
// ::operator<<(std::cout << "place ", _orig) << ", " << _k << ", " << _s.toString() << std::endl;
killNode(_orig);
if (_orig.isEmpty())
return RLPStream(2).appendString(hexPrefixEncode(_k, true)).appendString(_s).out();
@ -441,14 +596,16 @@ template <class DB> RLPStream& GenericTrieDB<DB>::streamNode(RLPStream& _s, byte
return _s;
}
// in: [K1 & K2, V] (DEL) : nibbles(K1) == _s, 0 < _s < nibbles(K1 & K2)
// in: [K1 & K2, V] (DEL) : nibbles(K1) == _s, 0 < _s <= nibbles(K1 & K2)
// out: [K1, H] (INS) ; [K2, V] => H (INS) (being [K1, [K2, V]] if necessary)
template <class DB> bytes GenericTrieDB<DB>::cleve(RLP const& _orig, uint _s)
{
// ::operator<<(std::cout << "cleve ", _orig) << ", " << _s << std::endl;
killNode(_orig);
assert(_orig.isList() && _orig.itemCount() == 2);
auto k = keyOf(_orig);
assert(_s && _s < k.size());
assert(_s && _s <= k.size());
RLPStream bottom(2);
bottom.appendString(hexPrefixEncode(k, isLeaf(_orig), _s));
@ -480,30 +637,27 @@ template <class DB> bytes GenericTrieDB<DB>::graft(RLP const& _orig)
assert(n.itemCount() == 2);
return (RLPStream(2) << hexPrefixEncode(keyOf(_orig), keyOf(n), isLeaf(n)) << n[1]).out();
// auto ret =
// std::cout << keyOf(_orig) << " ++ " << keyOf(n) << " == " << keyOf(RLP(ret)) << std::endl;
// return ret;
}
// in: [V0, ... V15, S] (DEL) : (exists unique i: !!Vi AND !S "out1") OR (all i: !Vi AND !!S "out2")
// out1: [k{i}, Vi] (INS)
// out2: [k{}, S] (INS)
template <class DB> bytes GenericTrieDB<DB>::merge(RLP const& _orig)
template <class DB> bytes GenericTrieDB<DB>::merge(RLP const& _orig, byte _i)
{
assert(_orig.isList() && _orig.itemCount() == 17);
RLPStream s(2);
if (_orig[16].isEmpty())
if (_i != 16)
{
for (byte i = 0; i < 16; ++i)
if (!_orig[i].isEmpty())
{
s << hexPrefixEncode(bytesConstRef(&i, 1), false, 1, 2, 0) << _orig[i];
return s.out();
}
assert(false);
assert(!_orig[_i].isEmpty());
s << hexPrefixEncode(bytesConstRef(&_i, 1), false, 1, 2, 0);
}
else
{
s << hexPrefixEncode(bytes(), true) << _orig[16];
return s.out();
}
s << hexPrefixEncode(bytes(), true);
s << _orig[_i];
return s.out();
}
// in: [k{}, S] (DEL)
@ -516,6 +670,8 @@ template <class DB> bytes GenericTrieDB<DB>::merge(RLP const& _orig)
// out: [null ** i, N, null ** (16 - i)] (INS)
template <class DB> bytes GenericTrieDB<DB>::branch(RLP const& _orig)
{
// ::operator<<(std::cout << "branch ", _orig) << std::endl;
assert(_orig.isList() && _orig.itemCount() == 2);
auto k = keyOf(_orig);

130
test/main.cpp
View File

@ -122,15 +122,28 @@ int main()
cout << t.root() << endl;
cout << hash256(StringMap()) << endl;
t.insert(string("test"), string("test"));
t.insert(string("tesz"), string("test"));
cout << m;
cout << t.root() << endl;
cout << hash256({{"test", "test"}}) << endl;
t.insert(string("te"), string("test"));
t.insert(string("tesa"), string("testy"));
cout << m;
cout << t.root() << endl;
cout << hash256({{"test", "test"}, {"te", "test"}}) << endl;
cout << hash256({{"test", "test"}, {"te", "testy"}}) << endl;
cout << t.at(string("test")) << endl;
cout << t.at(string("te")) << endl;
cout << t.at(string("t")) << endl;
t.remove(string("te"));
cout << m;
cout << t.root() << endl;
cout << hash256({{"test", "test"}}) << endl;
t.remove(string("test"));
cout << m;
cout << t.root() << endl;
cout << hash256(StringMap()) << endl;
}
{
BasicMap m;
@ -143,10 +156,6 @@ int main()
cout << hash256({{"b", "B"}, {"a", "A"}}) << endl;
cout << RLP(rlp256({{"b", "B"}, {"a", "A"}})) << endl;
}
return 0;
cout << escaped(asString(rlp256({{"b", "B"}, {"a", "A"}})), false) << " == " << RLP(rlp256({{"b", "B"}, {"a", "A"}})) << endl;
cout << escaped(asString(rlp256({{"test", "test"}})), false) << " == " << RLP(rlp256({{"test", "test"}})) << endl;
cout << asHex(rlp256({{"test", "test"}, {"te", "test"}})) << endl;
{
Trie t;
t.insert("dog", "puppy");
@ -170,58 +179,95 @@ int main()
cout << asHex(t.rlp()) << endl;
}
{
BasicMap m;
GenericTrieDB<BasicMap> d(&m);
d.init(); // initialise as empty tree.
Trie t;
StringMap s;
t.insert("dog", "puppy");
assert(t.hash256() == hash256({{"dog", "puppy"}}));
assert(t.at("dog") == "puppy");
t.insert("doe", "reindeer");
assert(t.hash256() == hash256({{"dog", "puppy"}, {"doe", "reindeer"}}));
assert(t.at("doe") == "reindeer");
assert(t.at("dog") == "puppy");
t.insert("dogglesworth", "cat");
assert(t.hash256() == hash256({{"doe", "reindeer"}, {"dog", "puppy"}, {"dogglesworth", "cat"}}));
assert(t.at("doe") == "reindeer");
assert(t.at("dog") == "puppy");
assert(t.at("dogglesworth") == "cat");
t.remove("dogglesworth");
t.remove("doe");
assert(t.at("doe").empty());
assert(t.at("dogglesworth").empty());
assert(t.at("dog") == "puppy");
assert(t.hash256() == hash256({{"dog", "puppy"}}));
t.insert("horse", "stallion");
t.insert("do", "verb");
t.insert("doge", "coin");
assert(t.hash256() == hash256({{"dog", "puppy"}, {"horse", "stallion"}, {"do", "verb"}, {"doge", "coin"}}));
assert(t.at("doge") == "coin");
assert(t.at("do") == "verb");
assert(t.at("horse") == "stallion");
assert(t.at("dog") == "puppy");
t.remove("horse");
t.remove("do");
t.remove("doge");
assert(t.hash256() == hash256({{"dog", "puppy"}}));
assert(t.at("dog") == "puppy");
t.remove("dog");
for (int a = 0; a < 20; ++a)
auto add = [&](char const* a, char const* b)
{
d.insert(string(a), string(b));
t.insert(a, b);
s[a] = b;
cout << endl << "-------------------------------" << endl;
cout << a << " -> " << b << endl;
cout << m;
cout << d.root() << endl;
cout << hash256(s) << endl;
assert(t.hash256() == hash256(s));
assert(d.root() == hash256(s));
for (auto const& i: s)
{
assert(t.at(i.first) == i.second);
assert(d.at(i.first) == i.second);
}
};
auto remove = [&](char const* a)
{
s.erase(a);
t.remove(a);
d.remove(string(a));
cout << endl << "-------------------------------" << endl;
cout << "X " << a << endl;
cout << m;
cout << d.root() << endl;
cout << hash256(s) << endl;
assert(t.at(a).empty());
assert(d.at(string(a)).empty());
assert(t.hash256() == hash256(s));
assert(d.root() == hash256(s));
for (auto const& i: s)
{
assert(t.at(i.first) == i.second);
assert(d.at(i.first) == i.second);
}
};
add("doe", "reindeer");
add("do", "verb");
add("doge", "coin");
m.clear();
d.init();
t.remove("doe"); t.remove("do"); t.remove("doge");
s.clear();
add("dogglesworth", "cat");
add("doe", "reindeer");
remove("dogglesworth");
add("horse", "stallion");
add("do", "verb");
add("doge", "coin");
remove("horse");
remove("do");
remove("doge");
remove("doe");
for (int a = 0; a < 200; ++a)
{
StringMap m;
for (int i = 0; i < 20; ++i)
for (int i = 0; i < 200; ++i)
{
auto k = randomWord();
auto v = toString(i);
m.insert(make_pair(k, v));
t.insert(k, v);
d.insert(k, v);
assert(hash256(m) == t.hash256());
assert(hash256(m) == d.root());
}
while (!m.empty())
{
auto k = m.begin()->first;
d.remove(k);
t.remove(k);
m.erase(k);
assert(hash256(m) == t.hash256());
assert(hash256(m) == d.root());
}
}
}

Write Preview
Loading…
Cancel
Save