TrieDB remove nodes and tests.

libethereum/RLP.h

@@ -215,7 +215,7 @@ public:
 
 	template <class _N> _N toHash(int _flags = Strict) const
 	{
-		if (!isString() || (items() >= _N::size && (_flags & FailIfTooBig)))
+		if (!isString() || (items() > _N::size && (_flags & FailIfTooBig)))
 			if (_flags & ThrowOnFail)
 				throw BadCast();
 			else

libethereum/Trie.cpp

@@ -713,9 +713,9 @@ std::string hexPrefixEncode(bytesConstRef _data, bool _terminated, int _beginNib
 std::string hexPrefixEncode(bytesConstRef _d1, uint _o1, bytesConstRef _d2, uint _o2, bool _terminated)
 {
 	uint begin1 = _o1;
-	uint end1 = _d1.size() * 2 - _o1;
+	uint end1 = _d1.size() * 2;
 	uint begin2 = _o2;
-	uint end2 = _d2.size() * 2 - _o2;
+	uint end2 = _d2.size() * 2;
 
 	bool odd = (end1 - begin1 + end2 - begin2) & 1;
 
@@ -767,4 +767,18 @@ uint NibbleSlice::shared(NibbleSlice _k) const
 	return sharedNibbles(data, offset, offset + size(), _k.data, _k.offset, _k.offset + _k.size());
 }
 
+byte uniqueInUse(RLP const& _orig, byte _except)
+{
+	byte used = 255;
+	for (unsigned i = 0; i < 17; ++i)
+		if (i != _except && !_orig[i].isEmpty())
+		{
+			if (used == 255)
+				used = i;
+			else
+				return 255;
+		}
+	return used;
+}
+
 }

libethereum/Trie.h

@@ -117,6 +117,7 @@ class BasicMap
 public:
 	BasicMap() {}
 
+	void clear() { m_over.clear(); }
 	std::map<h256, std::string> const& get() const { return m_over; }
 
 	std::string lookup(h256 _h) const { auto it = m_over.find(_h); if (it != m_over.end()) return it->second; return std::string(); }
@@ -154,6 +155,8 @@ public:
 	std::string lookup(h256 _h) const { std::string ret = BasicMap::lookup(_h); if (ret.empty()) m_db->Get(m_readOptions, ldb::Slice((char const*)_h.data(), 32), &ret); return ret; }
 
 private:
+	using BasicMap::clear;
+
 	ldb::DB* m_db = nullptr;
 
 	ldb::ReadOptions m_readOptions;
@@ -197,42 +200,53 @@ private:
 	void mergeAtAux(RLPStream& _out, RLP const& _replace, NibbleSlice _key, bytesConstRef _value);
 	bytes mergeAt(RLP const& _replace, NibbleSlice _k, bytesConstRef _v);
 
-	// in1: null (DEL)  -- OR --  [_k, V] (DEL)
+	bool deleteAtAux(RLPStream& _out, RLP const& _replace, NibbleSlice _key);
-	// out1: [_k, _s]
+	bytes deleteAt(RLP const& _replace, NibbleSlice _k);
-	// in2: [V0, ..., V15, S16] (DEL)  AND  _k == {}
+
-	// out2: [V0, ..., V15, _s]
+	// in: null (DEL)  -- OR --  [_k, V] (DEL)
+	// out: [_k, _s]
+	// -- OR --
+	// in: [V0, ..., V15, S16] (DEL)  AND  _k == {}
+	// out: [V0, ..., V15, _s]
 	bytes place(RLP const& _orig, NibbleSlice _k, bytesConstRef _s);
 
-	// in1: [K, S] (DEL)
+	// in: [K, S] (DEL)
-	// out1: null
+	// out: null
-	// in2: [V0, ..., V15, S] (DEL)
+	// -- OR --
-	// out2: [V0, ..., V15, null]
+	// in: [V0, ..., V15, S] (DEL)
+	// out: [V0, ..., V15, null]
 	bytes remove(RLP const& _orig);
 
-	// in: [K1 & K2, V] (DEL) : nibbles(K1) == _s, 0 < _s < nibbles(K1 & K2)
+	// in: [K1 & K2, V] (DEL) : nibbles(K1) == _s, 0 < _s <= nibbles(K1 & K2)
-	// out: [K1, H] ; [K2, V] => H (INS)  (being  [K1, [K2, V]] (INS)  if necessary)
+	// out: [K1, H] ; [K2, V] => H (INS)  (being  [K1, [K2, V]]  if necessary)
 	bytes cleve(RLP const& _orig, uint _s);
 
 	// in: [K1, H] (DEL) ; H <= [K2, V] (DEL)  (being  [K1, [K2, V]] (DEL)  if necessary)
 	// out: [K1 & K2, V]
 	bytes graft(RLP const& _orig);
 
-	// in: [V0, ... V15, S] (DEL) : (exists unique i: !!Vi  AND  !S  "out1") OR (all i: !Vi  AND  !!S  "out2")
+	// in: [V0, ... V15, S] (DEL)
-	// out1: [k{i}, Vi]
+	// out1: [k{i}, Vi]    where i < 16
-	// out2: [k{}, S]
+	// out2: [k{}, S]      where i == 16
-	bytes merge(RLP const& _orig);
+	bytes merge(RLP const& _orig, byte _i);
 
 	// in: [k{}, S] (DEL)
 	// out: [null ** 16, S]
 	// -- OR --
-	// in: [k{i}, V] (DEL)
+	// in: [k{i}, N] (DEL)
-	// out: [null ** i, V, null ** (16 - i)]
+	// out: [null ** i, N, null ** (16 - i)]
+	// -- OR --
+	// in: [k{i}K, V] (DEL)
+	// out: [null ** i, H, null ** (16 - i)] ; [K, V] => H (INS)  (being [null ** i, [K, V], null ** (16 - i)]  if necessary)
 	bytes branch(RLP const& _orig);
 
+	bool isTwoItemNode(RLP const& _n) const;
+
 	std::string node(h256 _h) const { return m_db->lookup(_h); }
 	void insertNode(h256 _h, bytesConstRef _v) { m_db->insert(_h, _v); }
 	void killNode(h256 _h) { m_db->kill(_h); }
 
+
 	h256 insertNode(bytesConstRef _v) { auto h = sha3(_v); insertNode(h, _v); return h; }
 	void killNode(RLP const& _d) { if (_d.data().size() >= 32) killNode(sha3(_d.data())); }
 
@@ -263,6 +277,7 @@ namespace eth
 
 uint sharedNibbles(bytesConstRef _a, uint _ab, uint _ae, bytesConstRef _b, uint _bb, uint _be);
 bool isLeaf(RLP const& _twoItem);
+byte uniqueInUse(RLP const& _orig, byte _except);
 NibbleSlice keyOf(RLP const& _twoItem);
 std::string hexPrefixEncode(bytesConstRef _data, bool _terminated, int _beginNibble, int _endNibble, uint _offset);
 std::string hexPrefixEncode(bytesConstRef _d1, uint _o1, bytesConstRef _d2, uint _o2, bool _terminated);
@@ -277,14 +292,14 @@ template <class DB> void GenericTrieDB<DB>::init()
 template <class DB> void GenericTrieDB<DB>::insert(bytesConstRef _key, bytesConstRef _value)
 {
 	std::string rv = node(m_root);
-	killNode(m_root);
-
 	bytes b = mergeAt(RLP(rv), NibbleSlice(_key), _value);
-	m_root = insertNode(&b);
-}
 
-template <class DB> void GenericTrieDB<DB>::remove(bytesConstRef _key)
+	// mergeAt won't attempt to delete the node is it's less than 32 bytes
-{
+	// However, we know it's the root node and thus always hashed.
+	// So, if it's less than 32 (and thus should have been deleted but wasn't) then we delete it here.
+	if (rv.size() < 32)
+		killNode(m_root);
+	m_root = insertNode(&b);
 }
 
 template <class DB> std::string GenericTrieDB<DB>::at(bytesConstRef _key) const
@@ -325,6 +340,8 @@ template <class DB> std::string GenericTrieDB<DB>::atAux(RLP const& _here, Nibbl
 
 template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSlice _k, bytesConstRef _v)
 {
+//	::operator<<(std::cout << "mergeAt ", _orig) << _k << _v.toString() << std::endl;
+
 	// The caller will make sure that the bytes are inserted properly.
 	// - This might mean inserting an entry into m_over
 	// We will take care to ensure that (our reference to) _orig is killed.
@@ -344,7 +361,7 @@ template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSli
 			return place(_orig, _k, _v);
 
 		// partial key is our key - move down.
-		if (_k.contains(k))
+		if (_k.contains(k) && !isLeaf(_orig))
 		{
 			killNode(sha3(_orig.data()));
 			RLPStream s(2);
@@ -354,12 +371,12 @@ template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSli
 		}
 
 		auto sh = _k.shared(k);
-//		std::cout << _k << " sh " << k << " = " << sh << std::endl;
+//		5 << _k << " sh " << k << " = " << sh << std::endl;
 		if (sh)
 			// shared stuff - cleve at disagreement.
 			return mergeAt(RLP(cleve(_orig, sh)), _k, _v);
 		else
-			// nothing shared - if we can branch, branch, otherwise cleve again.
+			// nothing shared - branch
 			return mergeAt(RLP(branch(_orig)), _k, _v);
 	}
 	else
@@ -388,17 +405,155 @@ template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSli
 
 template <class DB> void GenericTrieDB<DB>::mergeAtAux(RLPStream& _out, RLP const& _orig, NibbleSlice _k, bytesConstRef _v)
 {
-	bytes b = mergeAt(_orig, _k, _v);
+	RLP r = _orig;
-	killNode(_orig);
+	std::string s;
+	if (!r.isList() && !r.isEmpty())
+	{
+		s = node(_orig.toHash<h256>());
+		r = RLP(s);
+		assert(!r.isNull());
+		killNode(_orig.toHash<h256>());
+	}
+	else
+		killNode(_orig);
+	bytes b = mergeAt(r, _k, _v);
+//	::operator<<(std::cout, RLP(b)) << std::endl;
 	streamNode(_out, b);
 }
 
+template <class DB> void GenericTrieDB<DB>::remove(bytesConstRef _key)
+{
+	std::string rv = node(m_root);
+	bytes b = deleteAt(RLP(rv), NibbleSlice(_key));
+	if (b.size())
+	{
+		if (rv.size() < 32)
+			killNode(m_root);
+		m_root = insertNode(&b);
+	}
+}
+
+template <class DB> bool GenericTrieDB<DB>::isTwoItemNode(RLP const& _n) const
+{
+	return (_n.isString() && RLP(node(_n.toHash<h256>())).itemCount() == 2)
+			|| (_n.isList() && _n.itemCount() == 2);
+}
+
+template <class DB> bytes GenericTrieDB<DB>::deleteAt(RLP const& _orig, NibbleSlice _k)
+{
+	// The caller will make sure that the bytes are inserted properly.
+	// - This might mean inserting an entry into m_over
+	// We will take care to ensure that (our reference to) _orig is killed.
+
+	// Empty - not found - no change.
+	if (_orig.isEmpty())
+		return bytes();
+
+	assert(_orig.isList() && (_orig.itemCount() == 2 || _orig.itemCount() == 17));
+	if (_orig.itemCount() == 2)
+	{
+		// pair...
+		NibbleSlice k = keyOf(_orig);
+
+		// exactly our node - return null.
+		if (k == _k && isLeaf(_orig))
+			return RLPNull;
+
+		// partial key is our key - move down.
+		if (_k.contains(k))
+		{
+			RLPStream s;
+			s.appendList(2) << _orig[0];
+			if (!deleteAtAux(s, _orig[1], _k.mid(k.size())))
+				return bytes();
+			killNode(sha3(_orig.data()));
+			RLP r(s.out());
+			if (isTwoItemNode(r[1]))
+				return graft(r);
+			return s.out();
+		}
+		else
+			// not found - no change.
+			return bytes();
+	}
+	else
+	{
+		// branch...
+
+		// exactly our node - remove and rejig.
+		if (_k.size() == 0 && !_orig[16].isEmpty())
+		{
+			// Kill the node.
+			killNode(sha3(_orig.data()));
+
+			byte used = uniqueInUse(_orig, 16);
+			if (used != 255)
+				if (_orig[used].isList() && _orig[used].itemCount() == 2)
+					return graft(RLP(merge(_orig, used)));
+				else
+					return merge(_orig, used);
+			else
+			{
+				RLPStream r(17);
+				for (byte i = 0; i < 16; ++i)
+					r << _orig[i];
+				r << "";
+				return r.out();
+			}
+		}
+		else
+		{
+			// not exactly our node - delve to next level at the correct index.
+			RLPStream r(17);
+			byte n = _k[0];
+			for (byte i = 0; i < 17; ++i)
+				if (i == n)
+					if (!deleteAtAux(r, _orig[i], _k.mid(1)))	// bomb out if the key didn't turn up.
+						return bytes();
+					else {}
+				else
+					r << _orig[i];
+
+			// check if we ended up leaving the node invalid.
+			RLP rlp(r.out());
+			byte used = uniqueInUse(rlp, 255);
+			if (used == 255)	// no - all ok.
+				return r.out();
+
+			// yes; merge
+			if (isTwoItemNode(rlp[used]))
+				return graft(RLP(merge(rlp, used)));
+			else
+				return merge(rlp, used);
+		}
+	}
+
+}
+
+template <class DB> bool GenericTrieDB<DB>::deleteAtAux(RLPStream& _out, RLP const& _orig, NibbleSlice _k)
+{
+	bytes b = deleteAt(_orig.isList() ? _orig : RLP(node(_orig.toHash<h256>())), _k);
+
+	if (!b.size())	// not found - no change.
+		return false;
+
+	if (_orig.isList())
+		killNode(_orig);
+	else
+		killNode(_orig.toHash<h256>());
+
+	streamNode(_out, b);
+	return true;
+}
+
 // in1: null  -- OR --  [_k, V]
 // out1: [_k, _s]
 // in2: [V0, ..., V15, S16]  AND  _k == {}
 // out2: [V0, ..., V15, _s]
 template <class DB> bytes GenericTrieDB<DB>::place(RLP const& _orig, NibbleSlice _k, bytesConstRef _s)
 {
+//	::operator<<(std::cout << "place ", _orig) << ", " << _k << ", " << _s.toString() << std::endl;
+
 	killNode(_orig);
 	if (_orig.isEmpty())
 		return RLPStream(2).appendString(hexPrefixEncode(_k, true)).appendString(_s).out();
@@ -441,14 +596,16 @@ template <class DB> RLPStream& GenericTrieDB<DB>::streamNode(RLPStream& _s, byte
 	return _s;
 }
 
-// in: [K1 & K2, V] (DEL) : nibbles(K1) == _s, 0 < _s < nibbles(K1 & K2)
+// in: [K1 & K2, V] (DEL) : nibbles(K1) == _s, 0 < _s <= nibbles(K1 & K2)
 // out: [K1, H] (INS) ; [K2, V] => H (INS)  (being  [K1, [K2, V]]  if necessary)
 template <class DB> bytes GenericTrieDB<DB>::cleve(RLP const& _orig, uint _s)
 {
+//	::operator<<(std::cout << "cleve ", _orig) << ", " << _s << std::endl;
+
 	killNode(_orig);
 	assert(_orig.isList() && _orig.itemCount() == 2);
 	auto k = keyOf(_orig);
-	assert(_s && _s < k.size());
+	assert(_s && _s <= k.size());
 
 	RLPStream bottom(2);
 	bottom.appendString(hexPrefixEncode(k, isLeaf(_orig), _s));
@@ -480,30 +637,27 @@ template <class DB> bytes GenericTrieDB<DB>::graft(RLP const& _orig)
 	assert(n.itemCount() == 2);
 
 	return (RLPStream(2) << hexPrefixEncode(keyOf(_orig), keyOf(n), isLeaf(n)) << n[1]).out();
+//	auto ret =
+//	std::cout << keyOf(_orig) << " ++ " << keyOf(n) << " == " << keyOf(RLP(ret)) << std::endl;
+//	return ret;
 }
 
 // in: [V0, ... V15, S] (DEL) : (exists unique i: !!Vi  AND  !S  "out1") OR (all i: !Vi  AND  !!S  "out2")
 // out1: [k{i}, Vi] (INS)
 // out2: [k{}, S] (INS)
-template <class DB> bytes GenericTrieDB<DB>::merge(RLP const& _orig)
+template <class DB> bytes GenericTrieDB<DB>::merge(RLP const& _orig, byte _i)
 {
 	assert(_orig.isList() && _orig.itemCount() == 17);
 	RLPStream s(2);
-	if (_orig[16].isEmpty())
+	if (_i != 16)
 	{
-		for (byte i = 0; i < 16; ++i)
+		assert(!_orig[_i].isEmpty());
-			if (!_orig[i].isEmpty())
+		s << hexPrefixEncode(bytesConstRef(&_i, 1), false, 1, 2, 0);
-			{
-				s << hexPrefixEncode(bytesConstRef(&i, 1), false, 1, 2, 0) << _orig[i];
-				return s.out();
-			}
-		assert(false);
 	}
 	else
-	{
+		s << hexPrefixEncode(bytes(), true);
-		s << hexPrefixEncode(bytes(), true) << _orig[16];
+	s << _orig[_i];
-		return s.out();
+	return s.out();
-	}
 }
 
 // in: [k{}, S] (DEL)
@@ -516,6 +670,8 @@ template <class DB> bytes GenericTrieDB<DB>::merge(RLP const& _orig)
 // out: [null ** i, N, null ** (16 - i)] (INS)
 template <class DB> bytes GenericTrieDB<DB>::branch(RLP const& _orig)
 {
+//	::operator<<(std::cout << "branch ", _orig) << std::endl;
+
 	assert(_orig.isList() && _orig.itemCount() == 2);
 
 	auto k = keyOf(_orig);

test/main.cpp

@@ -122,15 +122,28 @@ int main()
 		cout << t.root() << endl;
 		cout << hash256(StringMap()) << endl;
 
-		t.insert(string("test"), string("test"));
+		t.insert(string("tesz"), string("test"));
 		cout << m;
 		cout << t.root() << endl;
 		cout << hash256({{"test", "test"}}) << endl;
 
-		t.insert(string("te"), string("test"));
+		t.insert(string("tesa"), string("testy"));
 		cout << m;
 		cout << t.root() << endl;
-		cout << hash256({{"test", "test"}, {"te", "test"}}) << endl;
+		cout << hash256({{"test", "test"}, {"te", "testy"}}) << endl;
+		cout << t.at(string("test")) << endl;
+		cout << t.at(string("te")) << endl;
+		cout << t.at(string("t")) << endl;
+
+		t.remove(string("te"));
+		cout << m;
+		cout << t.root() << endl;
+		cout << hash256({{"test", "test"}}) << endl;
+
+		t.remove(string("test"));
+		cout << m;
+		cout << t.root() << endl;
+		cout << hash256(StringMap()) << endl;
 	}
 	{
 		BasicMap m;
@@ -143,10 +156,6 @@ int main()
 		cout << hash256({{"b", "B"}, {"a", "A"}}) << endl;
 		cout << RLP(rlp256({{"b", "B"}, {"a", "A"}})) << endl;
 	}
-	return 0;
-	cout << escaped(asString(rlp256({{"b", "B"}, {"a", "A"}})), false) << " == " << RLP(rlp256({{"b", "B"}, {"a", "A"}})) << endl;
-	cout << escaped(asString(rlp256({{"test", "test"}})), false) << " == " << RLP(rlp256({{"test", "test"}})) << endl;
-	cout << asHex(rlp256({{"test", "test"}, {"te", "test"}})) << endl;
 	{
 		Trie t;
 		t.insert("dog", "puppy");
@@ -170,58 +179,95 @@ int main()
 		cout << asHex(t.rlp()) << endl;
 	}
 	{
+		BasicMap m;
+		GenericTrieDB<BasicMap> d(&m);
+		d.init();	// initialise as empty tree.
 		Trie t;
+		StringMap s;
 
-		t.insert("dog", "puppy");
+		auto add = [&](char const* a, char const* b)
-		assert(t.hash256() == hash256({{"dog", "puppy"}}));
+		{
-		assert(t.at("dog") == "puppy");
+			d.insert(string(a), string(b));
-		t.insert("doe", "reindeer");
+			t.insert(a, b);
-		assert(t.hash256() == hash256({{"dog", "puppy"}, {"doe", "reindeer"}}));
+			s[a] = b;
-		assert(t.at("doe") == "reindeer");
+
-		assert(t.at("dog") == "puppy");
+			cout << endl << "-------------------------------" << endl;
-		t.insert("dogglesworth", "cat");
+			cout << a << " -> " << b << endl;
-		assert(t.hash256() == hash256({{"doe", "reindeer"}, {"dog", "puppy"}, {"dogglesworth", "cat"}}));
+			cout << m;
-		assert(t.at("doe") == "reindeer");
+			cout << d.root() << endl;
-		assert(t.at("dog") == "puppy");
+			cout << hash256(s) << endl;
-		assert(t.at("dogglesworth") == "cat");
+
-		t.remove("dogglesworth");
+			assert(t.hash256() == hash256(s));
-		t.remove("doe");
+			assert(d.root() == hash256(s));
-		assert(t.at("doe").empty());
+			for (auto const& i: s)
-		assert(t.at("dogglesworth").empty());
+			{
-		assert(t.at("dog") == "puppy");
+				assert(t.at(i.first) == i.second);
-		assert(t.hash256() == hash256({{"dog", "puppy"}}));
+				assert(d.at(i.first) == i.second);
-		t.insert("horse", "stallion");
+			}
-		t.insert("do", "verb");
+		};
-		t.insert("doge", "coin");
+
-		assert(t.hash256() == hash256({{"dog", "puppy"}, {"horse", "stallion"}, {"do", "verb"}, {"doge", "coin"}}));
+		auto remove = [&](char const* a)
-		assert(t.at("doge") == "coin");
+		{
-		assert(t.at("do") == "verb");
+			s.erase(a);
-		assert(t.at("horse") == "stallion");
+			t.remove(a);
-		assert(t.at("dog") == "puppy");
+			d.remove(string(a));
-		t.remove("horse");
+
-		t.remove("do");
+			cout << endl << "-------------------------------" << endl;
-		t.remove("doge");
+			cout << "X " << a << endl;
-		assert(t.hash256() == hash256({{"dog", "puppy"}}));
+			cout << m;
-		assert(t.at("dog") == "puppy");
+			cout << d.root() << endl;
-		t.remove("dog");
+			cout << hash256(s) << endl;
-
+
-		for (int a = 0; a < 20; ++a)
+			assert(t.at(a).empty());
+			assert(d.at(string(a)).empty());
+			assert(t.hash256() == hash256(s));
+			assert(d.root() == hash256(s));
+			for (auto const& i: s)
+			{
+				assert(t.at(i.first) == i.second);
+				assert(d.at(i.first) == i.second);
+			}
+		};
+
+		add("doe", "reindeer");
+		add("do", "verb");
+		add("doge", "coin");
+		m.clear();
+		d.init();
+		t.remove("doe"); t.remove("do"); t.remove("doge");
+		s.clear();
+		add("dogglesworth", "cat");
+		add("doe", "reindeer");
+		remove("dogglesworth");
+		add("horse", "stallion");
+		add("do", "verb");
+		add("doge", "coin");
+		remove("horse");
+		remove("do");
+		remove("doge");
+		remove("doe");
+
+		for (int a = 0; a < 200; ++a)
 		{
 			StringMap m;
-			for (int i = 0; i < 20; ++i)
+			for (int i = 0; i < 200; ++i)
 			{
 				auto k = randomWord();
 				auto v = toString(i);
 				m.insert(make_pair(k, v));
 				t.insert(k, v);
+				d.insert(k, v);
 				assert(hash256(m) == t.hash256());
+				assert(hash256(m) == d.root());
 			}
 			while (!m.empty())
 			{
 				auto k = m.begin()->first;
+				d.remove(k);
 				t.remove(k);
 				m.erase(k);
 				assert(hash256(m) == t.hash256());
+				assert(hash256(m) == d.root());
 			}
 		}
 	}