From e19ac5054c6331ffcec05e553087409d9235c53b Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 18 May 2015 22:14:03 +0200 Subject: [PATCH 1/2] use TrieHash for transactions and receipts roots --- libdevcore/Common.h | 3 +- libdevcore/CommonData.cpp | 12 ++++++++ libdevcore/CommonData.h | 1 + libdevcrypto/TrieHash.cpp | 29 +++++++++++-------- libdevcrypto/TrieHash.h | 5 ++-- libethereum/State.cpp | 20 ++++--------- test/libdevcrypto/trie.cpp | 57 ++++++++++++++++++++++++-------------- 7 files changed, 76 insertions(+), 51 deletions(-) diff --git a/libdevcore/Common.h b/libdevcore/Common.h index 95817e41c..240746302 100644 --- a/libdevcore/Common.h +++ b/libdevcore/Common.h @@ -86,8 +86,9 @@ extern const u256 UndefinedU256; // Map types. using StringMap = std::map; +using BytesMap = std::map; using u256Map = std::map; -using HexMap = std::map; +using HexMap = std::map; // Hash types. using StringHashMap = std::unordered_map; diff --git a/libdevcore/CommonData.cpp b/libdevcore/CommonData.cpp index 6cad29952..ba2b412fd 100644 --- a/libdevcore/CommonData.cpp +++ b/libdevcore/CommonData.cpp @@ -115,6 +115,18 @@ bytes dev::fromHex(std::string const& _s, WhenError _throw) return ret; } +bytes dev::asNibbles(bytes const& _s) +{ + std::vector ret; + ret.reserve(_s.size() * 2); + for (auto i: _s) + { + ret.push_back(i / 16); + ret.push_back(i % 16); + } + return ret; +} + bytes dev::asNibbles(std::string const& _s) { std::vector ret; diff --git a/libdevcore/CommonData.h b/libdevcore/CommonData.h index 6c1f34667..29cc8e7c7 100644 --- a/libdevcore/CommonData.h +++ b/libdevcore/CommonData.h @@ -96,6 +96,7 @@ inline bytes asBytes(std::string const& _b) /// Converts a string into the big-endian base-16 stream of integers (NOT ASCII). /// @example asNibbles("A")[0] == 4 && asNibbles("A")[1] == 1 bytes asNibbles(std::string const& _s); +bytes asNibbles(bytes const& _s); // Big-endian to/from host endian conversion functions. diff --git a/libdevcrypto/TrieHash.cpp b/libdevcrypto/TrieHash.cpp index 0b02ce77f..4c4ca620d 100644 --- a/libdevcrypto/TrieHash.cpp +++ b/libdevcrypto/TrieHash.cpp @@ -158,43 +158,48 @@ void hash256aux(HexMap const& _s, HexMap::const_iterator _begin, HexMap::const_i } } -h256 hash256(StringMap const& _s) +bytes rlp256(BytesMap const& _s) { // build patricia tree. if (_s.empty()) - return sha3(rlp("")); + return rlp(""); HexMap hexMap; for (auto i = _s.rbegin(); i != _s.rend(); ++i) hexMap[asNibbles(i->first)] = i->second; RLPStream s; hash256rlp(hexMap, hexMap.cbegin(), hexMap.cend(), 0, s); - return sha3(s.out()); + return s.out(); } -bytes rlp256(StringMap const& _s) +h256 hash256(BytesMap const& _s) +{ + return sha3(rlp256(_s)); +} + +h256 hash256(StringMap const& _s) { // build patricia tree. if (_s.empty()) - return rlp(""); + return sha3(rlp("")); HexMap hexMap; for (auto i = _s.rbegin(); i != _s.rend(); ++i) - hexMap[asNibbles(i->first)] = i->second; + hexMap[asNibbles(i->first)] = bytes(i->second.begin(), i->second.end()); RLPStream s; - hash256aux(hexMap, hexMap.cbegin(), hexMap.cend(), 0, s); - return s.out(); + hash256rlp(hexMap, hexMap.cbegin(), hexMap.cend(), 0, s); + return sha3(s.out()); } -h256 hash256(u256Map const& _s) +bytes rlp256(StringMap const& _s) { // build patricia tree. if (_s.empty()) - return sha3(rlp("")); + return rlp(""); HexMap hexMap; for (auto i = _s.rbegin(); i != _s.rend(); ++i) - hexMap[asNibbles(toBigEndianString(i->first))] = asString(rlp(i->second)); + hexMap[asNibbles(i->first)] = bytes(i->second.begin(), i->second.end()); RLPStream s; hash256rlp(hexMap, hexMap.cbegin(), hexMap.cend(), 0, s); - return sha3(s.out()); + return s.out(); } /*h256 orderedTrieRoot(std::vector const& _data) diff --git a/libdevcrypto/TrieHash.h b/libdevcrypto/TrieHash.h index b0588fc38..46f9b1efb 100644 --- a/libdevcrypto/TrieHash.h +++ b/libdevcrypto/TrieHash.h @@ -27,9 +27,10 @@ namespace dev { +bytes rlp256(BytesMap const& _s); +h256 hash256(BytesMap const& _s); bytes rlp256(StringMap const& _s); h256 hash256(StringMap const& _s); -h256 hash256(u256Map const& _s); /*h256 orderedTrieRoot(std::vector const& _data); @@ -41,8 +42,6 @@ template inline h256 trieRootOver(unsigned _itemCount, T cons return hash256(m); }*/ -using bytesMap = std::unordered_map; - h256 orderedTrieRoot(std::vector const& _data); h256 orderedTrieRoot(std::vector const& _data); diff --git a/libethereum/State.cpp b/libethereum/State.cpp index b9e548ea3..d586941b4 100644 --- a/libethereum/State.cpp +++ b/libethereum/State.cpp @@ -840,16 +840,8 @@ void State::commitToMine(BlockChain const& _bc) } } - // TODO: move over to using TrieHash - - - MemoryDB tm; - GenericTrieDB transactionsTrie(&tm); - transactionsTrie.init(); - - MemoryDB rm; - GenericTrieDB receiptsTrie(&rm); - receiptsTrie.init(); + BytesMap transactionsMap; + BytesMap receiptsMap; RLPStream txs; txs.appendList(m_transactions.size()); @@ -861,11 +853,11 @@ void State::commitToMine(BlockChain const& _bc) RLPStream receiptrlp; m_receipts[i].streamRLP(receiptrlp); - receiptsTrie.insert(&k.out(), &receiptrlp.out()); + receiptsMap.insert(std::make_pair(k.out(), receiptrlp.out())); RLPStream txrlp; m_transactions[i].streamRLP(txrlp); - transactionsTrie.insert(&k.out(), &txrlp.out()); + transactionsMap.insert(std::make_pair(k.out(), txrlp.out())); txs.appendRaw(txrlp.out()); } @@ -874,8 +866,8 @@ void State::commitToMine(BlockChain const& _bc) RLPStream(unclesCount).appendRaw(unclesData.out(), unclesCount).swapOut(m_currentUncles); - m_currentBlock.transactionsRoot = transactionsTrie.root(); - m_currentBlock.receiptsRoot = receiptsTrie.root(); + m_currentBlock.transactionsRoot = hash256(transactionsMap); + m_currentBlock.receiptsRoot = hash256(receiptsMap); m_currentBlock.logBloom = logBloom(); m_currentBlock.sha3Uncles = sha3(m_currentUncles); diff --git a/test/libdevcrypto/trie.cpp b/test/libdevcrypto/trie.cpp index ecfc515aa..00bccd241 100644 --- a/test/libdevcrypto/trie.cpp +++ b/test/libdevcrypto/trie.cpp @@ -294,15 +294,27 @@ BOOST_AUTO_TEST_CASE(trie_tests_ordered) } } -inline h256 stringMapHash256(StringMap const& _s) +h256 stringMapHash256(StringMap const& _s) { - return hash256(_s); + //return hash256(_s); + BytesMap bytesMap; + for (auto const& _v: _s) + bytesMap.insert(std::make_pair(bytes(_v.first.begin(), _v.first.end()), bytes(_v.second.begin(), _v.second.end()))); + return hash256(bytesMap); +} + +bytes stringMapRlp256(StringMap const& _s) +{ + //return rlp256(_s); + BytesMap bytesMap; + for (auto const& _v: _s) + bytesMap.insert(std::make_pair(bytes(_v.first.begin(), _v.first.end()), bytes(_v.second.begin(), _v.second.end()))); + return rlp256(bytesMap); } BOOST_AUTO_TEST_CASE(moreTrieTests) { cnote << "Testing Trie more..."; -#if 0 // More tests... { MemoryDB m; @@ -311,7 +323,7 @@ BOOST_AUTO_TEST_CASE(moreTrieTests) cout << t; cout << m; cout << t.root() << endl; - cout << hash256(StringMap()) << endl; + cout << stringMapHash256(StringMap()) << endl; t.insert(string("tesz"), string("test")); cout << t; @@ -336,7 +348,7 @@ BOOST_AUTO_TEST_CASE(moreTrieTests) t.remove(string("test")); cout << m; cout << t.root() << endl; - cout << hash256(StringMap()) << endl; + cout << stringMapHash256(StringMap()) << endl; } { MemoryDB m; @@ -348,20 +360,23 @@ BOOST_AUTO_TEST_CASE(moreTrieTests) cout << m; cout << t.root() << endl; cout << stringMapHash256({{"b", "B"}, {"a", "A"}}) << endl; - cout << RLP(rlp256({{"b", "B"}, {"a", "A"}})) << endl; + bytes r(stringMapRlp256({{"b", "B"}, {"a", "A"}})); + cout << RLP(r) << endl; } { MemTrie t; t.insert("dog", "puppy"); cout << hex << t.hash256() << endl; - cout << RLP(t.rlp()) << endl; + bytes r(t.rlp()); + cout << RLP(r) << endl; } { MemTrie t; t.insert("bed", "d"); t.insert("be", "e"); cout << hex << t.hash256() << endl; - cout << RLP(t.rlp()) << endl; + bytes r(t.rlp()); + cout << RLP(r) << endl; } { cout << hex << stringMapHash256({{"dog", "puppy"}, {"doe", "reindeer"}}) << endl; @@ -369,10 +384,10 @@ BOOST_AUTO_TEST_CASE(moreTrieTests) t.insert("dog", "puppy"); t.insert("doe", "reindeer"); cout << hex << t.hash256() << endl; - cout << RLP(t.rlp()) << endl; + bytes r(t.rlp()); + cout << RLP(r) << endl; cout << toHex(t.rlp()) << endl; } -#endif { MemoryDB m; EnforceRefs r(m, true); @@ -387,16 +402,17 @@ BOOST_AUTO_TEST_CASE(moreTrieTests) t.insert(a, b); s[a] = b; - /*cout << endl << "-------------------------------" << endl; + cout << endl << "-------------------------------" << endl; cout << a << " -> " << b << endl; cout << d; cout << m; cout << d.root() << endl; - cout << hash256(s) << endl;*/ + cout << hash256(s) << endl; + cout << stringMapHash256(s) << endl; BOOST_REQUIRE(d.check(true)); - BOOST_REQUIRE_EQUAL(t.hash256(), hash256(s)); - BOOST_REQUIRE_EQUAL(d.root(), hash256(s)); + BOOST_REQUIRE_EQUAL(t.hash256(), stringMapHash256(s)); + BOOST_REQUIRE_EQUAL(d.root(), stringMapHash256(s)); for (auto const& i: s) { (void)i; @@ -421,8 +437,8 @@ BOOST_AUTO_TEST_CASE(moreTrieTests) BOOST_REQUIRE(d.check(true)); BOOST_REQUIRE(t.at(a).empty()); BOOST_REQUIRE(d.at(string(a)).empty()); - BOOST_REQUIRE_EQUAL(t.hash256(), hash256(s)); - BOOST_REQUIRE_EQUAL(d.root(), hash256(s)); + BOOST_REQUIRE_EQUAL(t.hash256(), stringMapHash256(s)); + BOOST_REQUIRE_EQUAL(d.root(), stringMapHash256(s)); for (auto const& i: s) { (void)i; @@ -493,7 +509,6 @@ BOOST_AUTO_TEST_CASE(trieLowerBound) BOOST_AUTO_TEST_CASE(trieStess) { cnote << "Stress-testing Trie..."; - if (0) { MemoryDB m; MemoryDB dm; @@ -512,8 +527,8 @@ BOOST_AUTO_TEST_CASE(trieStess) m[k] = v; t.insert(k, v); d.insert(k, v); - BOOST_REQUIRE_EQUAL(hash256(m), t.hash256()); - BOOST_REQUIRE_EQUAL(hash256(m), d.root()); + BOOST_REQUIRE_EQUAL(stringMapHash256(m), t.hash256()); + BOOST_REQUIRE_EQUAL(stringMapHash256(m), d.root()); BOOST_REQUIRE(d.check(true)); } while (!m.empty()) @@ -557,8 +572,8 @@ BOOST_AUTO_TEST_CASE(trieStess) cwarn << "Good?" << d2.root(); } BOOST_REQUIRE(d.check(true)); - BOOST_REQUIRE_EQUAL(hash256(m), t.hash256()); - BOOST_REQUIRE_EQUAL(hash256(m), d.root()); + BOOST_REQUIRE_EQUAL(stringMapHash256(m), t.hash256()); + BOOST_REQUIRE_EQUAL(stringMapHash256(m), d.root()); } } } From 4e358001acd9f478d2e69b16794cec4832afb1fa Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 19 May 2015 10:39:33 +0200 Subject: [PATCH 2/2] orderedTrieRoot testing --- libdevcore/CommonData.cpp | 14 +--------- libdevcore/CommonData.h | 3 +-- libdevcrypto/TrieHash.cpp | 55 ++++++-------------------------------- libdevcrypto/TrieHash.h | 10 +++---- libethcore/BlockInfo.cpp | 12 +-------- test/libdevcrypto/trie.cpp | 3 --- 6 files changed, 15 insertions(+), 82 deletions(-) diff --git a/libdevcore/CommonData.cpp b/libdevcore/CommonData.cpp index ba2b412fd..f8d8c172f 100644 --- a/libdevcore/CommonData.cpp +++ b/libdevcore/CommonData.cpp @@ -115,19 +115,7 @@ bytes dev::fromHex(std::string const& _s, WhenError _throw) return ret; } -bytes dev::asNibbles(bytes const& _s) -{ - std::vector ret; - ret.reserve(_s.size() * 2); - for (auto i: _s) - { - ret.push_back(i / 16); - ret.push_back(i % 16); - } - return ret; -} - -bytes dev::asNibbles(std::string const& _s) +bytes dev::asNibbles(bytesConstRef const& _s) { std::vector ret; ret.reserve(_s.size() * 2); diff --git a/libdevcore/CommonData.h b/libdevcore/CommonData.h index 29cc8e7c7..702f1f808 100644 --- a/libdevcore/CommonData.h +++ b/libdevcore/CommonData.h @@ -95,8 +95,7 @@ inline bytes asBytes(std::string const& _b) /// Converts a string into the big-endian base-16 stream of integers (NOT ASCII). /// @example asNibbles("A")[0] == 4 && asNibbles("A")[1] == 1 -bytes asNibbles(std::string const& _s); -bytes asNibbles(bytes const& _s); +bytes asNibbles(bytesConstRef const& _s); // Big-endian to/from host endian conversion functions. diff --git a/libdevcrypto/TrieHash.cpp b/libdevcrypto/TrieHash.cpp index 4c4ca620d..0f2d68b91 100644 --- a/libdevcrypto/TrieHash.cpp +++ b/libdevcrypto/TrieHash.cpp @@ -165,7 +165,7 @@ bytes rlp256(BytesMap const& _s) return rlp(""); HexMap hexMap; for (auto i = _s.rbegin(); i != _s.rend(); ++i) - hexMap[asNibbles(i->first)] = i->second; + hexMap[asNibbles(bytesConstRef(&i->first))] = i->second; RLPStream s; hash256rlp(hexMap, hexMap.cbegin(), hexMap.cend(), 0, s); return s.out(); @@ -176,61 +176,22 @@ h256 hash256(BytesMap const& _s) return sha3(rlp256(_s)); } -h256 hash256(StringMap const& _s) -{ - // build patricia tree. - if (_s.empty()) - return sha3(rlp("")); - HexMap hexMap; - for (auto i = _s.rbegin(); i != _s.rend(); ++i) - hexMap[asNibbles(i->first)] = bytes(i->second.begin(), i->second.end()); - RLPStream s; - hash256rlp(hexMap, hexMap.cbegin(), hexMap.cend(), 0, s); - return sha3(s.out()); -} - -bytes rlp256(StringMap const& _s) -{ - // build patricia tree. - if (_s.empty()) - return rlp(""); - HexMap hexMap; - for (auto i = _s.rbegin(); i != _s.rend(); ++i) - hexMap[asNibbles(i->first)] = bytes(i->second.begin(), i->second.end()); - RLPStream s; - hash256rlp(hexMap, hexMap.cbegin(), hexMap.cend(), 0, s); - return s.out(); -} - -/*h256 orderedTrieRoot(std::vector const& _data) +h256 orderedTrieRoot(std::vector const& _data) { - StringMap m; + BytesMap m; unsigned j = 0; for (auto i: _data) - m[asString(rlp(j++))] = asString(i); + m[rlp(j++)] = i; return hash256(m); -}*/ - -h256 orderedTrieRoot(std::vector const& _data) -{ - MemoryDB db; - GenericTrieDB t(&db); - t.init(); - unsigned j = 0; - for (auto i: _data) - t.insert(rlp(j++), i.toBytes()); - return t.root(); } -h256 orderedTrieRoot(std::vector const& _data) +h256 orderedTrieRoot(std::vector const& _data) { - MemoryDB db; - GenericTrieDB t(&db); - t.init(); + BytesMap m; unsigned j = 0; for (auto i: _data) - t.insert(rlp(j++), i); - return t.root(); + m[rlp(j++)] = i.toBytes(); + return hash256(m); } } diff --git a/libdevcrypto/TrieHash.h b/libdevcrypto/TrieHash.h index 46f9b1efb..9649ef0c7 100644 --- a/libdevcrypto/TrieHash.h +++ b/libdevcrypto/TrieHash.h @@ -29,18 +29,16 @@ namespace dev bytes rlp256(BytesMap const& _s); h256 hash256(BytesMap const& _s); -bytes rlp256(StringMap const& _s); -h256 hash256(StringMap const& _s); -/*h256 orderedTrieRoot(std::vector const& _data); +h256 orderedTrieRoot(std::vector const& _data); template inline h256 trieRootOver(unsigned _itemCount, T const& _getKey, U const& _getValue) { - StringMap m; + BytesMap m; for (unsigned i = 0; i < _itemCount; ++i) - m[asString(_getKey(i))] = asString(_getValue(i)); + m[_getKey(i)] = _getValue(i); return hash256(m); -}*/ +} h256 orderedTrieRoot(std::vector const& _data); h256 orderedTrieRoot(std::vector const& _data); diff --git a/libethcore/BlockInfo.cpp b/libethcore/BlockInfo.cpp index 00a5108c4..0c324d70f 100644 --- a/libethcore/BlockInfo.cpp +++ b/libethcore/BlockInfo.cpp @@ -191,22 +191,12 @@ void BlockInfo::populate(bytesConstRef _block, Strictness _s, h256 const& _h) struct BlockInfoDiagnosticsChannel: public LogChannel { static const char* name() { return EthBlue "▧" EthWhite " ◌"; } static const int verbosity = 9; }; -template h256 trieRootOver(unsigned _itemCount, T const& _getKey, U const& _getValue) -{ - MemoryDB db; - GenericTrieDB t(&db); - t.init(); - for (unsigned i = 0; i < _itemCount; ++i) - t.insert(_getKey(i), _getValue(i)); - return t.root(); -} - void BlockInfo::verifyInternals(bytesConstRef _block) const { RLP root(_block); auto txList = root[1]; - auto expectedRoot = trieRootOver(txList.itemCount(), [&](unsigned i){ return rlp(i); }, [&](unsigned i){ return txList[i].data(); }); + auto expectedRoot = trieRootOver(txList.itemCount(), [&](unsigned i){ return rlp(i); }, [&](unsigned i){ return txList[i].data().toBytes(); }); clog(BlockInfoDiagnosticsChannel) << "Expected trie root:" << toString(expectedRoot); if (transactionsRoot != expectedRoot) diff --git a/test/libdevcrypto/trie.cpp b/test/libdevcrypto/trie.cpp index 00bccd241..b32fd70e1 100644 --- a/test/libdevcrypto/trie.cpp +++ b/test/libdevcrypto/trie.cpp @@ -296,7 +296,6 @@ BOOST_AUTO_TEST_CASE(trie_tests_ordered) h256 stringMapHash256(StringMap const& _s) { - //return hash256(_s); BytesMap bytesMap; for (auto const& _v: _s) bytesMap.insert(std::make_pair(bytes(_v.first.begin(), _v.first.end()), bytes(_v.second.begin(), _v.second.end()))); @@ -305,7 +304,6 @@ h256 stringMapHash256(StringMap const& _s) bytes stringMapRlp256(StringMap const& _s) { - //return rlp256(_s); BytesMap bytesMap; for (auto const& _v: _s) bytesMap.insert(std::make_pair(bytes(_v.first.begin(), _v.first.end()), bytes(_v.second.begin(), _v.second.end()))); @@ -407,7 +405,6 @@ BOOST_AUTO_TEST_CASE(moreTrieTests) cout << d; cout << m; cout << d.root() << endl; - cout << hash256(s) << endl; cout << stringMapHash256(s) << endl; BOOST_REQUIRE(d.check(true));