From c1e71dae80c75f10cfcc472c68200c546c938536 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 11 Mar 2015 16:31:09 +0000 Subject: [PATCH] Remove skip & max, integrate new heirarchical bloom stuff with everything else. --- alethzero/MainWin.cpp | 12 ++-- libdevcore/FixedHash.h | 6 +- libethereum/BlockChain.cpp | 40 +++++++++++-- libethereum/BlockDetails.h | 2 +- libethereum/Client.cpp | 69 ++++++++--------------- libethereum/LogFilter.cpp | 14 ++++- libethereum/LogFilter.h | 10 +--- libweb3jsonrpc/WebThreeStubServerBase.cpp | 4 -- mix/MixClient.cpp | 12 +--- 9 files changed, 89 insertions(+), 80 deletions(-) diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index ab8e07e7c..165c10f90 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -1012,15 +1012,17 @@ void Main::refreshBlockChain() h256 h(f.toStdString()); if (bc.isKnown(h)) blocks.insert(h); + for (auto const& b: bc.withBlockBloom(LogBloom().shiftBloom<3, 32>(sha3(h)), 0, -1)) + blocks.insert(bc.numberHash(b)); } else if (f.toLongLong() <= bc.number()) blocks.insert(bc.numberHash(u256(f.toLongLong()))); - /*else if (f.size() == 40) + else if (f.size() == 40) { - Address h(f[0]); - if (bc.(h)) - blocks.insert(h); - }*/ + Address h(f.toStdString()); + for (auto const& b: bc.withBlockBloom(LogBloom().shiftBloom<3, 32>(sha3(h)), 0, -1)) + blocks.insert(bc.numberHash(b)); + } QByteArray oldSelected = ui->blocks->count() ? ui->blocks->currentItem()->data(Qt::UserRole).toByteArray() : QByteArray(); ui->blocks->clear(); diff --git a/libdevcore/FixedHash.h b/libdevcore/FixedHash.h index a52e1cb2e..bbc40e66c 100644 --- a/libdevcore/FixedHash.h +++ b/libdevcore/FixedHash.h @@ -156,15 +156,15 @@ public: template inline FixedHash& shiftBloom(FixedHash const& _h) { - return (*this |= _h.template nbloom()); + return (*this |= _h.template bloom()); } template inline bool containsBloom(FixedHash const& _h) { - return contains(_h.template nbloom()); + return contains(_h.template bloom()); } - template inline FixedHash nbloom() const + template inline FixedHash bloom() const { static const unsigned c_bloomBits = M * 8; unsigned mask = c_bloomBits - 1; diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 54f919647..95bfcb015 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -610,15 +610,26 @@ static inline unsigned ceilDiv(unsigned n, unsigned d) { return n / (n + d - 1); static inline unsigned floorDivPow(unsigned n, unsigned a, unsigned b) { return n / upow(a, b); } static inline unsigned ceilDivPow(unsigned n, unsigned a, unsigned b) { return ceilDiv(n, upow(a, b)); } +// Level 1 +// [xxx. ] + +// Level 0 +// [.x............F.] +// [........x.......] +// [T.............x.] +// [............ ] + +// F = 14. T = 32 + vector BlockChain::withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest) const { vector ret; // start from the top-level - unsigned u = upow(c_bloomIndexSize, c_bloomIndexLevels - 1); + unsigned u = upow(c_bloomIndexSize, c_bloomIndexLevels); // run through each of the top-level blockbloom blocks - for (unsigned index = _earliest / u; index < ceilDiv(_latest, u); ++index) + for (unsigned index = _earliest / u; index <= ceilDiv(_latest, u); ++index) // 0 ret += withBlockBloom(_b, _earliest, _latest, c_bloomIndexLevels - 1, index); return ret; @@ -626,12 +637,33 @@ vector BlockChain::withBlockBloom(LogBloom const& _b, unsigned _earlie vector BlockChain::withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest, unsigned _level, unsigned _index) const { + // 14, 32, 1, 0 + // 14, 32, 0, 0 + // 14, 32, 0, 1 + // 14, 32, 0, 2 + vector ret; -// unsigned u = upow(c_bloomIndexSize, _level); + unsigned uCourse = upow(c_bloomIndexSize, _level + 1); + // 256 + // 16 + unsigned uFine = upow(c_bloomIndexSize, _level); + // 16 + // 1 + + unsigned obegin = _index == _earliest / uCourse ? _earliest / uFine % c_bloomIndexSize : 0; + // 0 + // 14 + // 0 + // 0 + unsigned oend = _index == _latest / uCourse ? (_latest / uFine) % c_bloomIndexSize + 1 : c_bloomIndexSize; + // 3 + // 16 + // 16 + // 1 BlocksBlooms bb = blocksBlooms(_level, _index); - for (unsigned o = 0; o < 256; ++o) // TODO: crop. + for (unsigned o = obegin; o < oend; ++o) if (bb.blooms[o].contains(_b)) { // This level has something like what we want. diff --git a/libethereum/BlockDetails.h b/libethereum/BlockDetails.h index f3bb73059..572ed1888 100644 --- a/libethereum/BlockDetails.h +++ b/libethereum/BlockDetails.h @@ -39,7 +39,7 @@ namespace eth // TODO: OPTIMISE: constructors take bytes, RLP used only in necessary classes. static const unsigned c_bloomIndexSize = 16; -static const unsigned c_bloomIndexLevels = 6; +static const unsigned c_bloomIndexLevels = 2; struct BlockDetails { diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index 9099986c0..165ac20ca 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -833,8 +833,6 @@ LocalisedLogEntries Client::logs(LogFilter const& _f) const LocalisedLogEntries ret; unsigned begin = min(m_bc.number() + 1, (unsigned)_f.latest()); unsigned end = min(m_bc.number(), min(begin, (unsigned)_f.earliest())); - unsigned m = _f.max(); - unsigned s = _f.skip(); // Handle pending transactions differently as they're not on the block chain. if (begin > m_bc.number()) @@ -847,71 +845,52 @@ LocalisedLogEntries Client::logs(LogFilter const& _f) const auto sha3 = m_postMine.pending()[i].sha3(); LogEntries le = _f.matches(tr); if (le.size()) - { - for (unsigned j = 0; j < le.size() && ret.size() != m; ++j) - if (s) - s--; - else - ret.insert(ret.begin(), LocalisedLogEntry(le[j], begin, sha3)); - } + for (unsigned j = 0; j < le.size(); ++j) + ret.insert(ret.begin(), LocalisedLogEntry(le[j], begin, sha3)); } begin = m_bc.number(); } + set matchingBlocks; + for (auto const& i: _f.bloomPossibilities()) + for (auto u: m_bc.withBlockBloom(i, end, begin)) + matchingBlocks.insert(u); + #if ETH_DEBUG - // fill these params - unsigned skipped = 0; unsigned falsePos = 0; #endif - auto h = m_bc.numberHash(begin); - unsigned n = begin; - for (; ret.size() != m && n != end; n--, h = m_bc.details(h).parent) + for (auto n: matchingBlocks) { #if ETH_DEBUG int total = 0; #endif - // check block bloom - auto blockBloom = m_bc.blockBloom(n); - if (_f.matches(blockBloom)) + auto h = m_bc.numberHash(n); + auto receipts = m_bc.receipts(h).receipts; + for (size_t i = 0; i < receipts.size(); i++) { - auto receipts = m_bc.receipts(h).receipts; - for (size_t i = 0; i < receipts.size(); i++) + TransactionReceipt receipt = receipts[i]; + if (_f.matches(receipt.bloom())) { - TransactionReceipt receipt = receipts[i]; - if (_f.matches(receipt.bloom())) + auto info = m_bc.info(h); + auto h = transaction(info.hash, i).sha3(); + LogEntries le = _f.matches(receipt); + if (le.size()) { - auto info = m_bc.info(h); - auto h = transaction(info.hash, i).sha3(); - LogEntries le = _f.matches(receipt); - if (le.size()) - { #if ETH_DEBUG - total += le.size(); + total += le.size(); #endif - for (unsigned j = 0; j < le.size() && ret.size() != m; ++j) - { - if (s) - s--; - else - ret.insert(ret.begin(), LocalisedLogEntry(le[j], n, h)); - } - } + for (unsigned j = 0; j < le.size(); ++j) + ret.insert(ret.begin(), LocalisedLogEntry(le[j], n, h)); } -#if ETH_DEBUG - if (!total) - falsePos++; -#endif } #if ETH_DEBUG - } - else - skipped++; + if (!total) + falsePos++; #endif - if (n == end) - break; + } } #if ETH_DEBUG - cdebug << (begin - n) << "searched; " << skipped << "skipped; " << falsePos << "false +ves"; + cdebug << matchingBlocks.size() << "searched from" << (end - begin) << "skipped; " << falsePos << "false +ves"; #endif return ret; } diff --git a/libethereum/LogFilter.cpp b/libethereum/LogFilter.cpp index 1784094b0..d1b3cf437 100644 --- a/libethereum/LogFilter.cpp +++ b/libethereum/LogFilter.cpp @@ -30,13 +30,13 @@ using namespace dev::eth; std::ostream& dev::eth::operator<<(std::ostream& _out, LogFilter const& _s) { // TODO - _out << "(@" << _s.m_addresses << "#" << _s.m_topics << ">" << _s.m_earliest << "-" << _s.m_latest << "< +" << _s.m_skip << "^" << _s.m_max << ")"; + _out << "(@" << _s.m_addresses << "#" << _s.m_topics << ">" << _s.m_earliest << "-" << _s.m_latest << "< )"; return _out; } void LogFilter::streamRLP(RLPStream& _s) const { - _s.appendList(6) << m_addresses << m_topics << m_earliest << m_latest << m_max << m_skip; + _s.appendList(4) << m_addresses << m_topics << m_earliest << m_latest; } h256 LogFilter::sha3() const @@ -73,6 +73,16 @@ bool LogFilter::matches(State const& _s, unsigned _i) const return matches(_s.receipt(_i)).size() > 0; } +vector LogFilter::bloomPossibilities() const +{ + // return combination of each of the addresses/topics + vector ret; + // TODO proper combinatorics. + for (auto i: m_addresses) + ret.push_back(LogBloom().shiftBloom<3, 32>(dev::sha3(i))); + return ret; +} + LogEntries LogFilter::matches(TransactionReceipt const& _m) const { LogEntries ret; diff --git a/libethereum/LogFilter.h b/libethereum/LogFilter.h index 7b8922a03..a28bfe806 100644 --- a/libethereum/LogFilter.h +++ b/libethereum/LogFilter.h @@ -45,23 +45,21 @@ class State; class LogFilter { public: - LogFilter(int _earliest = 0, int _latest = -1, unsigned _max = 10, unsigned _skip = 0): m_earliest(_earliest), m_latest(_latest), m_max(_max), m_skip(_skip) {} + LogFilter(int _earliest = 0, int _latest = -1): m_earliest(_earliest), m_latest(_latest) {} void streamRLP(RLPStream& _s) const; h256 sha3() const; int earliest() const { return m_earliest; } int latest() const { return m_latest; } - unsigned max() const { return m_max; } - unsigned skip() const { return m_skip; } + + std::vector bloomPossibilities() const; bool matches(LogBloom _bloom) const; bool matches(State const& _s, unsigned _i) const; LogEntries matches(TransactionReceipt const& _r) const; LogFilter address(Address _a) { m_addresses.insert(_a); return *this; } LogFilter topic(unsigned _index, h256 const& _t) { if (_index < 4) m_topics[_index].insert(_t); return *this; } - LogFilter withMax(unsigned _m) { m_max = _m; return *this; } - LogFilter withSkip(unsigned _m) { m_skip = _m; return *this; } LogFilter withEarliest(int _e) { m_earliest = _e; return *this; } LogFilter withLatest(int _e) { m_latest = _e; return *this; } @@ -72,8 +70,6 @@ private: std::array m_topics; int m_earliest = 0; int m_latest = -1; - unsigned m_max = 10; - unsigned m_skip = 0; }; } diff --git a/libweb3jsonrpc/WebThreeStubServerBase.cpp b/libweb3jsonrpc/WebThreeStubServerBase.cpp index 70c8ca9de..09c34feb0 100644 --- a/libweb3jsonrpc/WebThreeStubServerBase.cpp +++ b/libweb3jsonrpc/WebThreeStubServerBase.cpp @@ -128,10 +128,6 @@ static dev::eth::LogFilter toLogFilter(Json::Value const& _json) // commented to filter.withEarliest(_json["earliest"].asInt()); if (_json["latest"].isInt()) filter.withLatest(_json["lastest"].asInt()); - if (_json["max"].isInt()) - filter.withMax(_json["max"].asInt()); - if (_json["skip"].isInt()) - filter.withSkip(_json["skip"].asInt()); if (!_json["address"].empty()) { if (_json["address"].isArray()) diff --git a/mix/MixClient.cpp b/mix/MixClient.cpp index 061057bb4..a06bbb02d 100644 --- a/mix/MixClient.cpp +++ b/mix/MixClient.cpp @@ -331,7 +331,6 @@ eth::LocalisedLogEntries MixClient::logs(eth::LogFilter const& _f) const unsigned lastBlock = bc().number(); unsigned block = std::min(lastBlock, (unsigned)_f.latest()); unsigned end = std::min(lastBlock, std::min(block, (unsigned)_f.earliest())); - unsigned skip = _f.skip(); // Pending transactions if (block > bc().number()) { @@ -341,9 +340,8 @@ eth::LocalisedLogEntries MixClient::logs(eth::LogFilter const& _f) const // Might have a transaction that contains a matching log. TransactionReceipt const& tr = m_state.receipt(i); LogEntries logEntries = _f.matches(tr); - for (unsigned entry = 0; entry < logEntries.size() && ret.size() != _f.max(); ++entry) + for (unsigned entry = 0; entry < logEntries.size(); ++entry) ret.insert(ret.begin(), LocalisedLogEntry(logEntries[entry], block)); - skip -= std::min(skip, static_cast(logEntries.size())); } block = bc().number(); } @@ -355,12 +353,8 @@ eth::LocalisedLogEntries MixClient::logs(eth::LogFilter const& _f) const if (_f.matches(bc().info(h).logBloom)) for (TransactionReceipt receipt: bc().receipts(h).receipts) if (_f.matches(receipt.bloom())) - { - LogEntries logEntries = _f.matches(receipt); - for (unsigned entry = skip; entry < logEntries.size() && ret.size() != _f.max(); ++entry) - ret.insert(ret.begin(), LocalisedLogEntry(logEntries[entry], block)); - skip -= std::min(skip, static_cast(logEntries.size())); - } + for (auto const& e: _f.matches(receipt)) + ret.insert(ret.begin(), LocalisedLogEntry(e, block)); h = bc().details(h).parent; } return ret;