diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index ab8e07e7c..165c10f90 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -1012,15 +1012,17 @@ void Main::refreshBlockChain() h256 h(f.toStdString()); if (bc.isKnown(h)) blocks.insert(h); + for (auto const& b: bc.withBlockBloom(LogBloom().shiftBloom<3, 32>(sha3(h)), 0, -1)) + blocks.insert(bc.numberHash(b)); } else if (f.toLongLong() <= bc.number()) blocks.insert(bc.numberHash(u256(f.toLongLong()))); - /*else if (f.size() == 40) + else if (f.size() == 40) { - Address h(f[0]); - if (bc.(h)) - blocks.insert(h); - }*/ + Address h(f.toStdString()); + for (auto const& b: bc.withBlockBloom(LogBloom().shiftBloom<3, 32>(sha3(h)), 0, -1)) + blocks.insert(bc.numberHash(b)); + } QByteArray oldSelected = ui->blocks->count() ? ui->blocks->currentItem()->data(Qt::UserRole).toByteArray() : QByteArray(); ui->blocks->clear(); diff --git a/libdevcore/Common.cpp b/libdevcore/Common.cpp index 431a95580..07e92f542 100644 --- a/libdevcore/Common.cpp +++ b/libdevcore/Common.cpp @@ -27,7 +27,7 @@ using namespace dev; namespace dev { -char const* Version = "0.8.2"; +char const* Version = "0.9.0"; } diff --git a/libdevcore/FixedHash.h b/libdevcore/FixedHash.h index 49c6ed2bf..bbc40e66c 100644 --- a/libdevcore/FixedHash.h +++ b/libdevcore/FixedHash.h @@ -154,25 +154,17 @@ public: } }; - inline FixedHash<32> bloom() const - { - FixedHash<32> ret; - for (auto i: m_data) - ret[i / 8] |= 1 << (i % 8); - return ret; - } - template inline FixedHash& shiftBloom(FixedHash const& _h) { - return (*this |= _h.template nbloom()); + return (*this |= _h.template bloom()); } template inline bool containsBloom(FixedHash const& _h) { - return contains(_h.template nbloom()); + return contains(_h.template bloom()); } - template inline FixedHash nbloom() const + template inline FixedHash bloom() const { static const unsigned c_bloomBits = M * 8; unsigned mask = c_bloomBits - 1; diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index dd0bfad67..95bfcb015 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -324,6 +324,18 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db) WriteGuard l(x_blockHashes); m_blockHashes[h256(bi.number)].value = newHash; } + { + WriteGuard l(x_blocksBlooms); + LogBloom blockBloom = bi.logBloom; + blockBloom.shiftBloom<3, 32>(sha3(bi.coinbaseAddress.ref())); + unsigned index = (unsigned)bi.number; + for (unsigned level = 0; level < c_bloomIndexLevels; level++, index /= c_bloomIndexSize) + { + unsigned i = index / c_bloomIndexSize % c_bloomIndexSize; + unsigned o = index % c_bloomIndexSize; + m_blocksBlooms[chunkId(level, i)].blooms[o] |= blockBloom; + } + } // Collate transaction hashes and remember who they were. h256s tas; { @@ -346,14 +358,23 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db) m_receipts[newHash] = br; } - m_blocksDB->Put(m_writeOptions, toSlice(newHash), (ldb::Slice)ref(_block)); - m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[newHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(bi.parentHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(h256(bi.number), ExtraBlockHash), (ldb::Slice)dev::ref(m_blockHashes[h256(bi.number)].rlp())); - for (auto const& h: tas) - m_extrasDB->Put(m_writeOptions, toSlice(h, ExtraTransactionAddress), (ldb::Slice)dev::ref(m_transactionAddresses[h].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraLogBlooms), (ldb::Slice)dev::ref(m_logBlooms[newHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraReceipts), (ldb::Slice)dev::ref(m_receipts[newHash].rlp())); + { + ReadGuard l1(x_blocksBlooms); + ReadGuard l2(x_details); + ReadGuard l3(x_blockHashes); + ReadGuard l4(x_receipts); + ReadGuard l5(x_logBlooms); + ReadGuard l6(x_transactionAddresses); + m_blocksDB->Put(m_writeOptions, toSlice(newHash), (ldb::Slice)ref(_block)); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[newHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(bi.parentHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(h256(bi.number), ExtraBlockHash), (ldb::Slice)dev::ref(m_blockHashes[h256(bi.number)].rlp())); + for (auto const& h: tas) + m_extrasDB->Put(m_writeOptions, toSlice(h, ExtraTransactionAddress), (ldb::Slice)dev::ref(m_transactionAddresses[h].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraLogBlooms), (ldb::Slice)dev::ref(m_logBlooms[newHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraReceipts), (ldb::Slice)dev::ref(m_receipts[newHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraBlocksBlooms), (ldb::Slice)dev::ref(m_blocksBlooms[newHash].rlp())); + } #if ETH_PARANOIA checkConsistency(); @@ -475,29 +496,30 @@ template static unsigned getHashSize(map const& _map) void BlockChain::updateStats() const { { - ReadGuard l1(x_blocks); + ReadGuard l(x_blocks); m_lastStats.memBlocks = 0; for (auto const& i: m_blocks) m_lastStats.memBlocks += i.second.size() + 64; } { - ReadGuard l2(x_details); + ReadGuard l(x_details); m_lastStats.memDetails = getHashSize(m_details); } { - ReadGuard l5(x_logBlooms); - m_lastStats.memLogBlooms = getHashSize(m_logBlooms); + ReadGuard l1(x_logBlooms); + ReadGuard l2(x_blocksBlooms); + m_lastStats.memLogBlooms = getHashSize(m_logBlooms) + getHashSize(m_blocksBlooms); } { - ReadGuard l4(x_receipts); + ReadGuard l(x_receipts); m_lastStats.memReceipts = getHashSize(m_receipts); } { - ReadGuard l3(x_blockHashes); + ReadGuard l(x_blockHashes); m_lastStats.memBlockHashes = getHashSize(m_blockHashes); } { - ReadGuard l6(x_transactionAddresses); + ReadGuard l(x_transactionAddresses); m_lastStats.memTransactionAddresses = getHashSize(m_transactionAddresses); } } @@ -520,6 +542,7 @@ void BlockChain::garbageCollect(bool _force) WriteGuard l4(x_receipts); WriteGuard l5(x_logBlooms); WriteGuard l6(x_transactionAddresses); + WriteGuard l7(x_blocksBlooms); for (CacheID const& id: m_cacheUsage.back()) { m_inUse.erase(id); @@ -544,6 +567,9 @@ void BlockChain::garbageCollect(bool _force) case ExtraTransactionAddress: m_transactionAddresses.erase(id.first); break; + case ExtraBlocksBlooms: + m_blocksBlooms.erase(id.first); + break; } } m_cacheUsage.pop_back(); @@ -579,6 +605,76 @@ void BlockChain::checkConsistency() delete it; } +static inline unsigned upow(unsigned a, unsigned b) { while (b-- > 0) a *= a; return a; } +static inline unsigned ceilDiv(unsigned n, unsigned d) { return n / (n + d - 1); } +static inline unsigned floorDivPow(unsigned n, unsigned a, unsigned b) { return n / upow(a, b); } +static inline unsigned ceilDivPow(unsigned n, unsigned a, unsigned b) { return ceilDiv(n, upow(a, b)); } + +// Level 1 +// [xxx. ] + +// Level 0 +// [.x............F.] +// [........x.......] +// [T.............x.] +// [............ ] + +// F = 14. T = 32 + +vector BlockChain::withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest) const +{ + vector ret; + + // start from the top-level + unsigned u = upow(c_bloomIndexSize, c_bloomIndexLevels); + + // run through each of the top-level blockbloom blocks + for (unsigned index = _earliest / u; index <= ceilDiv(_latest, u); ++index) // 0 + ret += withBlockBloom(_b, _earliest, _latest, c_bloomIndexLevels - 1, index); + + return ret; +} + +vector BlockChain::withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest, unsigned _level, unsigned _index) const +{ + // 14, 32, 1, 0 + // 14, 32, 0, 0 + // 14, 32, 0, 1 + // 14, 32, 0, 2 + + vector ret; + + unsigned uCourse = upow(c_bloomIndexSize, _level + 1); + // 256 + // 16 + unsigned uFine = upow(c_bloomIndexSize, _level); + // 16 + // 1 + + unsigned obegin = _index == _earliest / uCourse ? _earliest / uFine % c_bloomIndexSize : 0; + // 0 + // 14 + // 0 + // 0 + unsigned oend = _index == _latest / uCourse ? (_latest / uFine) % c_bloomIndexSize + 1 : c_bloomIndexSize; + // 3 + // 16 + // 16 + // 1 + + BlocksBlooms bb = blocksBlooms(_level, _index); + for (unsigned o = obegin; o < oend; ++o) + if (bb.blooms[o].contains(_b)) + { + // This level has something like what we want. + if (_level > 0) + ret += withBlockBloom(_b, _earliest, _latest, _level - 1, o + _index * c_bloomIndexSize); + else + ret.push_back(o + _index * c_bloomIndexSize); + } + return ret; +} + h256Set BlockChain::allUnclesFrom(h256 _parent) const { // Get all uncles cited given a parent (i.e. featured as uncles/main in parent, parent + 1, ... parent + 5). diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h index 0c1a81066..a6760a3f6 100644 --- a/libethereum/BlockChain.h +++ b/libethereum/BlockChain.h @@ -71,7 +71,8 @@ enum { ExtraBlockHash, ExtraTransactionAddress, ExtraLogBlooms, - ExtraReceipts + ExtraReceipts, + ExtraBlocksBlooms }; /** @@ -132,6 +133,26 @@ public: /// Get a list of transaction hashes for a given block. Thread-safe. h256 numberHash(u256 _index) const { if (!_index) return genesisHash(); return queryExtras(h256(_index), m_blockHashes, x_blockHashes, NullBlockHash).value; } + /** Get the block blooms for a number of blocks. Thread-safe. + * @returns the object pertaining to the blocks: + * level 0: + * 0x, 0x + 1, .. (1x - 1) + * 1x, 1x + 1, .. (2x - 1) + * ... + * (255x .. (256x - 1)) + * level 1: + * 0x .. (1x - 1), 1x .. (2x - 1), ..., (255x .. (256x - 1)) + * 256x .. (257x - 1), 257x .. (258x - 1), ..., (511x .. (512x - 1)) + * ... + * level n, index i, offset o: + * i * (x ^ n) + o * x ^ (n - 1) + */ + BlocksBlooms blocksBlooms(unsigned _level, unsigned _index) const { return blocksBlooms(chunkId(_level, _index)); } + BlocksBlooms blocksBlooms(h256 const& _chunkId) const { return queryExtras(_chunkId, m_blocksBlooms, x_blocksBlooms, NullBlocksBlooms); } + LogBloom blockBloom(unsigned _number) const { return blocksBlooms(chunkId(0, _number / c_bloomIndexSize)).blooms[_number % c_bloomIndexSize]; } + std::vector withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest) const; + std::vector withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest, unsigned _topLevel, unsigned _index) const; + /// Get a transaction from its hash. Thread-safe. bytes transaction(h256 _transactionHash) const { TransactionAddress ta = queryExtras(_transactionHash, m_transactionAddresses, x_transactionAddresses, NullTransactionAddress); if (!ta) return bytes(); return transaction(ta.blockHash, ta.index); } @@ -188,6 +209,8 @@ public: void garbageCollect(bool _force = false); private: + static h256 chunkId(unsigned _level, unsigned _index) { return h256(_index * 0xff + _level); } + void open(std::string _path, bool _killExisting = false); void close(); @@ -230,6 +253,8 @@ private: mutable TransactionAddressHash m_transactionAddresses; mutable SharedMutex x_blockHashes; mutable BlockHashHash m_blockHashes; + mutable SharedMutex x_blocksBlooms; + mutable BlocksBloomsHash m_blocksBlooms; using CacheID = std::pair; mutable Mutex x_cacheUsage; diff --git a/libethereum/BlockDetails.h b/libethereum/BlockDetails.h index ed478568d..572ed1888 100644 --- a/libethereum/BlockDetails.h +++ b/libethereum/BlockDetails.h @@ -36,6 +36,11 @@ namespace dev namespace eth { +// TODO: OPTIMISE: constructors take bytes, RLP used only in necessary classes. + +static const unsigned c_bloomIndexSize = 16; +static const unsigned c_bloomIndexLevels = 2; + struct BlockDetails { BlockDetails(): number(0), totalDifficulty(0) {} @@ -64,6 +69,16 @@ struct BlockLogBlooms mutable unsigned size; }; +struct BlocksBlooms +{ + BlocksBlooms() {} + BlocksBlooms(RLP const& _r) { blooms = _r.toArray(); size = _r.data().size(); } + bytes rlp() const { RLPStream s; s << blooms; size = s.out().size(); return s.out(); } + + std::array blooms; + mutable unsigned size; +}; + struct BlockReceipts { BlockReceipts() {} @@ -103,12 +118,14 @@ using BlockLogBloomsHash = std::map; using BlockReceiptsHash = std::map; using TransactionAddressHash = std::map; using BlockHashHash = std::map; +using BlocksBloomsHash = std::map; static const BlockDetails NullBlockDetails; static const BlockLogBlooms NullBlockLogBlooms; static const BlockReceipts NullBlockReceipts; static const TransactionAddress NullTransactionAddress; static const BlockHash NullBlockHash; +static const BlocksBlooms NullBlocksBlooms; } } diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index b6c310d34..165ac20ca 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -833,8 +833,6 @@ LocalisedLogEntries Client::logs(LogFilter const& _f) const LocalisedLogEntries ret; unsigned begin = min(m_bc.number() + 1, (unsigned)_f.latest()); unsigned end = min(m_bc.number(), min(begin, (unsigned)_f.earliest())); - unsigned m = _f.max(); - unsigned s = _f.skip(); // Handle pending transactions differently as they're not on the block chain. if (begin > m_bc.number()) @@ -847,68 +845,52 @@ LocalisedLogEntries Client::logs(LogFilter const& _f) const auto sha3 = m_postMine.pending()[i].sha3(); LogEntries le = _f.matches(tr); if (le.size()) - { - for (unsigned j = 0; j < le.size() && ret.size() != m; ++j) - if (s) - s--; - else - ret.insert(ret.begin(), LocalisedLogEntry(le[j], begin, sha3)); - } + for (unsigned j = 0; j < le.size(); ++j) + ret.insert(ret.begin(), LocalisedLogEntry(le[j], begin, sha3)); } begin = m_bc.number(); } + set matchingBlocks; + for (auto const& i: _f.bloomPossibilities()) + for (auto u: m_bc.withBlockBloom(i, end, begin)) + matchingBlocks.insert(u); + #if ETH_DEBUG - // fill these params - unsigned skipped = 0; unsigned falsePos = 0; #endif - auto h = m_bc.numberHash(begin); - unsigned n = begin; - for (; ret.size() != m && n != end; n--, h = m_bc.details(h).parent) + for (auto n: matchingBlocks) { #if ETH_DEBUG int total = 0; #endif - // check block bloom - auto info = m_bc.info(h); + auto h = m_bc.numberHash(n); auto receipts = m_bc.receipts(h).receipts; - if (_f.matches(info.logBloom)) - for (size_t i = 0; i < receipts.size(); i++) + for (size_t i = 0; i < receipts.size(); i++) + { + TransactionReceipt receipt = receipts[i]; + if (_f.matches(receipt.bloom())) { - TransactionReceipt receipt = receipts[i]; - if (_f.matches(receipt.bloom())) + auto info = m_bc.info(h); + auto h = transaction(info.hash, i).sha3(); + LogEntries le = _f.matches(receipt); + if (le.size()) { - auto h = transaction(info.hash, i).sha3(); - LogEntries le = _f.matches(receipt); - if (le.size()) - { #if ETH_DEBUG - total += le.size(); + total += le.size(); #endif - for (unsigned j = 0; j < le.size() && ret.size() != m; ++j) - { - if (s) - s--; - else - ret.insert(ret.begin(), LocalisedLogEntry(le[j], n, h)); - } - } + for (unsigned j = 0; j < le.size(); ++j) + ret.insert(ret.begin(), LocalisedLogEntry(le[j], n, h)); } -#if ETH_DEBUG - if (!total) - falsePos++; -#endif } #if ETH_DEBUG - else - skipped++; + if (!total) + falsePos++; #endif - if (n == end) - break; + } } #if ETH_DEBUG - cdebug << (begin - n) << "searched; " << skipped << "skipped; " << falsePos << "false +ves"; + cdebug << matchingBlocks.size() << "searched from" << (end - begin) << "skipped; " << falsePos << "false +ves"; #endif return ret; } diff --git a/libethereum/LogFilter.cpp b/libethereum/LogFilter.cpp index 1784094b0..d1b3cf437 100644 --- a/libethereum/LogFilter.cpp +++ b/libethereum/LogFilter.cpp @@ -30,13 +30,13 @@ using namespace dev::eth; std::ostream& dev::eth::operator<<(std::ostream& _out, LogFilter const& _s) { // TODO - _out << "(@" << _s.m_addresses << "#" << _s.m_topics << ">" << _s.m_earliest << "-" << _s.m_latest << "< +" << _s.m_skip << "^" << _s.m_max << ")"; + _out << "(@" << _s.m_addresses << "#" << _s.m_topics << ">" << _s.m_earliest << "-" << _s.m_latest << "< )"; return _out; } void LogFilter::streamRLP(RLPStream& _s) const { - _s.appendList(6) << m_addresses << m_topics << m_earliest << m_latest << m_max << m_skip; + _s.appendList(4) << m_addresses << m_topics << m_earliest << m_latest; } h256 LogFilter::sha3() const @@ -73,6 +73,16 @@ bool LogFilter::matches(State const& _s, unsigned _i) const return matches(_s.receipt(_i)).size() > 0; } +vector LogFilter::bloomPossibilities() const +{ + // return combination of each of the addresses/topics + vector ret; + // TODO proper combinatorics. + for (auto i: m_addresses) + ret.push_back(LogBloom().shiftBloom<3, 32>(dev::sha3(i))); + return ret; +} + LogEntries LogFilter::matches(TransactionReceipt const& _m) const { LogEntries ret; diff --git a/libethereum/LogFilter.h b/libethereum/LogFilter.h index 7b8922a03..a28bfe806 100644 --- a/libethereum/LogFilter.h +++ b/libethereum/LogFilter.h @@ -45,23 +45,21 @@ class State; class LogFilter { public: - LogFilter(int _earliest = 0, int _latest = -1, unsigned _max = 10, unsigned _skip = 0): m_earliest(_earliest), m_latest(_latest), m_max(_max), m_skip(_skip) {} + LogFilter(int _earliest = 0, int _latest = -1): m_earliest(_earliest), m_latest(_latest) {} void streamRLP(RLPStream& _s) const; h256 sha3() const; int earliest() const { return m_earliest; } int latest() const { return m_latest; } - unsigned max() const { return m_max; } - unsigned skip() const { return m_skip; } + + std::vector bloomPossibilities() const; bool matches(LogBloom _bloom) const; bool matches(State const& _s, unsigned _i) const; LogEntries matches(TransactionReceipt const& _r) const; LogFilter address(Address _a) { m_addresses.insert(_a); return *this; } LogFilter topic(unsigned _index, h256 const& _t) { if (_index < 4) m_topics[_index].insert(_t); return *this; } - LogFilter withMax(unsigned _m) { m_max = _m; return *this; } - LogFilter withSkip(unsigned _m) { m_skip = _m; return *this; } LogFilter withEarliest(int _e) { m_earliest = _e; return *this; } LogFilter withLatest(int _e) { m_latest = _e; return *this; } @@ -72,8 +70,6 @@ private: std::array m_topics; int m_earliest = 0; int m_latest = -1; - unsigned m_max = 10; - unsigned m_skip = 0; }; } diff --git a/libweb3jsonrpc/WebThreeStubServerBase.cpp b/libweb3jsonrpc/WebThreeStubServerBase.cpp index 70c8ca9de..09c34feb0 100644 --- a/libweb3jsonrpc/WebThreeStubServerBase.cpp +++ b/libweb3jsonrpc/WebThreeStubServerBase.cpp @@ -128,10 +128,6 @@ static dev::eth::LogFilter toLogFilter(Json::Value const& _json) // commented to filter.withEarliest(_json["earliest"].asInt()); if (_json["latest"].isInt()) filter.withLatest(_json["lastest"].asInt()); - if (_json["max"].isInt()) - filter.withMax(_json["max"].asInt()); - if (_json["skip"].isInt()) - filter.withSkip(_json["skip"].asInt()); if (!_json["address"].empty()) { if (_json["address"].isArray()) diff --git a/mix/MixClient.cpp b/mix/MixClient.cpp index 061057bb4..a06bbb02d 100644 --- a/mix/MixClient.cpp +++ b/mix/MixClient.cpp @@ -331,7 +331,6 @@ eth::LocalisedLogEntries MixClient::logs(eth::LogFilter const& _f) const unsigned lastBlock = bc().number(); unsigned block = std::min(lastBlock, (unsigned)_f.latest()); unsigned end = std::min(lastBlock, std::min(block, (unsigned)_f.earliest())); - unsigned skip = _f.skip(); // Pending transactions if (block > bc().number()) { @@ -341,9 +340,8 @@ eth::LocalisedLogEntries MixClient::logs(eth::LogFilter const& _f) const // Might have a transaction that contains a matching log. TransactionReceipt const& tr = m_state.receipt(i); LogEntries logEntries = _f.matches(tr); - for (unsigned entry = 0; entry < logEntries.size() && ret.size() != _f.max(); ++entry) + for (unsigned entry = 0; entry < logEntries.size(); ++entry) ret.insert(ret.begin(), LocalisedLogEntry(logEntries[entry], block)); - skip -= std::min(skip, static_cast(logEntries.size())); } block = bc().number(); } @@ -355,12 +353,8 @@ eth::LocalisedLogEntries MixClient::logs(eth::LogFilter const& _f) const if (_f.matches(bc().info(h).logBloom)) for (TransactionReceipt receipt: bc().receipts(h).receipts) if (_f.matches(receipt.bloom())) - { - LogEntries logEntries = _f.matches(receipt); - for (unsigned entry = skip; entry < logEntries.size() && ret.size() != _f.max(); ++entry) - ret.insert(ret.begin(), LocalisedLogEntry(logEntries[entry], block)); - skip -= std::min(skip, static_cast(logEntries.size())); - } + for (auto const& e: _f.matches(receipt)) + ret.insert(ret.begin(), LocalisedLogEntry(e, block)); h = bc().details(h).parent; } return ret;