From 4a684ea86fe269e027e015caf54589adba086182 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 11 Mar 2015 10:05:41 +0000 Subject: [PATCH] First draft of recursive bloom. --- libdevcore/Common.cpp | 2 +- libdevcore/FixedHash.h | 8 ---- libethereum/BlockChain.cpp | 94 ++++++++++++++++++++++++++++++++------ libethereum/BlockChain.h | 27 ++++++++++- libethereum/BlockDetails.h | 17 +++++++ libethereum/Client.cpp | 9 ++-- 6 files changed, 129 insertions(+), 28 deletions(-) diff --git a/libdevcore/Common.cpp b/libdevcore/Common.cpp index 431a95580..07e92f542 100644 --- a/libdevcore/Common.cpp +++ b/libdevcore/Common.cpp @@ -27,7 +27,7 @@ using namespace dev; namespace dev { -char const* Version = "0.8.2"; +char const* Version = "0.9.0"; } diff --git a/libdevcore/FixedHash.h b/libdevcore/FixedHash.h index 49c6ed2bf..a52e1cb2e 100644 --- a/libdevcore/FixedHash.h +++ b/libdevcore/FixedHash.h @@ -154,14 +154,6 @@ public: } }; - inline FixedHash<32> bloom() const - { - FixedHash<32> ret; - for (auto i: m_data) - ret[i / 8] |= 1 << (i % 8); - return ret; - } - template inline FixedHash& shiftBloom(FixedHash const& _h) { return (*this |= _h.template nbloom()); diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index dd0bfad67..54f919647 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -324,6 +324,18 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db) WriteGuard l(x_blockHashes); m_blockHashes[h256(bi.number)].value = newHash; } + { + WriteGuard l(x_blocksBlooms); + LogBloom blockBloom = bi.logBloom; + blockBloom.shiftBloom<3, 32>(sha3(bi.coinbaseAddress.ref())); + unsigned index = (unsigned)bi.number; + for (unsigned level = 0; level < c_bloomIndexLevels; level++, index /= c_bloomIndexSize) + { + unsigned i = index / c_bloomIndexSize % c_bloomIndexSize; + unsigned o = index % c_bloomIndexSize; + m_blocksBlooms[chunkId(level, i)].blooms[o] |= blockBloom; + } + } // Collate transaction hashes and remember who they were. h256s tas; { @@ -346,14 +358,23 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db) m_receipts[newHash] = br; } - m_blocksDB->Put(m_writeOptions, toSlice(newHash), (ldb::Slice)ref(_block)); - m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[newHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(bi.parentHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(h256(bi.number), ExtraBlockHash), (ldb::Slice)dev::ref(m_blockHashes[h256(bi.number)].rlp())); - for (auto const& h: tas) - m_extrasDB->Put(m_writeOptions, toSlice(h, ExtraTransactionAddress), (ldb::Slice)dev::ref(m_transactionAddresses[h].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraLogBlooms), (ldb::Slice)dev::ref(m_logBlooms[newHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraReceipts), (ldb::Slice)dev::ref(m_receipts[newHash].rlp())); + { + ReadGuard l1(x_blocksBlooms); + ReadGuard l2(x_details); + ReadGuard l3(x_blockHashes); + ReadGuard l4(x_receipts); + ReadGuard l5(x_logBlooms); + ReadGuard l6(x_transactionAddresses); + m_blocksDB->Put(m_writeOptions, toSlice(newHash), (ldb::Slice)ref(_block)); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[newHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(bi.parentHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(h256(bi.number), ExtraBlockHash), (ldb::Slice)dev::ref(m_blockHashes[h256(bi.number)].rlp())); + for (auto const& h: tas) + m_extrasDB->Put(m_writeOptions, toSlice(h, ExtraTransactionAddress), (ldb::Slice)dev::ref(m_transactionAddresses[h].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraLogBlooms), (ldb::Slice)dev::ref(m_logBlooms[newHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraReceipts), (ldb::Slice)dev::ref(m_receipts[newHash].rlp())); + m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraBlocksBlooms), (ldb::Slice)dev::ref(m_blocksBlooms[newHash].rlp())); + } #if ETH_PARANOIA checkConsistency(); @@ -475,29 +496,30 @@ template static unsigned getHashSize(map const& _map) void BlockChain::updateStats() const { { - ReadGuard l1(x_blocks); + ReadGuard l(x_blocks); m_lastStats.memBlocks = 0; for (auto const& i: m_blocks) m_lastStats.memBlocks += i.second.size() + 64; } { - ReadGuard l2(x_details); + ReadGuard l(x_details); m_lastStats.memDetails = getHashSize(m_details); } { - ReadGuard l5(x_logBlooms); - m_lastStats.memLogBlooms = getHashSize(m_logBlooms); + ReadGuard l1(x_logBlooms); + ReadGuard l2(x_blocksBlooms); + m_lastStats.memLogBlooms = getHashSize(m_logBlooms) + getHashSize(m_blocksBlooms); } { - ReadGuard l4(x_receipts); + ReadGuard l(x_receipts); m_lastStats.memReceipts = getHashSize(m_receipts); } { - ReadGuard l3(x_blockHashes); + ReadGuard l(x_blockHashes); m_lastStats.memBlockHashes = getHashSize(m_blockHashes); } { - ReadGuard l6(x_transactionAddresses); + ReadGuard l(x_transactionAddresses); m_lastStats.memTransactionAddresses = getHashSize(m_transactionAddresses); } } @@ -520,6 +542,7 @@ void BlockChain::garbageCollect(bool _force) WriteGuard l4(x_receipts); WriteGuard l5(x_logBlooms); WriteGuard l6(x_transactionAddresses); + WriteGuard l7(x_blocksBlooms); for (CacheID const& id: m_cacheUsage.back()) { m_inUse.erase(id); @@ -544,6 +567,9 @@ void BlockChain::garbageCollect(bool _force) case ExtraTransactionAddress: m_transactionAddresses.erase(id.first); break; + case ExtraBlocksBlooms: + m_blocksBlooms.erase(id.first); + break; } } m_cacheUsage.pop_back(); @@ -579,6 +605,44 @@ void BlockChain::checkConsistency() delete it; } +static inline unsigned upow(unsigned a, unsigned b) { while (b-- > 0) a *= a; return a; } +static inline unsigned ceilDiv(unsigned n, unsigned d) { return n / (n + d - 1); } +static inline unsigned floorDivPow(unsigned n, unsigned a, unsigned b) { return n / upow(a, b); } +static inline unsigned ceilDivPow(unsigned n, unsigned a, unsigned b) { return ceilDiv(n, upow(a, b)); } + +vector BlockChain::withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest) const +{ + vector ret; + + // start from the top-level + unsigned u = upow(c_bloomIndexSize, c_bloomIndexLevels - 1); + + // run through each of the top-level blockbloom blocks + for (unsigned index = _earliest / u; index < ceilDiv(_latest, u); ++index) + ret += withBlockBloom(_b, _earliest, _latest, c_bloomIndexLevels - 1, index); + + return ret; +} + +vector BlockChain::withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest, unsigned _level, unsigned _index) const +{ + vector ret; + +// unsigned u = upow(c_bloomIndexSize, _level); + + BlocksBlooms bb = blocksBlooms(_level, _index); + for (unsigned o = 0; o < 256; ++o) // TODO: crop. + if (bb.blooms[o].contains(_b)) + { + // This level has something like what we want. + if (_level > 0) + ret += withBlockBloom(_b, _earliest, _latest, _level - 1, o + _index * c_bloomIndexSize); + else + ret.push_back(o + _index * c_bloomIndexSize); + } + return ret; +} + h256Set BlockChain::allUnclesFrom(h256 _parent) const { // Get all uncles cited given a parent (i.e. featured as uncles/main in parent, parent + 1, ... parent + 5). diff --git a/libethereum/BlockChain.h b/libethereum/BlockChain.h index 0c1a81066..a6760a3f6 100644 --- a/libethereum/BlockChain.h +++ b/libethereum/BlockChain.h @@ -71,7 +71,8 @@ enum { ExtraBlockHash, ExtraTransactionAddress, ExtraLogBlooms, - ExtraReceipts + ExtraReceipts, + ExtraBlocksBlooms }; /** @@ -132,6 +133,26 @@ public: /// Get a list of transaction hashes for a given block. Thread-safe. h256 numberHash(u256 _index) const { if (!_index) return genesisHash(); return queryExtras(h256(_index), m_blockHashes, x_blockHashes, NullBlockHash).value; } + /** Get the block blooms for a number of blocks. Thread-safe. + * @returns the object pertaining to the blocks: + * level 0: + * 0x, 0x + 1, .. (1x - 1) + * 1x, 1x + 1, .. (2x - 1) + * ... + * (255x .. (256x - 1)) + * level 1: + * 0x .. (1x - 1), 1x .. (2x - 1), ..., (255x .. (256x - 1)) + * 256x .. (257x - 1), 257x .. (258x - 1), ..., (511x .. (512x - 1)) + * ... + * level n, index i, offset o: + * i * (x ^ n) + o * x ^ (n - 1) + */ + BlocksBlooms blocksBlooms(unsigned _level, unsigned _index) const { return blocksBlooms(chunkId(_level, _index)); } + BlocksBlooms blocksBlooms(h256 const& _chunkId) const { return queryExtras(_chunkId, m_blocksBlooms, x_blocksBlooms, NullBlocksBlooms); } + LogBloom blockBloom(unsigned _number) const { return blocksBlooms(chunkId(0, _number / c_bloomIndexSize)).blooms[_number % c_bloomIndexSize]; } + std::vector withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest) const; + std::vector withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest, unsigned _topLevel, unsigned _index) const; + /// Get a transaction from its hash. Thread-safe. bytes transaction(h256 _transactionHash) const { TransactionAddress ta = queryExtras(_transactionHash, m_transactionAddresses, x_transactionAddresses, NullTransactionAddress); if (!ta) return bytes(); return transaction(ta.blockHash, ta.index); } @@ -188,6 +209,8 @@ public: void garbageCollect(bool _force = false); private: + static h256 chunkId(unsigned _level, unsigned _index) { return h256(_index * 0xff + _level); } + void open(std::string _path, bool _killExisting = false); void close(); @@ -230,6 +253,8 @@ private: mutable TransactionAddressHash m_transactionAddresses; mutable SharedMutex x_blockHashes; mutable BlockHashHash m_blockHashes; + mutable SharedMutex x_blocksBlooms; + mutable BlocksBloomsHash m_blocksBlooms; using CacheID = std::pair; mutable Mutex x_cacheUsage; diff --git a/libethereum/BlockDetails.h b/libethereum/BlockDetails.h index ed478568d..f3bb73059 100644 --- a/libethereum/BlockDetails.h +++ b/libethereum/BlockDetails.h @@ -36,6 +36,11 @@ namespace dev namespace eth { +// TODO: OPTIMISE: constructors take bytes, RLP used only in necessary classes. + +static const unsigned c_bloomIndexSize = 16; +static const unsigned c_bloomIndexLevels = 6; + struct BlockDetails { BlockDetails(): number(0), totalDifficulty(0) {} @@ -64,6 +69,16 @@ struct BlockLogBlooms mutable unsigned size; }; +struct BlocksBlooms +{ + BlocksBlooms() {} + BlocksBlooms(RLP const& _r) { blooms = _r.toArray(); size = _r.data().size(); } + bytes rlp() const { RLPStream s; s << blooms; size = s.out().size(); return s.out(); } + + std::array blooms; + mutable unsigned size; +}; + struct BlockReceipts { BlockReceipts() {} @@ -103,12 +118,14 @@ using BlockLogBloomsHash = std::map; using BlockReceiptsHash = std::map; using TransactionAddressHash = std::map; using BlockHashHash = std::map; +using BlocksBloomsHash = std::map; static const BlockDetails NullBlockDetails; static const BlockLogBlooms NullBlockLogBlooms; static const BlockReceipts NullBlockReceipts; static const TransactionAddress NullTransactionAddress; static const BlockHash NullBlockHash; +static const BlocksBlooms NullBlocksBlooms; } } diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index b6c310d34..9099986c0 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -871,14 +871,16 @@ LocalisedLogEntries Client::logs(LogFilter const& _f) const int total = 0; #endif // check block bloom - auto info = m_bc.info(h); - auto receipts = m_bc.receipts(h).receipts; - if (_f.matches(info.logBloom)) + auto blockBloom = m_bc.blockBloom(n); + if (_f.matches(blockBloom)) + { + auto receipts = m_bc.receipts(h).receipts; for (size_t i = 0; i < receipts.size(); i++) { TransactionReceipt receipt = receipts[i]; if (_f.matches(receipt.bloom())) { + auto info = m_bc.info(h); auto h = transaction(info.hash, i).sha3(); LogEntries le = _f.matches(receipt); if (le.size()) @@ -901,6 +903,7 @@ LocalisedLogEntries Client::logs(LogFilter const& _f) const #endif } #if ETH_DEBUG + } else skipped++; #endif