First draft of recursive bloom.

libdevcore/Common.cpp

@@ -27,7 +27,7 @@ using namespace dev;
 namespace dev
 {
 
-char const* Version = "0.8.2";
+char const* Version = "0.9.0";
 
 }
 

libdevcore/FixedHash.h

@@ -154,14 +154,6 @@ public:
 		}
 	};
 
-	inline FixedHash<32> bloom() const
-	{
-		FixedHash<32> ret;
-		for (auto i: m_data)
-			ret[i / 8] |= 1 << (i % 8);
-		return ret;
-	}
-
 	template <unsigned P, unsigned M> inline FixedHash& shiftBloom(FixedHash<M> const& _h)
 	{
 		return (*this |= _h.template nbloom<P, N>());

libethereum/BlockChain.cpp

@@ -324,6 +324,18 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
 			WriteGuard l(x_blockHashes);
 			m_blockHashes[h256(bi.number)].value = newHash;
 		}
+		{
+			WriteGuard l(x_blocksBlooms);
+			LogBloom blockBloom = bi.logBloom;
+			blockBloom.shiftBloom<3, 32>(sha3(bi.coinbaseAddress.ref()));
+			unsigned index = (unsigned)bi.number;
+			for (unsigned level = 0; level < c_bloomIndexLevels; level++, index /= c_bloomIndexSize)
+			{
+				unsigned i = index / c_bloomIndexSize % c_bloomIndexSize;
+				unsigned o = index % c_bloomIndexSize;
+				m_blocksBlooms[chunkId(level, i)].blooms[o] |= blockBloom;
+			}
+		}
 		// Collate transaction hashes and remember who they were.
 		h256s tas;
 		{
@@ -346,14 +358,23 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
 			m_receipts[newHash] = br;
 		}
 
-		m_blocksDB->Put(m_writeOptions, toSlice(newHash), (ldb::Slice)ref(_block));
+		{
-		m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[newHash].rlp()));
+			ReadGuard l1(x_blocksBlooms);
-		m_extrasDB->Put(m_writeOptions, toSlice(bi.parentHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp()));
+			ReadGuard l2(x_details);
-		m_extrasDB->Put(m_writeOptions, toSlice(h256(bi.number), ExtraBlockHash), (ldb::Slice)dev::ref(m_blockHashes[h256(bi.number)].rlp()));
+			ReadGuard l3(x_blockHashes);
-		for (auto const& h: tas)
+			ReadGuard l4(x_receipts);
-			m_extrasDB->Put(m_writeOptions, toSlice(h, ExtraTransactionAddress), (ldb::Slice)dev::ref(m_transactionAddresses[h].rlp()));
+			ReadGuard l5(x_logBlooms);
-		m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraLogBlooms), (ldb::Slice)dev::ref(m_logBlooms[newHash].rlp()));
+			ReadGuard l6(x_transactionAddresses);
-		m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraReceipts), (ldb::Slice)dev::ref(m_receipts[newHash].rlp()));
+			m_blocksDB->Put(m_writeOptions, toSlice(newHash), (ldb::Slice)ref(_block));
+			m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[newHash].rlp()));
+			m_extrasDB->Put(m_writeOptions, toSlice(bi.parentHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp()));
+			m_extrasDB->Put(m_writeOptions, toSlice(h256(bi.number), ExtraBlockHash), (ldb::Slice)dev::ref(m_blockHashes[h256(bi.number)].rlp()));
+			for (auto const& h: tas)
+				m_extrasDB->Put(m_writeOptions, toSlice(h, ExtraTransactionAddress), (ldb::Slice)dev::ref(m_transactionAddresses[h].rlp()));
+			m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraLogBlooms), (ldb::Slice)dev::ref(m_logBlooms[newHash].rlp()));
+			m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraReceipts), (ldb::Slice)dev::ref(m_receipts[newHash].rlp()));
+			m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraBlocksBlooms), (ldb::Slice)dev::ref(m_blocksBlooms[newHash].rlp()));
+		}
 
 #if ETH_PARANOIA
 		checkConsistency();
@@ -475,29 +496,30 @@ template <class T> static unsigned getHashSize(map<h256, T> const& _map)
 void BlockChain::updateStats() const
 {
 	{
-		ReadGuard l1(x_blocks);
+		ReadGuard l(x_blocks);
 		m_lastStats.memBlocks = 0;
 		for (auto const& i: m_blocks)
 			m_lastStats.memBlocks += i.second.size() + 64;
 	}
 	{
-		ReadGuard l2(x_details);
+		ReadGuard l(x_details);
 		m_lastStats.memDetails = getHashSize(m_details);
 	}
 	{
-		ReadGuard l5(x_logBlooms);
+		ReadGuard l1(x_logBlooms);
-		m_lastStats.memLogBlooms = getHashSize(m_logBlooms);
+		ReadGuard l2(x_blocksBlooms);
+		m_lastStats.memLogBlooms = getHashSize(m_logBlooms) + getHashSize(m_blocksBlooms);
 	}
 	{
-		ReadGuard l4(x_receipts);
+		ReadGuard l(x_receipts);
 		m_lastStats.memReceipts = getHashSize(m_receipts);
 	}
 	{
-		ReadGuard l3(x_blockHashes);
+		ReadGuard l(x_blockHashes);
 		m_lastStats.memBlockHashes = getHashSize(m_blockHashes);
 	}
 	{
-		ReadGuard l6(x_transactionAddresses);
+		ReadGuard l(x_transactionAddresses);
 		m_lastStats.memTransactionAddresses = getHashSize(m_transactionAddresses);
 	}
 }
@@ -520,6 +542,7 @@ void BlockChain::garbageCollect(bool _force)
 	WriteGuard l4(x_receipts);
 	WriteGuard l5(x_logBlooms);
 	WriteGuard l6(x_transactionAddresses);
+	WriteGuard l7(x_blocksBlooms);
 	for (CacheID const& id: m_cacheUsage.back())
 	{
 		m_inUse.erase(id);
@@ -544,6 +567,9 @@ void BlockChain::garbageCollect(bool _force)
 		case ExtraTransactionAddress:
 			m_transactionAddresses.erase(id.first);
 			break;
+		case ExtraBlocksBlooms:
+			m_blocksBlooms.erase(id.first);
+			break;
 		}
 	}
 	m_cacheUsage.pop_back();
@@ -579,6 +605,44 @@ void BlockChain::checkConsistency()
 	delete it;
 }
 
+static inline unsigned upow(unsigned a, unsigned b) { while (b-- > 0) a *= a; return a; }
+static inline unsigned ceilDiv(unsigned n, unsigned d) { return n / (n + d - 1); }
+static inline unsigned floorDivPow(unsigned n, unsigned a, unsigned b) { return n / upow(a, b); }
+static inline unsigned ceilDivPow(unsigned n, unsigned a, unsigned b) { return ceilDiv(n, upow(a, b)); }
+
+vector<unsigned> BlockChain::withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest) const
+{
+	vector<unsigned> ret;
+
+	// start from the top-level
+	unsigned u = upow(c_bloomIndexSize, c_bloomIndexLevels - 1);
+
+	// run through each of the top-level blockbloom blocks
+	for (unsigned index = _earliest / u; index < ceilDiv(_latest, u); ++index)
+		ret += withBlockBloom(_b, _earliest, _latest, c_bloomIndexLevels - 1, index);
+
+	return ret;
+}
+
+vector<unsigned> BlockChain::withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest, unsigned _level, unsigned _index) const
+{
+	vector<unsigned> ret;
+
+//	unsigned u = upow(c_bloomIndexSize, _level);
+
+	BlocksBlooms bb = blocksBlooms(_level, _index);
+	for (unsigned o = 0; o < 256; ++o)	// TODO: crop.
+		if (bb.blooms[o].contains(_b))
+		{
+			// This level has something like what we want.
+			if (_level > 0)
+				ret += withBlockBloom(_b, _earliest, _latest, _level - 1, o + _index * c_bloomIndexSize);
+			else
+				ret.push_back(o + _index * c_bloomIndexSize);
+		}
+	return ret;
+}
+
 h256Set BlockChain::allUnclesFrom(h256 _parent) const
 {
 	// Get all uncles cited given a parent (i.e. featured as uncles/main in parent, parent + 1, ... parent + 5).

libethereum/BlockChain.h

@@ -71,7 +71,8 @@ enum {
 	ExtraBlockHash,
 	ExtraTransactionAddress,
 	ExtraLogBlooms,
-	ExtraReceipts
+	ExtraReceipts,
+	ExtraBlocksBlooms
 };
 
 /**
@@ -132,6 +133,26 @@ public:
 	/// Get a list of transaction hashes for a given block. Thread-safe.
 	h256 numberHash(u256 _index) const { if (!_index) return genesisHash(); return queryExtras<BlockHash, ExtraBlockHash>(h256(_index), m_blockHashes, x_blockHashes, NullBlockHash).value; }
 
+	/** Get the block blooms for a number of blocks. Thread-safe.
+	 * @returns the object pertaining to the blocks:
+	 * level 0:
+	 * 0x, 0x + 1, .. (1x - 1)
+	 * 1x, 1x + 1, .. (2x - 1)
+	 * ...
+	 * (255x .. (256x - 1))
+	 * level 1:
+	 * 0x .. (1x - 1), 1x .. (2x - 1), ..., (255x .. (256x - 1))
+	 * 256x .. (257x - 1), 257x .. (258x - 1), ..., (511x .. (512x - 1))
+	 * ...
+	 * level n, index i, offset o:
+	 * i * (x ^ n) + o * x ^ (n - 1)
+	 */
+	BlocksBlooms blocksBlooms(unsigned _level, unsigned _index) const { return blocksBlooms(chunkId(_level, _index)); }
+	BlocksBlooms blocksBlooms(h256 const& _chunkId) const { return queryExtras<BlocksBlooms, ExtraBlocksBlooms>(_chunkId, m_blocksBlooms, x_blocksBlooms, NullBlocksBlooms); }
+	LogBloom blockBloom(unsigned _number) const { return blocksBlooms(chunkId(0, _number / c_bloomIndexSize)).blooms[_number % c_bloomIndexSize]; }
+	std::vector<unsigned> withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest) const;
+	std::vector<unsigned> withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest, unsigned _topLevel, unsigned _index) const;
+
 	/// Get a transaction from its hash. Thread-safe.
 	bytes transaction(h256 _transactionHash) const { TransactionAddress ta = queryExtras<TransactionAddress, ExtraTransactionAddress>(_transactionHash, m_transactionAddresses, x_transactionAddresses, NullTransactionAddress); if (!ta) return bytes(); return transaction(ta.blockHash, ta.index); }
 
@@ -188,6 +209,8 @@ public:
 	void garbageCollect(bool _force = false);
 
 private:
+	static h256 chunkId(unsigned _level, unsigned _index) { return h256(_index * 0xff + _level); }
+
 	void open(std::string _path, bool _killExisting = false);
 	void close();
 
@@ -230,6 +253,8 @@ private:
 	mutable TransactionAddressHash m_transactionAddresses;
 	mutable SharedMutex x_blockHashes;
 	mutable BlockHashHash m_blockHashes;
+	mutable SharedMutex x_blocksBlooms;
+	mutable BlocksBloomsHash m_blocksBlooms;
 
 	using CacheID = std::pair<h256, unsigned>;
 	mutable Mutex x_cacheUsage;

libethereum/BlockDetails.h

@@ -36,6 +36,11 @@ namespace dev
 namespace eth
 {
 
+// TODO: OPTIMISE: constructors take bytes, RLP used only in necessary classes.
+
+static const unsigned c_bloomIndexSize = 16;
+static const unsigned c_bloomIndexLevels = 6;
+
 struct BlockDetails
 {
 	BlockDetails(): number(0), totalDifficulty(0) {}
@@ -64,6 +69,16 @@ struct BlockLogBlooms
 	mutable unsigned size;
 };
 
+struct BlocksBlooms
+{
+	BlocksBlooms() {}
+	BlocksBlooms(RLP const& _r) { blooms = _r.toArray<LogBloom, c_bloomIndexSize>(); size = _r.data().size(); }
+	bytes rlp() const { RLPStream s; s << blooms; size = s.out().size(); return s.out(); }
+
+	std::array<LogBloom, c_bloomIndexSize> blooms;
+	mutable unsigned size;
+};
+
 struct BlockReceipts
 {
 	BlockReceipts() {}
@@ -103,12 +118,14 @@ using BlockLogBloomsHash = std::map<h256, BlockLogBlooms>;
 using BlockReceiptsHash = std::map<h256, BlockReceipts>;
 using TransactionAddressHash = std::map<h256, TransactionAddress>;
 using BlockHashHash = std::map<h256, BlockHash>;
+using BlocksBloomsHash = std::map<h256, BlocksBlooms>;
 
 static const BlockDetails NullBlockDetails;
 static const BlockLogBlooms NullBlockLogBlooms;
 static const BlockReceipts NullBlockReceipts;
 static const TransactionAddress NullTransactionAddress;
 static const BlockHash NullBlockHash;
+static const BlocksBlooms NullBlocksBlooms;
 
 }
 }

libethereum/Client.cpp

@@ -871,14 +871,16 @@ LocalisedLogEntries Client::logs(LogFilter const& _f) const
 		int total = 0;
 #endif
 		// check block bloom
-		auto info = m_bc.info(h);
+		auto blockBloom = m_bc.blockBloom(n);
-		auto receipts = m_bc.receipts(h).receipts;
+		if (_f.matches(blockBloom))
-		if (_f.matches(info.logBloom))
+		{
+			auto receipts = m_bc.receipts(h).receipts;
 			for (size_t i = 0; i < receipts.size(); i++)
 			{
 				TransactionReceipt receipt = receipts[i];
 				if (_f.matches(receipt.bloom()))
 				{
+					auto info = m_bc.info(h);
 					auto h = transaction(info.hash, i).sha3();
 					LogEntries le = _f.matches(receipt);
 					if (le.size())
@@ -901,6 +903,7 @@ LocalisedLogEntries Client::logs(LogFilter const& _f) const
 #endif
 			}
 #if ETH_DEBUG
+		}
 		else
 			skipped++;
 #endif