Merge branch 'recursivebloom' into develop

alethzero/MainWin.cpp

@@ -1012,15 +1012,17 @@ void Main::refreshBlockChain()
 			h256 h(f.toStdString());
 			if (bc.isKnown(h))
 				blocks.insert(h);
+			for (auto const& b: bc.withBlockBloom(LogBloom().shiftBloom<3, 32>(sha3(h)), 0, -1))
+				blocks.insert(bc.numberHash(b));
 		}
 		else if (f.toLongLong() <= bc.number())
 			blocks.insert(bc.numberHash(u256(f.toLongLong())));
-		/*else if (f.size() == 40)
+		else if (f.size() == 40)
 		{
-			Address h(f[0]);
-			if (bc.(h))
-				blocks.insert(h);
-		}*/
+			Address h(f.toStdString());
+			for (auto const& b: bc.withBlockBloom(LogBloom().shiftBloom<3, 32>(sha3(h)), 0, -1))
+				blocks.insert(bc.numberHash(b));
+		}
 
 	QByteArray oldSelected = ui->blocks->count() ? ui->blocks->currentItem()->data(Qt::UserRole).toByteArray() : QByteArray();
 	ui->blocks->clear();

libdevcore/Common.cpp

@@ -27,7 +27,7 @@ using namespace dev;
 namespace dev
 {
 
-char const* Version = "0.8.2";
+char const* Version = "0.9.0";
 
 }
 

libdevcore/FixedHash.h

@@ -154,25 +154,17 @@ public:
 		}
 	};
 
-	inline FixedHash<32> bloom() const
-	{
-		FixedHash<32> ret;
-		for (auto i: m_data)
-			ret[i / 8] |= 1 << (i % 8);
-		return ret;
-	}
-
 	template <unsigned P, unsigned M> inline FixedHash& shiftBloom(FixedHash<M> const& _h)
 	{
-		return (*this |= _h.template nbloom<P, N>());
+		return (*this |= _h.template bloom<P, N>());
 	}
 
 	template <unsigned P, unsigned M> inline bool containsBloom(FixedHash<M> const& _h)
 	{
-		return contains(_h.template nbloom<P, N>());
+		return contains(_h.template bloom<P, N>());
 	}
 
-	template <unsigned P, unsigned M> inline FixedHash<M> nbloom() const
+	template <unsigned P, unsigned M> inline FixedHash<M> bloom() const
 	{
 		static const unsigned c_bloomBits = M * 8;
 		unsigned mask = c_bloomBits - 1;

libethereum/BlockChain.cpp

@@ -324,6 +324,18 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
 			WriteGuard l(x_blockHashes);
 			m_blockHashes[h256(bi.number)].value = newHash;
 		}
+		{
+			WriteGuard l(x_blocksBlooms);
+			LogBloom blockBloom = bi.logBloom;
+			blockBloom.shiftBloom<3, 32>(sha3(bi.coinbaseAddress.ref()));
+			unsigned index = (unsigned)bi.number;
+			for (unsigned level = 0; level < c_bloomIndexLevels; level++, index /= c_bloomIndexSize)
+			{
+				unsigned i = index / c_bloomIndexSize % c_bloomIndexSize;
+				unsigned o = index % c_bloomIndexSize;
+				m_blocksBlooms[chunkId(level, i)].blooms[o] |= blockBloom;
+			}
+		}
 		// Collate transaction hashes and remember who they were.
 		h256s tas;
 		{
@@ -346,6 +358,13 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
 			m_receipts[newHash] = br;
 		}
 
+		{
+			ReadGuard l1(x_blocksBlooms);
+			ReadGuard l2(x_details);
+			ReadGuard l3(x_blockHashes);
+			ReadGuard l4(x_receipts);
+			ReadGuard l5(x_logBlooms);
+			ReadGuard l6(x_transactionAddresses);
 			m_blocksDB->Put(m_writeOptions, toSlice(newHash), (ldb::Slice)ref(_block));
 			m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[newHash].rlp()));
 			m_extrasDB->Put(m_writeOptions, toSlice(bi.parentHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp()));
@@ -354,6 +373,8 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
 				m_extrasDB->Put(m_writeOptions, toSlice(h, ExtraTransactionAddress), (ldb::Slice)dev::ref(m_transactionAddresses[h].rlp()));
 			m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraLogBlooms), (ldb::Slice)dev::ref(m_logBlooms[newHash].rlp()));
 			m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraReceipts), (ldb::Slice)dev::ref(m_receipts[newHash].rlp()));
+			m_extrasDB->Put(m_writeOptions, toSlice(newHash, ExtraBlocksBlooms), (ldb::Slice)dev::ref(m_blocksBlooms[newHash].rlp()));
+		}
 
 #if ETH_PARANOIA
 		checkConsistency();
@@ -475,29 +496,30 @@ template <class T> static unsigned getHashSize(map<h256, T> const& _map)
 void BlockChain::updateStats() const
 {
 	{
-		ReadGuard l1(x_blocks);
+		ReadGuard l(x_blocks);
 		m_lastStats.memBlocks = 0;
 		for (auto const& i: m_blocks)
 			m_lastStats.memBlocks += i.second.size() + 64;
 	}
 	{
-		ReadGuard l2(x_details);
+		ReadGuard l(x_details);
 		m_lastStats.memDetails = getHashSize(m_details);
 	}
 	{
-		ReadGuard l5(x_logBlooms);
-		m_lastStats.memLogBlooms = getHashSize(m_logBlooms);
+		ReadGuard l1(x_logBlooms);
+		ReadGuard l2(x_blocksBlooms);
+		m_lastStats.memLogBlooms = getHashSize(m_logBlooms) + getHashSize(m_blocksBlooms);
 	}
 	{
-		ReadGuard l4(x_receipts);
+		ReadGuard l(x_receipts);
 		m_lastStats.memReceipts = getHashSize(m_receipts);
 	}
 	{
-		ReadGuard l3(x_blockHashes);
+		ReadGuard l(x_blockHashes);
 		m_lastStats.memBlockHashes = getHashSize(m_blockHashes);
 	}
 	{
-		ReadGuard l6(x_transactionAddresses);
+		ReadGuard l(x_transactionAddresses);
 		m_lastStats.memTransactionAddresses = getHashSize(m_transactionAddresses);
 	}
 }
@@ -520,6 +542,7 @@ void BlockChain::garbageCollect(bool _force)
 	WriteGuard l4(x_receipts);
 	WriteGuard l5(x_logBlooms);
 	WriteGuard l6(x_transactionAddresses);
+	WriteGuard l7(x_blocksBlooms);
 	for (CacheID const& id: m_cacheUsage.back())
 	{
 		m_inUse.erase(id);
@@ -544,6 +567,9 @@ void BlockChain::garbageCollect(bool _force)
 		case ExtraTransactionAddress:
 			m_transactionAddresses.erase(id.first);
 			break;
+		case ExtraBlocksBlooms:
+			m_blocksBlooms.erase(id.first);
+			break;
 		}
 	}
 	m_cacheUsage.pop_back();
@@ -579,6 +605,76 @@ void BlockChain::checkConsistency()
 	delete it;
 }
 
+static inline unsigned upow(unsigned a, unsigned b) { while (b-- > 0) a *= a; return a; }
+static inline unsigned ceilDiv(unsigned n, unsigned d) { return n / (n + d - 1); }
+static inline unsigned floorDivPow(unsigned n, unsigned a, unsigned b) { return n / upow(a, b); }
+static inline unsigned ceilDivPow(unsigned n, unsigned a, unsigned b) { return ceilDiv(n, upow(a, b)); }
+
+// Level 1
+// [xxx.            ]
+
+// Level 0
+// [.x............F.]
+// [........x.......]
+// [T.............x.]
+// [............    ]
+
+// F = 14. T = 32
+
+vector<unsigned> BlockChain::withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest) const
+{
+	vector<unsigned> ret;
+
+	// start from the top-level
+	unsigned u = upow(c_bloomIndexSize, c_bloomIndexLevels);
+
+	// run through each of the top-level blockbloom blocks
+	for (unsigned index = _earliest / u; index <= ceilDiv(_latest, u); ++index)				// 0
+		ret += withBlockBloom(_b, _earliest, _latest, c_bloomIndexLevels - 1, index);
+
+	return ret;
+}
+
+vector<unsigned> BlockChain::withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest, unsigned _level, unsigned _index) const
+{
+	// 14, 32, 1, 0
+		// 14, 32, 0, 0
+		// 14, 32, 0, 1
+		// 14, 32, 0, 2
+
+	vector<unsigned> ret;
+
+	unsigned uCourse = upow(c_bloomIndexSize, _level + 1);
+	// 256
+		// 16
+	unsigned uFine = upow(c_bloomIndexSize, _level);
+	// 16
+		// 1
+
+	unsigned obegin = _index == _earliest / uCourse ? _earliest / uFine % c_bloomIndexSize : 0;
+	// 0
+		// 14
+		// 0
+		// 0
+	unsigned oend = _index == _latest / uCourse ? (_latest / uFine) % c_bloomIndexSize + 1 : c_bloomIndexSize;
+	// 3
+		// 16
+		// 16
+		// 1
+
+	BlocksBlooms bb = blocksBlooms(_level, _index);
+	for (unsigned o = obegin; o < oend; ++o)
+		if (bb.blooms[o].contains(_b))
+		{
+			// This level has something like what we want.
+			if (_level > 0)
+				ret += withBlockBloom(_b, _earliest, _latest, _level - 1, o + _index * c_bloomIndexSize);
+			else
+				ret.push_back(o + _index * c_bloomIndexSize);
+		}
+	return ret;
+}
+
 h256Set BlockChain::allUnclesFrom(h256 _parent) const
 {
 	// Get all uncles cited given a parent (i.e. featured as uncles/main in parent, parent + 1, ... parent + 5).

libethereum/BlockChain.h

@@ -71,7 +71,8 @@ enum {
 	ExtraBlockHash,
 	ExtraTransactionAddress,
 	ExtraLogBlooms,
-	ExtraReceipts
+	ExtraReceipts,
+	ExtraBlocksBlooms
 };
 
 /**
@@ -132,6 +133,26 @@ public:
 	/// Get a list of transaction hashes for a given block. Thread-safe.
 	h256 numberHash(u256 _index) const { if (!_index) return genesisHash(); return queryExtras<BlockHash, ExtraBlockHash>(h256(_index), m_blockHashes, x_blockHashes, NullBlockHash).value; }
 
+	/** Get the block blooms for a number of blocks. Thread-safe.
+	 * @returns the object pertaining to the blocks:
+	 * level 0:
+	 * 0x, 0x + 1, .. (1x - 1)
+	 * 1x, 1x + 1, .. (2x - 1)
+	 * ...
+	 * (255x .. (256x - 1))
+	 * level 1:
+	 * 0x .. (1x - 1), 1x .. (2x - 1), ..., (255x .. (256x - 1))
+	 * 256x .. (257x - 1), 257x .. (258x - 1), ..., (511x .. (512x - 1))
+	 * ...
+	 * level n, index i, offset o:
+	 * i * (x ^ n) + o * x ^ (n - 1)
+	 */
+	BlocksBlooms blocksBlooms(unsigned _level, unsigned _index) const { return blocksBlooms(chunkId(_level, _index)); }
+	BlocksBlooms blocksBlooms(h256 const& _chunkId) const { return queryExtras<BlocksBlooms, ExtraBlocksBlooms>(_chunkId, m_blocksBlooms, x_blocksBlooms, NullBlocksBlooms); }
+	LogBloom blockBloom(unsigned _number) const { return blocksBlooms(chunkId(0, _number / c_bloomIndexSize)).blooms[_number % c_bloomIndexSize]; }
+	std::vector<unsigned> withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest) const;
+	std::vector<unsigned> withBlockBloom(LogBloom const& _b, unsigned _earliest, unsigned _latest, unsigned _topLevel, unsigned _index) const;
+
 	/// Get a transaction from its hash. Thread-safe.
 	bytes transaction(h256 _transactionHash) const { TransactionAddress ta = queryExtras<TransactionAddress, ExtraTransactionAddress>(_transactionHash, m_transactionAddresses, x_transactionAddresses, NullTransactionAddress); if (!ta) return bytes(); return transaction(ta.blockHash, ta.index); }
 
@@ -188,6 +209,8 @@ public:
 	void garbageCollect(bool _force = false);
 
 private:
+	static h256 chunkId(unsigned _level, unsigned _index) { return h256(_index * 0xff + _level); }
+
 	void open(std::string _path, bool _killExisting = false);
 	void close();
 
@@ -230,6 +253,8 @@ private:
 	mutable TransactionAddressHash m_transactionAddresses;
 	mutable SharedMutex x_blockHashes;
 	mutable BlockHashHash m_blockHashes;
+	mutable SharedMutex x_blocksBlooms;
+	mutable BlocksBloomsHash m_blocksBlooms;
 
 	using CacheID = std::pair<h256, unsigned>;
 	mutable Mutex x_cacheUsage;

libethereum/BlockDetails.h

@@ -36,6 +36,11 @@ namespace dev
 namespace eth
 {
 
+// TODO: OPTIMISE: constructors take bytes, RLP used only in necessary classes.
+
+static const unsigned c_bloomIndexSize = 16;
+static const unsigned c_bloomIndexLevels = 2;
+
 struct BlockDetails
 {
 	BlockDetails(): number(0), totalDifficulty(0) {}
@@ -64,6 +69,16 @@ struct BlockLogBlooms
 	mutable unsigned size;
 };
 
+struct BlocksBlooms
+{
+	BlocksBlooms() {}
+	BlocksBlooms(RLP const& _r) { blooms = _r.toArray<LogBloom, c_bloomIndexSize>(); size = _r.data().size(); }
+	bytes rlp() const { RLPStream s; s << blooms; size = s.out().size(); return s.out(); }
+
+	std::array<LogBloom, c_bloomIndexSize> blooms;
+	mutable unsigned size;
+};
+
 struct BlockReceipts
 {
 	BlockReceipts() {}
@@ -103,12 +118,14 @@ using BlockLogBloomsHash = std::map<h256, BlockLogBlooms>;
 using BlockReceiptsHash = std::map<h256, BlockReceipts>;
 using TransactionAddressHash = std::map<h256, TransactionAddress>;
 using BlockHashHash = std::map<h256, BlockHash>;
+using BlocksBloomsHash = std::map<h256, BlocksBlooms>;
 
 static const BlockDetails NullBlockDetails;
 static const BlockLogBlooms NullBlockLogBlooms;
 static const BlockReceipts NullBlockReceipts;
 static const TransactionAddress NullTransactionAddress;
 static const BlockHash NullBlockHash;
+static const BlocksBlooms NullBlocksBlooms;
 
 }
 }

libethereum/Client.cpp

@@ -833,8 +833,6 @@ LocalisedLogEntries Client::logs(LogFilter const& _f) const
 	LocalisedLogEntries ret;
 	unsigned begin = min<unsigned>(m_bc.number() + 1, (unsigned)_f.latest());
 	unsigned end = min(m_bc.number(), min(begin, (unsigned)_f.earliest()));
-	unsigned m = _f.max();
-	unsigned s = _f.skip();
 
 	// Handle pending transactions differently as they're not on the block chain.
 	if (begin > m_bc.number())
@@ -847,38 +845,33 @@ LocalisedLogEntries Client::logs(LogFilter const& _f) const
 			auto sha3 = m_postMine.pending()[i].sha3();
 			LogEntries le = _f.matches(tr);
 			if (le.size())
-			{
-				for (unsigned j = 0; j < le.size() && ret.size() != m; ++j)
-					if (s)
-						s--;
-					else
+				for (unsigned j = 0; j < le.size(); ++j)
 					ret.insert(ret.begin(), LocalisedLogEntry(le[j], begin, sha3));
 		}
-		}
 		begin = m_bc.number();
 	}
 
+	set<unsigned> matchingBlocks;
+	for (auto const& i: _f.bloomPossibilities())
+		for (auto u: m_bc.withBlockBloom(i, end, begin))
+			matchingBlocks.insert(u);
+
 #if ETH_DEBUG
-	// fill these params
-	unsigned skipped = 0;
 	unsigned falsePos = 0;
 #endif
-	auto h = m_bc.numberHash(begin);
-	unsigned n = begin;
-	for (; ret.size() != m && n != end; n--, h = m_bc.details(h).parent)
+	for (auto n: matchingBlocks)
 	{
 #if ETH_DEBUG
 		int total = 0;
 #endif
-		// check block bloom
-		auto info = m_bc.info(h);
+		auto h = m_bc.numberHash(n);
 		auto receipts = m_bc.receipts(h).receipts;
-		if (_f.matches(info.logBloom))
 		for (size_t i = 0; i < receipts.size(); i++)
 		{
 			TransactionReceipt receipt = receipts[i];
 			if (_f.matches(receipt.bloom()))
 			{
+				auto info = m_bc.info(h);
 				auto h = transaction(info.hash, i).sha3();
 				LogEntries le = _f.matches(receipt);
 				if (le.size())
@@ -886,29 +879,18 @@ LocalisedLogEntries Client::logs(LogFilter const& _f) const
 #if ETH_DEBUG
 					total += le.size();
 #endif
-						for (unsigned j = 0; j < le.size() && ret.size() != m; ++j)
-						{
-							if (s)
-								s--;
-							else
+					for (unsigned j = 0; j < le.size(); ++j)
 						ret.insert(ret.begin(), LocalisedLogEntry(le[j], n, h));
 				}
 			}
-				}
 #if ETH_DEBUG
 			if (!total)
 				falsePos++;
 #endif
 		}
-#if ETH_DEBUG
-		else
-			skipped++;
-#endif
-		if (n == end)
-			break;
 	}
 #if ETH_DEBUG
-	cdebug << (begin - n) << "searched; " << skipped << "skipped; " << falsePos << "false +ves";
+	cdebug << matchingBlocks.size() << "searched from" << (end - begin) << "skipped; " << falsePos << "false +ves";
 #endif
 	return ret;
 }

libethereum/LogFilter.cpp

@@ -30,13 +30,13 @@ using namespace dev::eth;
 std::ostream& dev::eth::operator<<(std::ostream& _out, LogFilter const& _s)
 {
 	// TODO
-	_out << "(@" << _s.m_addresses << "#" << _s.m_topics << ">" << _s.m_earliest << "-" << _s.m_latest << "< +" << _s.m_skip << "^" << _s.m_max << ")";
+	_out << "(@" << _s.m_addresses << "#" << _s.m_topics << ">" << _s.m_earliest << "-" << _s.m_latest << "< )";
 	return _out;
 }
 
 void LogFilter::streamRLP(RLPStream& _s) const
 {
-	_s.appendList(6) << m_addresses << m_topics << m_earliest << m_latest << m_max << m_skip;
+	_s.appendList(4) << m_addresses << m_topics << m_earliest << m_latest;
 }
 
 h256 LogFilter::sha3() const
@@ -73,6 +73,16 @@ bool LogFilter::matches(State const& _s, unsigned _i) const
 	return matches(_s.receipt(_i)).size() > 0;
 }
 
+vector<LogBloom> LogFilter::bloomPossibilities() const
+{
+	// return combination of each of the addresses/topics
+	vector<LogBloom> ret;
+	// TODO proper combinatorics.
+	for (auto i: m_addresses)
+		ret.push_back(LogBloom().shiftBloom<3, 32>(dev::sha3(i)));
+	return ret;
+}
+
 LogEntries LogFilter::matches(TransactionReceipt const& _m) const
 {
 	LogEntries ret;

libethereum/LogFilter.h

@@ -45,23 +45,21 @@ class State;
 class LogFilter
 {
 public:
-	LogFilter(int _earliest = 0, int _latest = -1, unsigned _max = 10, unsigned _skip = 0): m_earliest(_earliest), m_latest(_latest), m_max(_max), m_skip(_skip) {}
+	LogFilter(int _earliest = 0, int _latest = -1): m_earliest(_earliest), m_latest(_latest) {}
 
 	void streamRLP(RLPStream& _s) const;
 	h256 sha3() const;
 
 	int earliest() const { return m_earliest; }
 	int latest() const { return m_latest; }
-	unsigned max() const { return m_max; }
-	unsigned skip() const { return m_skip; }
+
+	std::vector<LogBloom> bloomPossibilities() const;
 	bool matches(LogBloom _bloom) const;
 	bool matches(State const& _s, unsigned _i) const;
 	LogEntries matches(TransactionReceipt const& _r) const;
 
 	LogFilter address(Address _a) { m_addresses.insert(_a); return *this; }
 	LogFilter topic(unsigned _index, h256 const& _t) { if (_index < 4) m_topics[_index].insert(_t); return *this; }
-	LogFilter withMax(unsigned _m) { m_max = _m; return *this; }
-	LogFilter withSkip(unsigned _m) { m_skip = _m; return *this; }
 	LogFilter withEarliest(int _e) { m_earliest = _e; return *this; }
 	LogFilter withLatest(int _e) { m_latest = _e; return *this; }
 
@@ -72,8 +70,6 @@ private:
 	std::array<h256Set, 4> m_topics;
 	int m_earliest = 0;
 	int m_latest = -1;
-	unsigned m_max = 10;
-	unsigned m_skip = 0;
 };
 
 }

libweb3jsonrpc/WebThreeStubServerBase.cpp

@@ -128,10 +128,6 @@ static dev::eth::LogFilter toLogFilter(Json::Value const& _json)	// commented to
 		filter.withEarliest(_json["earliest"].asInt());
 	if (_json["latest"].isInt())
 		filter.withLatest(_json["lastest"].asInt());
-	if (_json["max"].isInt())
-		filter.withMax(_json["max"].asInt());
-	if (_json["skip"].isInt())
-		filter.withSkip(_json["skip"].asInt());
 	if (!_json["address"].empty())
 	{
 		if (_json["address"].isArray())

mix/MixClient.cpp

@@ -331,7 +331,6 @@ eth::LocalisedLogEntries MixClient::logs(eth::LogFilter const& _f) const
 	unsigned lastBlock = bc().number();
 	unsigned block = std::min<unsigned>(lastBlock, (unsigned)_f.latest());
 	unsigned end = std::min(lastBlock, std::min(block, (unsigned)_f.earliest()));
-	unsigned skip = _f.skip();
 	// Pending transactions
 	if (block > bc().number())
 	{
@@ -341,9 +340,8 @@ eth::LocalisedLogEntries MixClient::logs(eth::LogFilter const& _f) const
 			// Might have a transaction that contains a matching log.
 			TransactionReceipt const& tr = m_state.receipt(i);
 			LogEntries logEntries = _f.matches(tr);
-			for (unsigned entry = 0; entry < logEntries.size() && ret.size() != _f.max(); ++entry)
+			for (unsigned entry = 0; entry < logEntries.size(); ++entry)
 				ret.insert(ret.begin(), LocalisedLogEntry(logEntries[entry], block));
-			skip -= std::min(skip, static_cast<unsigned>(logEntries.size()));
 		}
 		block = bc().number();
 	}
@@ -355,12 +353,8 @@ eth::LocalisedLogEntries MixClient::logs(eth::LogFilter const& _f) const
 		if (_f.matches(bc().info(h).logBloom))
 			for (TransactionReceipt receipt: bc().receipts(h).receipts)
 				if (_f.matches(receipt.bloom()))
-				{
-					LogEntries logEntries = _f.matches(receipt);
-					for (unsigned entry = skip; entry < logEntries.size() && ret.size() != _f.max(); ++entry)
-						ret.insert(ret.begin(), LocalisedLogEntry(logEntries[entry], block));
-					skip -= std::min(skip, static_cast<unsigned>(logEntries.size()));
-				}
+					for (auto const& e: _f.matches(receipt))
+						ret.insert(ret.begin(), LocalisedLogEntry(e, block));
 		h = bc().details(h).parent;
 	}
 	return ret;