Merge pull request #2378 from gluk256/_2_rating

Initial version of the rating system

libwhisper/BloomFilter.h

@@ -35,17 +35,20 @@ public:
 	TopicBloomFilterBase() { init(); }
 	TopicBloomFilterBase(FixedHash<N> const& _h): FixedHash<N>(_h) { init(); }
 
-	void addBloom(dev::shh::AbridgedTopic const& _h) { addRaw(_h.template bloomPart<BitsPerBloom, N>()); }
-	void removeBloom(dev::shh::AbridgedTopic const& _h) { removeRaw(_h.template bloomPart<BitsPerBloom, N>()); }
-	bool containsBloom(dev::shh::AbridgedTopic const& _h) const { return this->contains(_h.template bloomPart<BitsPerBloom, N>()); }
+	void addBloom(AbridgedTopic const& _h) { addRaw(bloom(_h)); }
+	void removeBloom(AbridgedTopic const& _h) { removeRaw(bloom(_h)); }
+	bool containsBloom(AbridgedTopic const& _h) const { return this->contains(bloom(_h)); }
 
 	void addRaw(FixedHash<N> const& _h);
 	void removeRaw(FixedHash<N> const& _h);
 	bool containsRaw(FixedHash<N> const& _h) const { return this->contains(_h); }
 
+	static FixedHash<N> bloom(AbridgedTopic const& _h);
+	static void setBit(FixedHash<N>& _h, unsigned index);
+	static bool isBitSet(FixedHash<N> const& _h, unsigned _index);
+	
 private:
 	void init() { for (unsigned i = 0; i < CounterSize; ++i) m_refCounter[i] = 0; }
-	static bool isBitSet(FixedHash<N> const& _h, unsigned _index);
 
 	static const unsigned CounterSize = N * 8;
 	std::array<uint16_t, CounterSize> m_refCounter;
@@ -85,10 +88,49 @@ template <unsigned N>
 bool TopicBloomFilterBase<N>::isBitSet(FixedHash<N> const& _h, unsigned _index)
 {
 	unsigned iByte = _index / 8;
-	unsigned iBit = _index % 8;
+	unsigned iBit = _index & 0x7;
 	return (_h[iByte] & c_powerOfTwoBitMmask[iBit]) != 0;
 }
 
+template <unsigned N>
+void TopicBloomFilterBase<N>::setBit(FixedHash<N>& _h, unsigned _index)
+{
+	unsigned iByte = _index / 8;
+	unsigned iBit = _index & 0x7;
+	_h[iByte] |= c_powerOfTwoBitMmask[iBit];
+}
+
+template <unsigned N>
+FixedHash<N> TopicBloomFilterBase<N>::bloom(AbridgedTopic const& _h)
+{
+	// The size of AbridgedTopic is 32 bits, and 27 of them participate in this algorithm.
+
+	// We need to review the algorithm if any of the following constants will be changed.
+	static_assert(4 == AbridgedTopic::size, "wrong template parameter in TopicBloomFilterBase<N>::bloom()");
+	static_assert(3 == BitsPerBloom, "wrong template parameter in TopicBloomFilterBase<N>::bloom()");
+
+	FixedHash<N> ret;
+
+	if (TopicBloomFilterSize == N)
+		for (unsigned i = 0; i < BitsPerBloom; ++i)
+		{
+			unsigned x = _h[i];
+			if (_h[BitsPerBloom] & c_powerOfTwoBitMmask[i])
+				x += 256;
+
+			setBit(ret, x);
+		}
+	else
+		for (unsigned i = 0; i < BitsPerBloom; ++i)
+		{
+			unsigned x = unsigned(_h[i]) + unsigned(_h[i + 1]);
+			x %= N * 8;
+			setBit(ret, x);
+		}
+
+	return ret;
+}
+
 using TopicBloomFilter = TopicBloomFilterBase<TopicBloomFilterSize>;
 
 }

libwhisper/Common.cpp

@@ -100,7 +100,7 @@ TopicBloomFilterHash TopicFilter::exportBloom() const
 	TopicBloomFilterHash ret;
 	for (TopicMask const& t: m_topicMasks)
 		for (auto const& i: t)
-			ret |= i.first.template bloomPart<BitsPerBloom, TopicBloomFilterSize>();
+			ret |= TopicBloomFilter::bloom(i.first);
 		
 	return ret;
 }

libwhisper/Message.cpp

@@ -20,6 +20,7 @@
  */
 
 #include "Message.h"
+#include "BloomFilter.h"
 
 using namespace std;
 using namespace dev;
@@ -161,31 +162,31 @@ unsigned Envelope::workProved() const
 
 void Envelope::proveWork(unsigned _ms)
 {
-	// PoW
 	h256 d[2];
 	d[0] = sha3(WithoutNonce);
-	uint32_t& n = *(uint32_t*)&(d[1][28]);
 	unsigned bestBitSet = 0;
 	bytesConstRef chuck(d[0].data(), 64);
 
 	chrono::high_resolution_clock::time_point then = chrono::high_resolution_clock::now() + chrono::milliseconds(_ms);
-	for (n = 0; chrono::high_resolution_clock::now() < then; )
+	while (chrono::high_resolution_clock::now() < then)
 		// do it rounds of 1024 for efficiency
-		for (unsigned i = 0; i < 1024; ++i, ++n)
+		for (unsigned i = 0; i < 1024; ++i)
 		{
 			auto fbs = dev::sha3(chuck).firstBitSet();
 			if (fbs > bestBitSet)
 			{
 				bestBitSet = fbs;
-				m_nonce = n;
+				m_nonce = (h256::Arith)d[1];
 			}
+
+			incrementHash(d[1]);
 		}
 }
 
 bool Envelope::matchesBloomFilter(TopicBloomFilterHash const& f) const
 {
 	for (AbridgedTopic t: m_topic)
-		if (f.contains(t.template bloomPart<BitsPerBloom, TopicBloomFilterSize>()))
+		if (f.contains(TopicBloomFilter::bloom(t)))
 			return true;
 
 	return false;

libwhisper/Message.h

@@ -81,6 +81,7 @@ public:
 	void proveWork(unsigned _ms);
 
 	bool matchesBloomFilter(TopicBloomFilterHash const& f) const;
+	static void incrementHash(h256& _h) { for (unsigned i = h256::size; i > 0 && !++_h[--i]; ) {} }
 
 private:
 	Envelope(unsigned _exp, unsigned _ttl, AbridgedTopics const& _topic): m_expiry(_exp), m_ttl(_ttl), m_topic(_topic) {}

libwhisper/WhisperHost.cpp

@@ -51,7 +51,8 @@ void WhisperHost::streamMessage(h256 _m, RLPStream& _s) const
 
 void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p)
 {
-	// this function processes messages originated both by local host (_p == null), and by remote peers (_p != null)
+	// this function processes both outgoing messages originated both by local host (_p == null)
+	// and incoming messages from remote peers (_p != null)
 
 	cnote << this << ": inject: " << _m.expiry() << _m.ttl() << _m.topic() << toHex(_m.data());
 
@@ -68,17 +69,32 @@ void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p)
 		m_expiryQueue.insert(make_pair(_m.expiry(), h));
 	}
 
-	int rating = 1; // rating for local host is based upon: 1. installed watch; 2. proof of work
+	// rating of incoming message from remote host is assessed according to the following criteria:
+	// 1. installed watch match; 2. bloom filter match; 2. ttl; 3. proof of work
+
+	int rating = 0;
 
-	if (_p) // incoming message from remote peer
 	DEV_GUARDED(m_filterLock)
+		if (_m.matchesBloomFilter(m_bloom))
+		{
+			++rating;
 			for (auto const& f: m_filters)
 				if (f.second.filter.matches(_m))
 					for (auto& i: m_watches)
 						if (i.second.id == f.first)
 						{
 							i.second.changes.push_back(h);
-							rating += 10; // subject to review
+							rating += 2;
+						}
+		}
+
+	if (_p) // incoming message from remote peer
+	{
+		rating *= 256;
+		unsigned ttlReward = (256 > _m.ttl() ? 256 - _m.ttl() : 0);
+		rating += ttlReward;
+		rating *= 256;
+		rating += _m.workProved();
 	}
 
 	// TODO p2p: capability-based rating

libwhisper/WhisperPeer.cpp

@@ -122,16 +122,20 @@ unsigned WhisperPeer::ratingForPeer(Envelope const& e) const
 	// we try to estimate, how valuable this nessage will be for the remote peer,
 	// according to the following criteria:
 	// 1. bloom filter
-	// 2. proof of work
+	// 2. time to live
+	// 3. proof of work
 
-	static const unsigned BloomFilterMatchReward = 256; // vlad todo: move to common.h
 	unsigned rating = 0;
 
-	DEV_GUARDED(x_bloom)
-		if (e.matchesBloomFilter(m_bloom))
-			rating += BloomFilterMatchReward;
+	if (e.matchesBloomFilter(bloom()))
+		++rating;
 
-	rating += e.sha3().firstBitSet();
+	rating *= 256;
+	unsigned ttlReward = (256 > e.ttl() ? 256 - e.ttl() : 0);
+	rating += ttlReward;
+
+	rating *= 256;
+	rating += e.workProved();
 	return rating;
 }
 

test/libwhisper/bloomFilter.cpp

@@ -244,7 +244,7 @@ BOOST_AUTO_TEST_CASE(bloomFilterRaw)
 	BOOST_REQUIRE(!f.contains(b00110111));
 }
 
-static const unsigned DistributionTestSize = 8;
+static const unsigned DistributionTestSize = TopicBloomFilterSize;
 static const unsigned TestArrSize = 8 * DistributionTestSize;
 
 void updateDistribution(FixedHash<DistributionTestSize> const& _h, array<unsigned, TestArrSize>& _distribution)
@@ -271,10 +271,10 @@ BOOST_AUTO_TEST_CASE(distributionRate)
 
 	Topic x(0xC0FFEE); // deterministic pseudorandom value
 
-	for (unsigned i = 0; i < 22000; ++i)
+	for (unsigned i = 0; i < 26000; ++i)
 	{
 		x = sha3(x);
-		FixedHash<DistributionTestSize> h = x.template bloomPart<BitsPerBloom, DistributionTestSize>();
+		FixedHash<DistributionTestSize> h = TopicBloomFilter::bloom(abridge(x));
 		updateDistribution(h, distribution);
 	}
 
@@ -283,16 +283,25 @@ BOOST_AUTO_TEST_CASE(distributionRate)
 		average += distribution[i];
 
 	average /= TestArrSize;
-	unsigned deviation = average / 10; // approx. 10%
+	unsigned deviation = average / 3;
 	unsigned maxAllowed = average + deviation;
 	unsigned minAllowed = average - deviation;
 
+	unsigned maximum = 0;
+	unsigned minimum = 0xFFFFFFFF;
+
 	for (unsigned i = 0; i < TestArrSize; ++i)
 	{
-		//cnote << i << ":" << distribution[i];
-		BOOST_REQUIRE(distribution[i] > minAllowed);
-		BOOST_REQUIRE(distribution[i] < maxAllowed);
+		unsigned const& z = distribution[i];
+		if (z > maximum)
+			maximum = z;
+		else if (z < minimum)
+			minimum = z;
 	}
+
+	cnote << minimum << average << maximum;
+	BOOST_REQUIRE(minimum > minAllowed);
+	BOOST_REQUIRE(maximum < maxAllowed);
 }
 
 BOOST_AUTO_TEST_SUITE_END()

test/libwhisper/whisperMessage.cpp

@@ -89,4 +89,24 @@ BOOST_AUTO_TEST_CASE(seal)
 		sealAndOpenSingleMessage(i);
 }
 
+BOOST_AUTO_TEST_CASE(work)
+{
+	VerbosityHolder setTemporaryLevel(10);
+	cnote << "Testing proof of work...";
+
+	Secret zero;
+	unsigned r = 0xC0DEFEED;
+
+	for (int i = 0; i < 20; ++i)
+	{
+		Topics topics = createRandomTopics(++r);
+		bytes const payload = createRandomPayload(++r);
+		Message m(payload);
+		Envelope e = m.seal(zero, topics, 1, 50);
+		unsigned x = e.workProved();
+		//cnote << x;
+		BOOST_REQUIRE(x > 4);
+	}
+}
+
 BOOST_AUTO_TEST_SUITE_END()

test/libwhisper/whisperTopic.cpp

@@ -78,7 +78,6 @@ BOOST_AUTO_TEST_CASE(topic)
 			}
 			this_thread::sleep_for(chrono::milliseconds(50));
 		}
-
 	});
 
 	Host host2("Test", NetworkPreferences("127.0.0.1", 30300, false));
@@ -384,4 +383,39 @@ BOOST_AUTO_TEST_CASE(topicAdvertising)
 	whost2->uninstallWatch(w2);
 }
 
+BOOST_AUTO_TEST_CASE(selfAddressed)
+{
+	VerbosityHolder setTemporaryLevel(10);
+	cnote << "Testing self-addressed messaging with bloom filter matching...";
+
+	char const* text = "deterministic pseudorandom test";
+	BuildTopicMask mask(text);
+
+	Host host("first", NetworkPreferences("127.0.0.1", 30305, false));
+	auto wh = host.registerCapability(new WhisperHost());
+	auto watch = wh->installWatch(BuildTopicMask(text));
+
+	unsigned const sample = 0xFEED;
+	KeyPair us = KeyPair::create();
+	wh->post(us.sec(), RLPStream().append(sample).out(), BuildTopic(text));
+
+	TopicBloomFilterHash f = wh->bloom();
+	Envelope e = Message(RLPStream().append(sample).out()).seal(us.sec(), BuildTopic(text), 50, 50);
+	bool ok = e.matchesBloomFilter(f);
+	BOOST_REQUIRE(ok);
+
+	this_thread::sleep_for(chrono::milliseconds(50));
+
+	unsigned single = 0;
+	unsigned result = 0;
+	for (auto j: wh->checkWatch(watch))
+	{
+		Message msg = wh->envelope(j).open(wh->fullTopics(watch));
+		single = RLP(msg.payload()).toInt<unsigned>();
+		result += single;
+	}
+
+	BOOST_REQUIRE_EQUAL(sample, result);
+}
+
 BOOST_AUTO_TEST_SUITE_END()