diff --git a/libwhisper/BloomFilter.h b/libwhisper/BloomFilter.h index 7b5d179c6..32ac15043 100644 --- a/libwhisper/BloomFilter.h +++ b/libwhisper/BloomFilter.h @@ -35,17 +35,20 @@ public: TopicBloomFilterBase() { init(); } TopicBloomFilterBase(FixedHash const& _h): FixedHash(_h) { init(); } - void addBloom(dev::shh::AbridgedTopic const& _h) { addRaw(_h.template bloomPart()); } - void removeBloom(dev::shh::AbridgedTopic const& _h) { removeRaw(_h.template bloomPart()); } - bool containsBloom(dev::shh::AbridgedTopic const& _h) const { return this->contains(_h.template bloomPart()); } + void addBloom(AbridgedTopic const& _h) { addRaw(bloom(_h)); } + void removeBloom(AbridgedTopic const& _h) { removeRaw(bloom(_h)); } + bool containsBloom(AbridgedTopic const& _h) const { return this->contains(bloom(_h)); } void addRaw(FixedHash const& _h); void removeRaw(FixedHash const& _h); bool containsRaw(FixedHash const& _h) const { return this->contains(_h); } + + static FixedHash bloom(AbridgedTopic const& _h); + static void setBit(FixedHash& _h, unsigned index); + static bool isBitSet(FixedHash const& _h, unsigned _index); private: void init() { for (unsigned i = 0; i < CounterSize; ++i) m_refCounter[i] = 0; } - static bool isBitSet(FixedHash const& _h, unsigned _index); static const unsigned CounterSize = N * 8; std::array m_refCounter; @@ -83,12 +86,51 @@ void TopicBloomFilterBase::removeRaw(FixedHash const& _h) template bool TopicBloomFilterBase::isBitSet(FixedHash const& _h, unsigned _index) -{ +{ unsigned iByte = _index / 8; - unsigned iBit = _index % 8; + unsigned iBit = _index & 0x7; return (_h[iByte] & c_powerOfTwoBitMmask[iBit]) != 0; } +template +void TopicBloomFilterBase::setBit(FixedHash& _h, unsigned _index) +{ + unsigned iByte = _index / 8; + unsigned iBit = _index & 0x7; + _h[iByte] |= c_powerOfTwoBitMmask[iBit]; +} + +template +FixedHash TopicBloomFilterBase::bloom(AbridgedTopic const& _h) +{ + // The size of AbridgedTopic is 32 bits, and 27 of them participate in this algorithm. + + // We need to review the algorithm if any of the following constants will be changed. + static_assert(4 == AbridgedTopic::size, "wrong template parameter in TopicBloomFilterBase::bloom()"); + static_assert(3 == BitsPerBloom, "wrong template parameter in TopicBloomFilterBase::bloom()"); + + FixedHash ret; + + if (TopicBloomFilterSize == N) + for (unsigned i = 0; i < BitsPerBloom; ++i) + { + unsigned x = _h[i]; + if (_h[BitsPerBloom] & c_powerOfTwoBitMmask[i]) + x += 256; + + setBit(ret, x); + } + else + for (unsigned i = 0; i < BitsPerBloom; ++i) + { + unsigned x = unsigned(_h[i]) + unsigned(_h[i + 1]); + x %= N * 8; + setBit(ret, x); + } + + return ret; +} + using TopicBloomFilter = TopicBloomFilterBase; } diff --git a/libwhisper/Common.cpp b/libwhisper/Common.cpp index 7c7a9801b..4681a21b3 100644 --- a/libwhisper/Common.cpp +++ b/libwhisper/Common.cpp @@ -100,7 +100,7 @@ TopicBloomFilterHash TopicFilter::exportBloom() const TopicBloomFilterHash ret; for (TopicMask const& t: m_topicMasks) for (auto const& i: t) - ret |= i.first.template bloomPart(); + ret |= TopicBloomFilter::bloom(i.first); return ret; } diff --git a/libwhisper/Message.cpp b/libwhisper/Message.cpp index 0b235b984..435811a3b 100644 --- a/libwhisper/Message.cpp +++ b/libwhisper/Message.cpp @@ -20,6 +20,7 @@ */ #include "Message.h" +#include "BloomFilter.h" using namespace std; using namespace dev; @@ -161,31 +162,31 @@ unsigned Envelope::workProved() const void Envelope::proveWork(unsigned _ms) { - // PoW h256 d[2]; d[0] = sha3(WithoutNonce); - uint32_t& n = *(uint32_t*)&(d[1][28]); unsigned bestBitSet = 0; bytesConstRef chuck(d[0].data(), 64); chrono::high_resolution_clock::time_point then = chrono::high_resolution_clock::now() + chrono::milliseconds(_ms); - for (n = 0; chrono::high_resolution_clock::now() < then; ) + while (chrono::high_resolution_clock::now() < then) // do it rounds of 1024 for efficiency - for (unsigned i = 0; i < 1024; ++i, ++n) + for (unsigned i = 0; i < 1024; ++i) { auto fbs = dev::sha3(chuck).firstBitSet(); if (fbs > bestBitSet) { bestBitSet = fbs; - m_nonce = n; + m_nonce = (h256::Arith)d[1]; } + + incrementHash(d[1]); } } bool Envelope::matchesBloomFilter(TopicBloomFilterHash const& f) const { for (AbridgedTopic t: m_topic) - if (f.contains(t.template bloomPart())) + if (f.contains(TopicBloomFilter::bloom(t))) return true; return false; diff --git a/libwhisper/Message.h b/libwhisper/Message.h index ae61f210a..fb89576e9 100644 --- a/libwhisper/Message.h +++ b/libwhisper/Message.h @@ -81,6 +81,7 @@ public: void proveWork(unsigned _ms); bool matchesBloomFilter(TopicBloomFilterHash const& f) const; + static void incrementHash(h256& _h) { for (unsigned i = h256::size; i > 0 && !++_h[--i]; ) {} } private: Envelope(unsigned _exp, unsigned _ttl, AbridgedTopics const& _topic): m_expiry(_exp), m_ttl(_ttl), m_topic(_topic) {} diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index f9ff33658..5d83dd399 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -51,7 +51,8 @@ void WhisperHost::streamMessage(h256 _m, RLPStream& _s) const void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p) { - // this function processes messages originated both by local host (_p == null), and by remote peers (_p != null) + // this function processes both outgoing messages originated both by local host (_p == null) + // and incoming messages from remote peers (_p != null) cnote << this << ": inject: " << _m.expiry() << _m.ttl() << _m.topic() << toHex(_m.data()); @@ -68,18 +69,33 @@ void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p) m_expiryQueue.insert(make_pair(_m.expiry(), h)); } - int rating = 1; // rating for local host is based upon: 1. installed watch; 2. proof of work + // rating of incoming message from remote host is assessed according to the following criteria: + // 1. installed watch match; 2. bloom filter match; 2. ttl; 3. proof of work - if (_p) // incoming message from remote peer - DEV_GUARDED(m_filterLock) + int rating = 0; + + DEV_GUARDED(m_filterLock) + if (_m.matchesBloomFilter(m_bloom)) + { + ++rating; for (auto const& f: m_filters) if (f.second.filter.matches(_m)) for (auto& i: m_watches) if (i.second.id == f.first) { i.second.changes.push_back(h); - rating += 10; // subject to review + rating += 2; } + } + + if (_p) // incoming message from remote peer + { + rating *= 256; + unsigned ttlReward = (256 > _m.ttl() ? 256 - _m.ttl() : 0); + rating += ttlReward; + rating *= 256; + rating += _m.workProved(); + } // TODO p2p: capability-based rating for (auto i: peerSessions()) diff --git a/libwhisper/WhisperPeer.cpp b/libwhisper/WhisperPeer.cpp index 393c3749c..30a188f2d 100644 --- a/libwhisper/WhisperPeer.cpp +++ b/libwhisper/WhisperPeer.cpp @@ -121,17 +121,21 @@ unsigned WhisperPeer::ratingForPeer(Envelope const& e) const { // we try to estimate, how valuable this nessage will be for the remote peer, // according to the following criteria: - // 1. bloom filter - // 2. proof of work + // 1. bloom filter + // 2. time to live + // 3. proof of work - static const unsigned BloomFilterMatchReward = 256; // vlad todo: move to common.h unsigned rating = 0; - DEV_GUARDED(x_bloom) - if (e.matchesBloomFilter(m_bloom)) - rating += BloomFilterMatchReward; - - rating += e.sha3().firstBitSet(); + if (e.matchesBloomFilter(bloom())) + ++rating; + + rating *= 256; + unsigned ttlReward = (256 > e.ttl() ? 256 - e.ttl() : 0); + rating += ttlReward; + + rating *= 256; + rating += e.workProved(); return rating; } diff --git a/test/libwhisper/bloomFilter.cpp b/test/libwhisper/bloomFilter.cpp index 197a7258d..d7d4bd849 100644 --- a/test/libwhisper/bloomFilter.cpp +++ b/test/libwhisper/bloomFilter.cpp @@ -244,7 +244,7 @@ BOOST_AUTO_TEST_CASE(bloomFilterRaw) BOOST_REQUIRE(!f.contains(b00110111)); } -static const unsigned DistributionTestSize = 8; +static const unsigned DistributionTestSize = TopicBloomFilterSize; static const unsigned TestArrSize = 8 * DistributionTestSize; void updateDistribution(FixedHash const& _h, array& _distribution) @@ -271,10 +271,10 @@ BOOST_AUTO_TEST_CASE(distributionRate) Topic x(0xC0FFEE); // deterministic pseudorandom value - for (unsigned i = 0; i < 22000; ++i) + for (unsigned i = 0; i < 26000; ++i) { x = sha3(x); - FixedHash h = x.template bloomPart(); + FixedHash h = TopicBloomFilter::bloom(abridge(x)); updateDistribution(h, distribution); } @@ -283,16 +283,25 @@ BOOST_AUTO_TEST_CASE(distributionRate) average += distribution[i]; average /= TestArrSize; - unsigned deviation = average / 10; // approx. 10% + unsigned deviation = average / 3; unsigned maxAllowed = average + deviation; unsigned minAllowed = average - deviation; + unsigned maximum = 0; + unsigned minimum = 0xFFFFFFFF; + for (unsigned i = 0; i < TestArrSize; ++i) { - //cnote << i << ":" << distribution[i]; - BOOST_REQUIRE(distribution[i] > minAllowed); - BOOST_REQUIRE(distribution[i] < maxAllowed); + unsigned const& z = distribution[i]; + if (z > maximum) + maximum = z; + else if (z < minimum) + minimum = z; } + + cnote << minimum << average << maximum; + BOOST_REQUIRE(minimum > minAllowed); + BOOST_REQUIRE(maximum < maxAllowed); } BOOST_AUTO_TEST_SUITE_END() diff --git a/test/libwhisper/whisperMessage.cpp b/test/libwhisper/whisperMessage.cpp index 343573713..f0c0aedc9 100644 --- a/test/libwhisper/whisperMessage.cpp +++ b/test/libwhisper/whisperMessage.cpp @@ -89,4 +89,24 @@ BOOST_AUTO_TEST_CASE(seal) sealAndOpenSingleMessage(i); } +BOOST_AUTO_TEST_CASE(work) +{ + VerbosityHolder setTemporaryLevel(10); + cnote << "Testing proof of work..."; + + Secret zero; + unsigned r = 0xC0DEFEED; + + for (int i = 0; i < 20; ++i) + { + Topics topics = createRandomTopics(++r); + bytes const payload = createRandomPayload(++r); + Message m(payload); + Envelope e = m.seal(zero, topics, 1, 50); + unsigned x = e.workProved(); + //cnote << x; + BOOST_REQUIRE(x > 4); + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/test/libwhisper/whisperTopic.cpp b/test/libwhisper/whisperTopic.cpp index 82825c5d0..8d0f603bb 100644 --- a/test/libwhisper/whisperTopic.cpp +++ b/test/libwhisper/whisperTopic.cpp @@ -78,7 +78,6 @@ BOOST_AUTO_TEST_CASE(topic) } this_thread::sleep_for(chrono::milliseconds(50)); } - }); Host host2("Test", NetworkPreferences("127.0.0.1", 30300, false)); @@ -384,4 +383,39 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) whost2->uninstallWatch(w2); } +BOOST_AUTO_TEST_CASE(selfAddressed) +{ + VerbosityHolder setTemporaryLevel(10); + cnote << "Testing self-addressed messaging with bloom filter matching..."; + + char const* text = "deterministic pseudorandom test"; + BuildTopicMask mask(text); + + Host host("first", NetworkPreferences("127.0.0.1", 30305, false)); + auto wh = host.registerCapability(new WhisperHost()); + auto watch = wh->installWatch(BuildTopicMask(text)); + + unsigned const sample = 0xFEED; + KeyPair us = KeyPair::create(); + wh->post(us.sec(), RLPStream().append(sample).out(), BuildTopic(text)); + + TopicBloomFilterHash f = wh->bloom(); + Envelope e = Message(RLPStream().append(sample).out()).seal(us.sec(), BuildTopic(text), 50, 50); + bool ok = e.matchesBloomFilter(f); + BOOST_REQUIRE(ok); + + this_thread::sleep_for(chrono::milliseconds(50)); + + unsigned single = 0; + unsigned result = 0; + for (auto j: wh->checkWatch(watch)) + { + Message msg = wh->envelope(j).open(wh->fullTopics(watch)); + single = RLP(msg.payload()).toInt(); + result += single; + } + + BOOST_REQUIRE_EQUAL(sample, result); +} + BOOST_AUTO_TEST_SUITE_END()