From 86b16df9c2c7b62bf0c3b87d432ff87470fc6bb6 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Wed, 22 Jul 2015 01:34:45 +0200 Subject: [PATCH 01/10] first version --- libwhisper/Interface.h | 4 +- libwhisper/WhisperDB.cpp | 19 ++++++- libwhisper/WhisperDB.h | 10 +++- libwhisper/WhisperHost.cpp | 67 +++++++++++++++++++++++- libwhisper/WhisperHost.h | 2 + test/libwhisper/whisperDB.cpp | 96 ++++++++++++++++++++++++++++++++--- 6 files changed, 183 insertions(+), 15 deletions(-) diff --git a/libwhisper/Interface.h b/libwhisper/Interface.h index f53cb17a7..29101a35e 100644 --- a/libwhisper/Interface.h +++ b/libwhisper/Interface.h @@ -64,15 +64,15 @@ public: virtual ~Interface(); virtual void inject(Envelope const& _m, WhisperPeer* _from = nullptr) = 0; - virtual Topics const& fullTopics(unsigned _id) const = 0; virtual unsigned installWatch(Topics const& _filter) = 0; virtual void uninstallWatch(unsigned _watchId) = 0; virtual h256s peekWatch(unsigned _watchId) const = 0; virtual h256s checkWatch(unsigned _watchId) = 0; virtual h256s watchMessages(unsigned _watchId) = 0; - virtual Envelope envelope(h256 _m) const = 0; + virtual void saveTopicsToDB(std::string const& _app, std::string const& _password) = 0; + virtual std::vector restoreTopicsFromDB(std::string const& _app, std::string const& _password) = 0; void post(bytes const& _payload, Topics _topics, unsigned _ttl = 50, unsigned _workToProve = 50) { inject(Message(_payload).seal(_topics, _ttl, _workToProve)); } void post(Public _to, bytes const& _payload, Topics _topics, unsigned _ttl = 50, unsigned _workToProve = 50) { inject(Message(_payload).sealTo(_to, _topics, _ttl, _workToProve)); } diff --git a/libwhisper/WhisperDB.cpp b/libwhisper/WhisperDB.cpp index 84c24d5a8..5ab04d9da 100644 --- a/libwhisper/WhisperDB.cpp +++ b/libwhisper/WhisperDB.cpp @@ -27,7 +27,7 @@ using namespace dev; using namespace dev::shh; namespace fs = boost::filesystem; -WhisperDB::WhisperDB() +WhisperDB::WhisperDB(DBType _t): m_type(_t) { m_readOptions.verify_checksums = true; string path = dev::getDataDir("shh"); @@ -36,13 +36,25 @@ WhisperDB::WhisperDB() leveldb::Options op; op.create_if_missing = true; op.max_open_files = 256; + string suffix = getTypeSuffix(); leveldb::DB* p = nullptr; - leveldb::Status status = leveldb::DB::Open(op, path + "/messages", &p); + leveldb::Status status = leveldb::DB::Open(op, path + suffix, &p); m_db.reset(p); if (!status.ok()) BOOST_THROW_EXCEPTION(FailedToOpenLevelDB(status.ToString())); } +string WhisperDB::getTypeSuffix() +{ + switch(m_type) + { + case Messages: return "\\messages"; + case Filters: return "\\filters"; + } + + return "\\misc"; +} + string WhisperDB::lookup(dev::h256 const& _key) const { string ret; @@ -137,6 +149,9 @@ void WhisperDB::loadAll(std::map& o_dst) void WhisperDB::save(h256 const& _key, Envelope const& _e) { + if (m_type != Messages) + BOOST_THROW_EXCEPTION(WrongTypeLevelDB()); + try { RLPStream rlp; diff --git a/libwhisper/WhisperDB.h b/libwhisper/WhisperDB.h index f56ae1005..03c6038b5 100644 --- a/libwhisper/WhisperDB.h +++ b/libwhisper/WhisperDB.h @@ -31,6 +31,7 @@ namespace dev namespace shh { +struct WrongTypeLevelDB: virtual Exception {}; struct FailedToOpenLevelDB: virtual Exception { FailedToOpenLevelDB(std::string const& _message): Exception(_message) {} }; struct FailedInsertInLevelDB: virtual Exception { FailedInsertInLevelDB(std::string const& _message): Exception(_message) {} }; struct FailedLookupInLevelDB: virtual Exception { FailedLookupInLevelDB(std::string const& _message): Exception(_message) {} }; @@ -38,8 +39,10 @@ struct FailedDeleteInLevelDB: virtual Exception { FailedDeleteInLevelDB(std::str class WhisperDB { - public: - WhisperDB(); +public: + enum DBType { Messages, Filters }; + + WhisperDB(DBType _t); ~WhisperDB() {} std::string lookup(dev::h256 const& _key) const; @@ -50,9 +53,12 @@ class WhisperDB void save(dev::h256 const& _key, Envelope const& _e); private: + std::string getTypeSuffix(); + leveldb::ReadOptions m_readOptions; leveldb::WriteOptions m_writeOptions; std::unique_ptr m_db; + DBType m_type; enum MetaInformation { StoreForeverFlag = 1, WatchedFlag = 2 }; }; diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index 1f5d28bcc..8506607e5 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -223,7 +223,7 @@ void WhisperHost::saveMessagesToBD() try { - WhisperDB db; + WhisperDB db(WhisperDB::Messages); ReadGuard g(x_messages); unsigned now = (unsigned)time(0); for (auto const& m: m_messages) @@ -253,7 +253,7 @@ void WhisperHost::loadMessagesFromBD() try { map m; - WhisperDB db; + WhisperDB db(WhisperDB::Messages); db.loadAll(m); WriteGuard g(x_messages); m_messages.swap(m); @@ -269,3 +269,66 @@ void WhisperHost::loadMessagesFromBD() cwarn << "Unknown Exception in WhisperHost::loadMessagesFromBD()"; } } + +void WhisperHost::saveTopicsToDB(string const& _app, string const& _password) +{ + bytes plain; + + DEV_GUARDED(m_filterLock) + { + RLPStream rlp(m_filters.size()); + + for (auto const& x: m_filters) + { + Topics const& topics = x.second.full; + unsigned const RawDataSize = topics.size() * h256::size; + unique_ptr p(new byte[RawDataSize]); + unsigned i = 0; + + for (auto const& t: topics) + memcpy(p.get() + i * h256::size, t.data(), h256::size); + + bytesConstRef ref(p.get(), RawDataSize); + rlp.append(ref); + } + + rlp.swapOut(plain); + } + + // todo: encrypt after tests + + h256 h = sha3(_app); + WhisperDB db(WhisperDB::Messages); + db.insert(h, plain); +} + +vector WhisperHost::restoreTopicsFromDB(string const& _app, string const& _password) +{ + vector ret; + h256 h = sha3(_app); + WhisperDB db(WhisperDB::Messages); + string raw = db.lookup(h); + + // todo: decrypt after tests + + RLP rlp(raw); + auto sz = rlp.itemCountStrict(); + + for (unsigned i = 0; i < sz; ++i) + { + RLP r = rlp[i]; + bytesConstRef ref(r.data()); + Topics topics; + unsigned num = ref.size() / h256::size; + for (unsigned j = 0; j < num; ++j) + { + h256 topic(ref.data() + j * h256::size, h256::ConstructFromPointerType()); + topics.push_back(topic); + } + + unsigned w = installWatch(topics); + ret.push_back(w); + } + + return ret; +} diff --git a/libwhisper/WhisperHost.h b/libwhisper/WhisperHost.h index d9a71b6b2..ccba220f5 100644 --- a/libwhisper/WhisperHost.h +++ b/libwhisper/WhisperHost.h @@ -63,6 +63,8 @@ public: virtual h256s checkWatch(unsigned _watchId) override { cleanup(); dev::Guard l(m_filterLock); h256s ret; try { ret = m_watches.at(_watchId).changes; m_watches.at(_watchId).changes.clear(); } catch (...) {} return ret; } virtual h256s watchMessages(unsigned _watchId) override; ///< returns IDs of messages, which match specific watch criteria virtual Envelope envelope(h256 _m) const override { try { dev::ReadGuard l(x_messages); return m_messages.at(_m); } catch (...) { return Envelope(); } } + virtual void saveTopicsToDB(std::string const& _app, std::string const& _password); + virtual std::vector restoreTopicsFromDB(std::string const& _app, std::string const& _password); protected: virtual void doWork() override; diff --git a/test/libwhisper/whisperDB.cpp b/test/libwhisper/whisperDB.cpp index 6b96410a0..2531d54a8 100644 --- a/test/libwhisper/whisperDB.cpp +++ b/test/libwhisper/whisperDB.cpp @@ -28,6 +28,7 @@ along with cpp-ethereum. If not, see . using namespace std; using namespace dev; using namespace dev::shh; +using namespace dev::p2p; struct P2PFixture { @@ -47,7 +48,7 @@ BOOST_AUTO_TEST_CASE(basic) string const text2 = "ipsum"; h256 h1(0xBEEF); h256 h2(0xC0FFEE); - WhisperDB db; + WhisperDB db(WhisperDB::Messages); db.kill(h1); db.kill(h2); @@ -94,7 +95,7 @@ BOOST_AUTO_TEST_CASE(persistence) h256 const h2(0xBADD00DE); { - WhisperDB db; + WhisperDB db(WhisperDB::Messages); db.kill(h1); db.kill(h2); s = db.lookup(h1); @@ -109,7 +110,7 @@ BOOST_AUTO_TEST_CASE(persistence) this_thread::sleep_for(chrono::milliseconds(20)); { - WhisperDB db; + WhisperDB db(WhisperDB::Messages); db.insert(h1, text1); db.insert(h2, text2); } @@ -117,7 +118,7 @@ BOOST_AUTO_TEST_CASE(persistence) this_thread::sleep_for(chrono::milliseconds(20)); { - WhisperDB db; + WhisperDB db(WhisperDB::Messages); s = db.lookup(h2); BOOST_REQUIRE(!s.compare(text2)); s = db.lookup(h1); @@ -169,7 +170,7 @@ BOOST_AUTO_TEST_CASE(messages) } } - WhisperDB db; + WhisperDB db(WhisperDB::Messages); unsigned x = 0; for (auto i: m1) @@ -190,7 +191,7 @@ BOOST_AUTO_TEST_CASE(corruptedData) h256 x = h256::random(); { - WhisperDB db; + WhisperDB db(WhisperDB::Messages); db.insert(x, "this is a test input, representing corrupt data"); } @@ -202,10 +203,91 @@ BOOST_AUTO_TEST_CASE(corruptedData) } { - WhisperDB db; + WhisperDB db(WhisperDB::Messages); string s = db.lookup(x); BOOST_REQUIRE(s.empty()); } } +BOOST_AUTO_TEST_CASE(filters) +{ + cnote << "Testing filters saving..."; + VerbosityHolder setTemporaryLevel(2); + string const app("test suite whisperDB/filters"); + string const password("some pseudorandom stuff"); + + { + p2p::Host h("Test"); + auto wh = h.registerCapability(new WhisperHost()); + wh->installWatch(BuildTopic("t1")); + wh->installWatch(BuildTopic("t2")); + wh->saveTopicsToDB(app, password); + } + + short unsigned port1 = 30313; + Host host1("Test", NetworkPreferences("127.0.0.1", port1, false)); + host1.setIdealPeerCount(1); + auto whost1 = host1.registerCapability(new WhisperHost()); + host1.start(); + auto ids = whost1->restoreTopicsFromDB(app, password); + bool host1Ready = false; + unsigned result = 0; + unsigned messageCount = 0; + + std::thread listener([&]() + { + setThreadName("other"); + host1Ready = true; + + for (int j = 0; j < 200 && messageCount < 2; ++j) + { + for (unsigned id: ids) + for (auto e: whost1->checkWatch(id)) + { + Message msg = whost1->envelope(e).open(whost1->fullTopics(id)); + unsigned x = RLP(msg.payload()).toInt(); + cnote << "New message:" << x; + result += x; + ++messageCount; + } + + this_thread::sleep_for(chrono::milliseconds(50)); + } + }); + + Host host2("Test", NetworkPreferences("127.0.0.1", 30314, false)); + host2.setIdealPeerCount(1); + auto whost2 = host2.registerCapability(new WhisperHost()); + host2.start(); + + while (!host1.haveNetwork()) + this_thread::sleep_for(chrono::milliseconds(5)); + + host2.requirePeer(host1.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), port1, port1)); + + while (!host1Ready) + this_thread::sleep_for(chrono::milliseconds(10)); + + while (!host1.peerCount() && !host2.peerCount()) + this_thread::sleep_for(chrono::milliseconds(10)); + + unsigned ttl = 1000000; + whost2->post(RLPStream().append(8).out(), BuildTopic("t8"), ttl); + this_thread::sleep_for(chrono::milliseconds(10)); + whost2->post(RLPStream().append(4).out(), BuildTopic("t4"), ttl); + this_thread::sleep_for(chrono::milliseconds(10)); + whost2->post(RLPStream().append(1).out(), BuildTopic("t1"), ttl); + this_thread::sleep_for(chrono::milliseconds(10)); + whost2->post(RLPStream().append(2).out(), BuildTopic("t2"), ttl); + this_thread::sleep_for(chrono::milliseconds(10)); + whost2->post(RLPStream().append(16).out(), BuildTopic("t16"), ttl); + + while (messageCount < 2) + this_thread::sleep_for(chrono::milliseconds(10)); + + listener.join(); + BOOST_REQUIRE_EQUAL(messageCount, 2); + BOOST_REQUIRE_EQUAL(result, 3); +} + BOOST_AUTO_TEST_SUITE_END() From ff4cbd8dabdec28d95c1c1fc5fb3fc6c883bede0 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Wed, 22 Jul 2015 13:44:41 +0200 Subject: [PATCH 02/10] bugfix: vector::iterator --- libp2p/Common.h | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/libp2p/Common.h b/libp2p/Common.h index 47fcbd126..14edb817f 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -242,8 +242,26 @@ public: void stop() { m_stopped = true; DEV_GUARDED(x_timers) m_timers.clear(); } protected: - void reap(); - + void reap() + { + Guard l(x_timers); + std::vector::iterator t = m_timers.begin(); + while (t != m_timers.end()) + if (t->expired()) + { + t->wait(); + t = m_timers.erase(t); + } + else + t++; + + m_timers.emplace_back(m_io, m_reapIntervalMs, [this](boost::system::error_code const& ec) + { + if (!ec) + reap(); + }); + } + private: ba::io_service& m_io; unsigned m_reapIntervalMs; From 0d4ffff47ee12f944e6e8abf0449b397bb2fbe67 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Wed, 22 Jul 2015 15:22:14 +0200 Subject: [PATCH 03/10] bugfix: reap() --- libp2p/Common.h | 1 + 1 file changed, 1 insertion(+) diff --git a/libp2p/Common.h b/libp2p/Common.h index 14edb817f..4fd9ac189 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -300,3 +300,4 @@ template <> struct hash }; } + From ddfd80247fb5c98f7bc757e05d50b22c58f855ed Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Wed, 22 Jul 2015 16:57:09 +0200 Subject: [PATCH 04/10] saving & restoring topics --- libwhisper/WhisperDB.cpp | 2 +- libwhisper/WhisperDB.h | 2 +- libwhisper/WhisperHost.cpp | 62 ++++++++++++++++++++++------------- libwhisper/WhisperHost.h | 2 +- test/libwhisper/whisperDB.cpp | 39 ++++++++++++---------- 5 files changed, 64 insertions(+), 43 deletions(-) diff --git a/libwhisper/WhisperDB.cpp b/libwhisper/WhisperDB.cpp index 5ab04d9da..829ac4d8a 100644 --- a/libwhisper/WhisperDB.cpp +++ b/libwhisper/WhisperDB.cpp @@ -91,7 +91,7 @@ void WhisperDB::kill(dev::h256 const& _key) BOOST_THROW_EXCEPTION(FailedDeleteInLevelDB(status.ToString())); } -void WhisperDB::loadAll(std::map& o_dst) +void WhisperDB::loadAllMessages(std::map& o_dst) { leveldb::ReadOptions op; op.fill_cache = false; diff --git a/libwhisper/WhisperDB.h b/libwhisper/WhisperDB.h index 03c6038b5..3c7fc4375 100644 --- a/libwhisper/WhisperDB.h +++ b/libwhisper/WhisperDB.h @@ -49,7 +49,7 @@ public: void insert(dev::h256 const& _key, std::string const& _value); void insert(dev::h256 const& _key, bytes const& _value); void kill(dev::h256 const& _key); - void loadAll(std::map& o_dst); + void loadAllMessages(std::map& o_dst); void save(dev::h256 const& _key, Envelope const& _e); private: diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index 8506607e5..fbefe7ce9 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -135,6 +135,31 @@ unsigned WhisperHost::installWatch(shh::Topics const& _t) return ret; } +void WhisperHost::uninstallWatch(unsigned _i) +{ + cwatshh << "XXX" << _i; + + DEV_GUARDED(m_filterLock) + { + auto it = m_watches.find(_i); + if (it == m_watches.end()) + return; + + auto id = it->second.id; + m_watches.erase(it); + + auto fit = m_filters.find(id); + if (fit != m_filters.end()) + { + m_bloom.removeRaw(fit->second.filter.exportBloom()); + if (!--fit->second.refCount) + m_filters.erase(fit); + } + } + + noteAdvertiseTopicsOfInterest(); +} + h256s WhisperHost::watchMessages(unsigned _watchId) { h256s ret; @@ -156,29 +181,22 @@ h256s WhisperHost::watchMessages(unsigned _watchId) return ret; } -void WhisperHost::uninstallWatch(unsigned _i) +h256s WhisperHost::checkWatch(unsigned _watchId) { - cwatshh << "XXX" << _i; + h256s ret; + cleanup(); - DEV_GUARDED(m_filterLock) + dev::Guard l(m_filterLock); + try + { + ret = m_watches.at(_watchId).changes; + m_watches.at(_watchId).changes.clear(); + } + catch (...) { - auto it = m_watches.find(_i); - if (it == m_watches.end()) - return; - - auto id = it->second.id; - m_watches.erase(it); - - auto fit = m_filters.find(id); - if (fit != m_filters.end()) - { - m_bloom.removeRaw(fit->second.filter.exportBloom()); - if (!--fit->second.refCount) - m_filters.erase(fit); - } } - noteAdvertiseTopicsOfInterest(); + return ret; } void WhisperHost::doWork() @@ -254,7 +272,7 @@ void WhisperHost::loadMessagesFromBD() { map m; WhisperDB db(WhisperDB::Messages); - db.loadAll(m); + db.loadAllMessages(m); WriteGuard g(x_messages); m_messages.swap(m); for (auto const& msg: m) @@ -298,7 +316,7 @@ void WhisperHost::saveTopicsToDB(string const& _app, string const& _password) // todo: encrypt after tests h256 h = sha3(_app); - WhisperDB db(WhisperDB::Messages); + WhisperDB db(WhisperDB::Filters); db.insert(h, plain); } @@ -306,7 +324,7 @@ vector WhisperHost::restoreTopicsFromDB(string const& _app, string con { vector ret; h256 h = sha3(_app); - WhisperDB db(WhisperDB::Messages); + WhisperDB db(WhisperDB::Filters); string raw = db.lookup(h); // todo: decrypt after tests @@ -317,7 +335,7 @@ vector WhisperHost::restoreTopicsFromDB(string const& _app, string con for (unsigned i = 0; i < sz; ++i) { RLP r = rlp[i]; - bytesConstRef ref(r.data()); + bytesConstRef ref(r.toBytesConstRef()); Topics topics; unsigned num = ref.size() / h256::size; for (unsigned j = 0; j < num; ++j) diff --git a/libwhisper/WhisperHost.h b/libwhisper/WhisperHost.h index ccba220f5..ab49b8392 100644 --- a/libwhisper/WhisperHost.h +++ b/libwhisper/WhisperHost.h @@ -60,7 +60,7 @@ public: virtual unsigned installWatch(Topics const& _filter) override; virtual void uninstallWatch(unsigned _watchId) override; virtual h256s peekWatch(unsigned _watchId) const override { dev::Guard l(m_filterLock); try { return m_watches.at(_watchId).changes; } catch (...) { return h256s(); } } - virtual h256s checkWatch(unsigned _watchId) override { cleanup(); dev::Guard l(m_filterLock); h256s ret; try { ret = m_watches.at(_watchId).changes; m_watches.at(_watchId).changes.clear(); } catch (...) {} return ret; } + virtual h256s checkWatch(unsigned _watchId) override; virtual h256s watchMessages(unsigned _watchId) override; ///< returns IDs of messages, which match specific watch criteria virtual Envelope envelope(h256 _m) const override { try { dev::ReadGuard l(x_messages); return m_messages.at(_m); } catch (...) { return Envelope(); } } virtual void saveTopicsToDB(std::string const& _app, std::string const& _password); diff --git a/test/libwhisper/whisperDB.cpp b/test/libwhisper/whisperDB.cpp index 2531d54a8..902b9cf98 100644 --- a/test/libwhisper/whisperDB.cpp +++ b/test/libwhisper/whisperDB.cpp @@ -225,24 +225,30 @@ BOOST_AUTO_TEST_CASE(filters) } short unsigned port1 = 30313; + unsigned const step = 10; + bool host1Ready = false; + bool sent = false; + unsigned result = 0; + unsigned messageCount = 0; + Host host1("Test", NetworkPreferences("127.0.0.1", port1, false)); host1.setIdealPeerCount(1); auto whost1 = host1.registerCapability(new WhisperHost()); host1.start(); auto ids = whost1->restoreTopicsFromDB(app, password); - bool host1Ready = false; - unsigned result = 0; - unsigned messageCount = 0; std::thread listener([&]() { setThreadName("other"); host1Ready = true; - for (int j = 0; j < 200 && messageCount < 2; ++j) + for (unsigned i = 0; i < 16000 && !sent; i += step) + this_thread::sleep_for(chrono::milliseconds(step)); + + for (unsigned j = 0; j < 200 && messageCount < 2; ++j) { for (unsigned id: ids) - for (auto e: whost1->checkWatch(id)) + for (auto const& e: whost1->checkWatch(id)) { Message msg = whost1->envelope(e).open(whost1->fullTopics(id)); unsigned x = RLP(msg.payload()).toInt(); @@ -260,30 +266,27 @@ BOOST_AUTO_TEST_CASE(filters) auto whost2 = host2.registerCapability(new WhisperHost()); host2.start(); - while (!host1.haveNetwork()) - this_thread::sleep_for(chrono::milliseconds(5)); + for (unsigned i = 0; i < 3000 && !host1.haveNetwork(); i += step) + this_thread::sleep_for(chrono::milliseconds(step)); host2.requirePeer(host1.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), port1, port1)); - while (!host1Ready) - this_thread::sleep_for(chrono::milliseconds(10)); + for (unsigned i = 0; i < 3000 && !host1Ready; i += step) + this_thread::sleep_for(chrono::milliseconds(step)); - while (!host1.peerCount() && !host2.peerCount()) - this_thread::sleep_for(chrono::milliseconds(10)); + for (unsigned i = 0; i < 3000 && (!host1.peerCount() || !host2.peerCount()); i += step) + this_thread::sleep_for(chrono::milliseconds(step)); - unsigned ttl = 1000000; + unsigned ttl = 777000; whost2->post(RLPStream().append(8).out(), BuildTopic("t8"), ttl); - this_thread::sleep_for(chrono::milliseconds(10)); whost2->post(RLPStream().append(4).out(), BuildTopic("t4"), ttl); - this_thread::sleep_for(chrono::milliseconds(10)); whost2->post(RLPStream().append(1).out(), BuildTopic("t1"), ttl); - this_thread::sleep_for(chrono::milliseconds(10)); whost2->post(RLPStream().append(2).out(), BuildTopic("t2"), ttl); - this_thread::sleep_for(chrono::milliseconds(10)); whost2->post(RLPStream().append(16).out(), BuildTopic("t16"), ttl); + sent = true; - while (messageCount < 2) - this_thread::sleep_for(chrono::milliseconds(10)); + for (unsigned i = 0; i < 3000 && messageCount < 2; i += step) + this_thread::sleep_for(chrono::milliseconds(step)); listener.join(); BOOST_REQUIRE_EQUAL(messageCount, 2); From 29124eafd91135e96a751988cecba34eba00a2e0 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Thu, 23 Jul 2015 00:48:57 +0200 Subject: [PATCH 05/10] encryption added --- libwhisper/WhisperDB.cpp | 3 +++ libwhisper/WhisperHost.cpp | 12 ++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/libwhisper/WhisperDB.cpp b/libwhisper/WhisperDB.cpp index 829ac4d8a..a31e2a6eb 100644 --- a/libwhisper/WhisperDB.cpp +++ b/libwhisper/WhisperDB.cpp @@ -93,6 +93,9 @@ void WhisperDB::kill(dev::h256 const& _key) void WhisperDB::loadAllMessages(std::map& o_dst) { + if (m_type != Messages) + BOOST_THROW_EXCEPTION(WrongTypeLevelDB()); + leveldb::ReadOptions op; op.fill_cache = false; op.verify_checksums = true; diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index fbefe7ce9..856d7be7c 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -313,11 +313,13 @@ void WhisperHost::saveTopicsToDB(string const& _app, string const& _password) rlp.swapOut(plain); } - // todo: encrypt after tests + bytes encrypted; + h256 s = sha3(_password); + encryptSym(s, &plain, encrypted); h256 h = sha3(_app); WhisperDB db(WhisperDB::Filters); - db.insert(h, plain); + db.insert(h, encrypted); } vector WhisperHost::restoreTopicsFromDB(string const& _app, string const& _password) @@ -327,9 +329,11 @@ vector WhisperHost::restoreTopicsFromDB(string const& _app, string con WhisperDB db(WhisperDB::Filters); string raw = db.lookup(h); - // todo: decrypt after tests + bytes plain; + h256 s = sha3(_password); + decryptSym(s, raw, plain); - RLP rlp(raw); + RLP rlp(plain); auto sz = rlp.itemCountStrict(); for (unsigned i = 0; i < sz; ++i) From 2764ca4dbb3a8477b1598866d0a950570e03a68c Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Fri, 24 Jul 2015 01:32:31 +0200 Subject: [PATCH 06/10] WhisperDB class split --- libp2p/Common.h | 2 +- libwhisper/Interface.h | 4 +-- libwhisper/WhisperDB.cpp | 63 +++++++++++++++++++++++------------ libwhisper/WhisperDB.h | 33 +++++++++++------- libwhisper/WhisperHost.cpp | 45 ++++--------------------- libwhisper/WhisperHost.h | 1 - test/libwhisper/whisperDB.cpp | 20 ++++++----- 7 files changed, 82 insertions(+), 86 deletions(-) diff --git a/libp2p/Common.h b/libp2p/Common.h index 4fd9ac189..4c7fa837b 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -239,6 +239,7 @@ public: ~DeadlineOps() { stop(); } void schedule(unsigned _msInFuture, std::function const& _f) { if (m_stopped) return; DEV_GUARDED(x_timers) m_timers.emplace_back(m_io, _msInFuture, _f); } + void stop() { m_stopped = true; DEV_GUARDED(x_timers) m_timers.clear(); } protected: @@ -300,4 +301,3 @@ template <> struct hash }; } - diff --git a/libwhisper/Interface.h b/libwhisper/Interface.h index 29101a35e..f53cb17a7 100644 --- a/libwhisper/Interface.h +++ b/libwhisper/Interface.h @@ -64,15 +64,15 @@ public: virtual ~Interface(); virtual void inject(Envelope const& _m, WhisperPeer* _from = nullptr) = 0; + virtual Topics const& fullTopics(unsigned _id) const = 0; virtual unsigned installWatch(Topics const& _filter) = 0; virtual void uninstallWatch(unsigned _watchId) = 0; virtual h256s peekWatch(unsigned _watchId) const = 0; virtual h256s checkWatch(unsigned _watchId) = 0; virtual h256s watchMessages(unsigned _watchId) = 0; + virtual Envelope envelope(h256 _m) const = 0; - virtual void saveTopicsToDB(std::string const& _app, std::string const& _password) = 0; - virtual std::vector restoreTopicsFromDB(std::string const& _app, std::string const& _password) = 0; void post(bytes const& _payload, Topics _topics, unsigned _ttl = 50, unsigned _workToProve = 50) { inject(Message(_payload).seal(_topics, _ttl, _workToProve)); } void post(Public _to, bytes const& _payload, Topics _topics, unsigned _ttl = 50, unsigned _workToProve = 50) { inject(Message(_payload).sealTo(_to, _topics, _ttl, _workToProve)); } diff --git a/libwhisper/WhisperDB.cpp b/libwhisper/WhisperDB.cpp index a31e2a6eb..118c1cb12 100644 --- a/libwhisper/WhisperDB.cpp +++ b/libwhisper/WhisperDB.cpp @@ -22,39 +22,30 @@ #include "WhisperDB.h" #include #include +#include "WhisperHost.h" + using namespace std; using namespace dev; using namespace dev::shh; namespace fs = boost::filesystem; -WhisperDB::WhisperDB(DBType _t): m_type(_t) +WhisperDB::WhisperDB(string const& _type) { m_readOptions.verify_checksums = true; string path = dev::getDataDir("shh"); fs::create_directories(path); fs::permissions(path, fs::owner_all); + path.append("\\").append(_type); leveldb::Options op; op.create_if_missing = true; op.max_open_files = 256; - string suffix = getTypeSuffix(); leveldb::DB* p = nullptr; - leveldb::Status status = leveldb::DB::Open(op, path + suffix, &p); + leveldb::Status status = leveldb::DB::Open(op, path, &p); m_db.reset(p); if (!status.ok()) BOOST_THROW_EXCEPTION(FailedToOpenLevelDB(status.ToString())); } -string WhisperDB::getTypeSuffix() -{ - switch(m_type) - { - case Messages: return "\\messages"; - case Filters: return "\\filters"; - } - - return "\\misc"; -} - string WhisperDB::lookup(dev::h256 const& _key) const { string ret; @@ -91,11 +82,8 @@ void WhisperDB::kill(dev::h256 const& _key) BOOST_THROW_EXCEPTION(FailedDeleteInLevelDB(status.ToString())); } -void WhisperDB::loadAllMessages(std::map& o_dst) +void WhisperMessagesDB::loadAllMessages(std::map& o_dst) { - if (m_type != Messages) - BOOST_THROW_EXCEPTION(WrongTypeLevelDB()); - leveldb::ReadOptions op; op.fill_cache = false; op.verify_checksums = true; @@ -150,11 +138,8 @@ void WhisperDB::loadAllMessages(std::map& o_dst) } } -void WhisperDB::save(h256 const& _key, Envelope const& _e) +void WhisperMessagesDB::saveSingleMessage(h256 const& _key, Envelope const& _e) { - if (m_type != Messages) - BOOST_THROW_EXCEPTION(WrongTypeLevelDB()); - try { RLPStream rlp; @@ -172,3 +157,37 @@ void WhisperDB::save(h256 const& _key, Envelope const& _e) cwarn << boost::diagnostic_information(ex); } } + +vector WhisperFiltersDB::restoreTopicsFromDB(WhisperHost* _host, string const& _app) +{ + vector ret; + h256 s = sha3(_app); + h256 h = sha3(s); + string raw = lookup(h); + + bytes plain; + + decryptSym(s, raw, plain); + + RLP rlp(plain); + auto sz = rlp.itemCountStrict(); + + for (unsigned i = 0; i < sz; ++i) + { + RLP r = rlp[i]; + bytesConstRef ref(r.toBytesConstRef()); + Topics topics; + unsigned num = ref.size() / h256::size; + for (unsigned j = 0; j < num; ++j) + { + h256 topic(ref.data() + j * h256::size, h256::ConstructFromPointerType()); + topics.push_back(topic); + } + + unsigned w = _host->installWatch(topics); + ret.push_back(w); + } + + return ret; +} + diff --git a/libwhisper/WhisperDB.h b/libwhisper/WhisperDB.h index 3c7fc4375..1ea29d5a3 100644 --- a/libwhisper/WhisperDB.h +++ b/libwhisper/WhisperDB.h @@ -37,30 +37,39 @@ struct FailedInsertInLevelDB: virtual Exception { FailedInsertInLevelDB(std::str struct FailedLookupInLevelDB: virtual Exception { FailedLookupInLevelDB(std::string const& _message): Exception(_message) {} }; struct FailedDeleteInLevelDB: virtual Exception { FailedDeleteInLevelDB(std::string const& _message): Exception(_message) {} }; +class WhisperHost; + class WhisperDB { public: - enum DBType { Messages, Filters }; - - WhisperDB(DBType _t); - ~WhisperDB() {} - + WhisperDB(std::string const& _type); + virtual ~WhisperDB() {} std::string lookup(dev::h256 const& _key) const; void insert(dev::h256 const& _key, std::string const& _value); void insert(dev::h256 const& _key, bytes const& _value); void kill(dev::h256 const& _key); - void loadAllMessages(std::map& o_dst); - void save(dev::h256 const& _key, Envelope const& _e); - -private: - std::string getTypeSuffix(); +protected: leveldb::ReadOptions m_readOptions; leveldb::WriteOptions m_writeOptions; std::unique_ptr m_db; - DBType m_type; +}; - enum MetaInformation { StoreForeverFlag = 1, WatchedFlag = 2 }; +class WhisperMessagesDB: public WhisperDB +{ +public: + WhisperMessagesDB(): WhisperDB("messages") {} + virtual ~WhisperMessagesDB() {} + void loadAllMessages(std::map& o_dst); + void saveSingleMessage(dev::h256 const& _key, Envelope const& _e); +}; + +class WhisperFiltersDB: public WhisperDB +{ +public: + WhisperFiltersDB(): WhisperDB("filters") {} + virtual ~WhisperFiltersDB() {} + std::vector restoreTopicsFromDB(WhisperHost* _host, std::string const& _app); }; } diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index 856d7be7c..b50698ed1 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -241,13 +241,13 @@ void WhisperHost::saveMessagesToBD() try { - WhisperDB db(WhisperDB::Messages); + WhisperMessagesDB db; ReadGuard g(x_messages); unsigned now = (unsigned)time(0); for (auto const& m: m_messages) if (m.second.expiry() > now) if (isWatched(m.second)) - db.save(m.first, m.second); + db.saveSingleMessage(m.first, m.second); } catch(FailedToOpenLevelDB const& ex) { @@ -271,7 +271,7 @@ void WhisperHost::loadMessagesFromBD() try { map m; - WhisperDB db(WhisperDB::Messages); + WhisperMessagesDB db; db.loadAllMessages(m); WriteGuard g(x_messages); m_messages.swap(m); @@ -313,44 +313,11 @@ void WhisperHost::saveTopicsToDB(string const& _app, string const& _password) rlp.swapOut(plain); } + h256 s = sha3(_app); + h256 h = sha3(s); bytes encrypted; - h256 s = sha3(_password); encryptSym(s, &plain, encrypted); - h256 h = sha3(_app); - WhisperDB db(WhisperDB::Filters); + WhisperFiltersDB db; db.insert(h, encrypted); } - -vector WhisperHost::restoreTopicsFromDB(string const& _app, string const& _password) -{ - vector ret; - h256 h = sha3(_app); - WhisperDB db(WhisperDB::Filters); - string raw = db.lookup(h); - - bytes plain; - h256 s = sha3(_password); - decryptSym(s, raw, plain); - - RLP rlp(plain); - auto sz = rlp.itemCountStrict(); - - for (unsigned i = 0; i < sz; ++i) - { - RLP r = rlp[i]; - bytesConstRef ref(r.toBytesConstRef()); - Topics topics; - unsigned num = ref.size() / h256::size; - for (unsigned j = 0; j < num; ++j) - { - h256 topic(ref.data() + j * h256::size, h256::ConstructFromPointerType()); - topics.push_back(topic); - } - - unsigned w = installWatch(topics); - ret.push_back(w); - } - - return ret; -} diff --git a/libwhisper/WhisperHost.h b/libwhisper/WhisperHost.h index ab49b8392..6357043e9 100644 --- a/libwhisper/WhisperHost.h +++ b/libwhisper/WhisperHost.h @@ -64,7 +64,6 @@ public: virtual h256s watchMessages(unsigned _watchId) override; ///< returns IDs of messages, which match specific watch criteria virtual Envelope envelope(h256 _m) const override { try { dev::ReadGuard l(x_messages); return m_messages.at(_m); } catch (...) { return Envelope(); } } virtual void saveTopicsToDB(std::string const& _app, std::string const& _password); - virtual std::vector restoreTopicsFromDB(std::string const& _app, std::string const& _password); protected: virtual void doWork() override; diff --git a/test/libwhisper/whisperDB.cpp b/test/libwhisper/whisperDB.cpp index 902b9cf98..b6fabfa07 100644 --- a/test/libwhisper/whisperDB.cpp +++ b/test/libwhisper/whisperDB.cpp @@ -48,7 +48,7 @@ BOOST_AUTO_TEST_CASE(basic) string const text2 = "ipsum"; h256 h1(0xBEEF); h256 h2(0xC0FFEE); - WhisperDB db(WhisperDB::Messages); + WhisperMessagesDB db; db.kill(h1); db.kill(h2); @@ -95,7 +95,7 @@ BOOST_AUTO_TEST_CASE(persistence) h256 const h2(0xBADD00DE); { - WhisperDB db(WhisperDB::Messages); + WhisperMessagesDB db; db.kill(h1); db.kill(h2); s = db.lookup(h1); @@ -110,7 +110,7 @@ BOOST_AUTO_TEST_CASE(persistence) this_thread::sleep_for(chrono::milliseconds(20)); { - WhisperDB db(WhisperDB::Messages); + WhisperMessagesDB db; db.insert(h1, text1); db.insert(h2, text2); } @@ -118,7 +118,7 @@ BOOST_AUTO_TEST_CASE(persistence) this_thread::sleep_for(chrono::milliseconds(20)); { - WhisperDB db(WhisperDB::Messages); + WhisperMessagesDB db; s = db.lookup(h2); BOOST_REQUIRE(!s.compare(text2)); s = db.lookup(h1); @@ -170,7 +170,7 @@ BOOST_AUTO_TEST_CASE(messages) } } - WhisperDB db(WhisperDB::Messages); + WhisperMessagesDB db; unsigned x = 0; for (auto i: m1) @@ -191,7 +191,7 @@ BOOST_AUTO_TEST_CASE(corruptedData) h256 x = h256::random(); { - WhisperDB db(WhisperDB::Messages); + WhisperMessagesDB db; db.insert(x, "this is a test input, representing corrupt data"); } @@ -203,7 +203,7 @@ BOOST_AUTO_TEST_CASE(corruptedData) } { - WhisperDB db(WhisperDB::Messages); + WhisperMessagesDB db; string s = db.lookup(x); BOOST_REQUIRE(s.empty()); } @@ -214,7 +214,7 @@ BOOST_AUTO_TEST_CASE(filters) cnote << "Testing filters saving..."; VerbosityHolder setTemporaryLevel(2); string const app("test suite whisperDB/filters"); - string const password("some pseudorandom stuff"); + string const password("some pseudorandom stuff"); // todo: delete { p2p::Host h("Test"); @@ -235,7 +235,9 @@ BOOST_AUTO_TEST_CASE(filters) host1.setIdealPeerCount(1); auto whost1 = host1.registerCapability(new WhisperHost()); host1.start(); - auto ids = whost1->restoreTopicsFromDB(app, password); + //auto ids = whost1->restoreTopicsFromDB(app, password); // todo: delete + WhisperFiltersDB db; + auto ids = db.restoreTopicsFromDB(whost1.get(), app); std::thread listener([&]() { From 822878c46934aa8de81405e20f6d6d3e632c3676 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Fri, 24 Jul 2015 11:21:06 +0200 Subject: [PATCH 07/10] bugfix in saveTopicsToDB() --- libwhisper/WhisperHost.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index b50698ed1..71fafe365 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -291,10 +291,11 @@ void WhisperHost::loadMessagesFromBD() void WhisperHost::saveTopicsToDB(string const& _app, string const& _password) { bytes plain; + RLPStream rlp; DEV_GUARDED(m_filterLock) { - RLPStream rlp(m_filters.size()); + rlp.appendList(m_filters.size()); for (auto const& x: m_filters) { @@ -304,15 +305,14 @@ void WhisperHost::saveTopicsToDB(string const& _app, string const& _password) unsigned i = 0; for (auto const& t: topics) - memcpy(p.get() + i * h256::size, t.data(), h256::size); + memcpy(p.get() + h256::size * i++, t.data(), h256::size); bytesConstRef ref(p.get(), RawDataSize); rlp.append(ref); - } - - rlp.swapOut(plain); + } } + rlp.swapOut(plain); h256 s = sha3(_app); h256 h = sha3(s); bytes encrypted; From 68d6aa89062594312336e2f58b14fd3fde94284d Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Fri, 24 Jul 2015 14:50:27 +0200 Subject: [PATCH 08/10] moved some fucntions to WhsiperDB class --- libwhisper/WhisperDB.cpp | 54 +++++++++++++++++++++-------------- libwhisper/WhisperDB.h | 3 +- libwhisper/WhisperHost.cpp | 24 ++++------------ libwhisper/WhisperHost.h | 7 +++-- test/libwhisper/whisperDB.cpp | 14 +++++---- 5 files changed, 53 insertions(+), 49 deletions(-) diff --git a/libwhisper/WhisperDB.cpp b/libwhisper/WhisperDB.cpp index 118c1cb12..4e8f7cf92 100644 --- a/libwhisper/WhisperDB.cpp +++ b/libwhisper/WhisperDB.cpp @@ -158,36 +158,48 @@ void WhisperMessagesDB::saveSingleMessage(h256 const& _key, Envelope const& _e) } } -vector WhisperFiltersDB::restoreTopicsFromDB(WhisperHost* _host, string const& _app) +vector WhisperFiltersDB::restoreTopicsFromDB(WhisperHost* _host, string const& _password) { vector ret; - h256 s = sha3(_app); + h256 s = sha3(_password); h256 h = sha3(s); string raw = lookup(h); - - bytes plain; - - decryptSym(s, raw, plain); - - RLP rlp(plain); - auto sz = rlp.itemCountStrict(); - - for (unsigned i = 0; i < sz; ++i) + if (!raw.empty()) { - RLP r = rlp[i]; - bytesConstRef ref(r.toBytesConstRef()); - Topics topics; - unsigned num = ref.size() / h256::size; - for (unsigned j = 0; j < num; ++j) + bytes plain; + decryptSym(s, raw, plain); + RLP rlp(plain); + auto sz = rlp.itemCountStrict(); + + for (unsigned i = 0; i < sz; ++i) { - h256 topic(ref.data() + j * h256::size, h256::ConstructFromPointerType()); - topics.push_back(topic); - } + RLP r = rlp[i]; + bytesConstRef ref(r.toBytesConstRef()); + Topics topics; + unsigned num = ref.size() / h256::size; + for (unsigned j = 0; j < num; ++j) + { + h256 topic(ref.data() + j * h256::size, h256::ConstructFromPointerType()); + topics.push_back(topic); + } - unsigned w = _host->installWatch(topics); - ret.push_back(w); + unsigned w = _host->installWatch(topics); + ret.push_back(w); + } } return ret; } +void WhisperFiltersDB::saveTopicsToDB(WhisperHost const& _host, std::string const& _password) +{ + bytes plain; + RLPStream rlp; + _host.exportFilters(rlp); + rlp.swapOut(plain); + h256 s = sha3(_password); + h256 h = sha3(s); + bytes encrypted; + encryptSym(s, &plain, encrypted); + insert(h, encrypted); +} diff --git a/libwhisper/WhisperDB.h b/libwhisper/WhisperDB.h index 1ea29d5a3..6ca79666f 100644 --- a/libwhisper/WhisperDB.h +++ b/libwhisper/WhisperDB.h @@ -69,7 +69,8 @@ class WhisperFiltersDB: public WhisperDB public: WhisperFiltersDB(): WhisperDB("filters") {} virtual ~WhisperFiltersDB() {} - std::vector restoreTopicsFromDB(WhisperHost* _host, std::string const& _app); + std::vector restoreTopicsFromDB(WhisperHost* _host, std::string const& _password); + void saveTopicsToDB(WhisperHost const& _host, std::string const& _password); }; } diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index 71fafe365..1892c3acd 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -30,7 +30,7 @@ using namespace dev; using namespace dev::p2p; using namespace dev::shh; -WhisperHost::WhisperHost(bool _useDB): Worker("shh"), m_useDB(_useDB) +WhisperHost::WhisperHost(bool _storeMessagesInDB): Worker("shh"), m_storeMessagesInDB(_storeMessagesInDB) { loadMessagesFromBD(); } @@ -236,7 +236,7 @@ bool WhisperHost::isWatched(Envelope const& _e) const void WhisperHost::saveMessagesToBD() { - if (!m_useDB) + if (!m_storeMessagesInDB) return; try @@ -265,7 +265,7 @@ void WhisperHost::saveMessagesToBD() void WhisperHost::loadMessagesFromBD() { - if (!m_useDB) + if (!m_storeMessagesInDB) return; try @@ -288,14 +288,11 @@ void WhisperHost::loadMessagesFromBD() } } -void WhisperHost::saveTopicsToDB(string const& _app, string const& _password) +void WhisperHost::exportFilters(RLPStream& o_dst) const { - bytes plain; - RLPStream rlp; - DEV_GUARDED(m_filterLock) { - rlp.appendList(m_filters.size()); + o_dst.appendList(m_filters.size()); for (auto const& x: m_filters) { @@ -308,16 +305,7 @@ void WhisperHost::saveTopicsToDB(string const& _app, string const& _password) memcpy(p.get() + h256::size * i++, t.data(), h256::size); bytesConstRef ref(p.get(), RawDataSize); - rlp.append(ref); + o_dst.append(ref); } } - - rlp.swapOut(plain); - h256 s = sha3(_app); - h256 h = sha3(s); - bytes encrypted; - encryptSym(s, &plain, encrypted); - - WhisperFiltersDB db; - db.insert(h, encrypted); } diff --git a/libwhisper/WhisperHost.h b/libwhisper/WhisperHost.h index 6357043e9..db2f60bfd 100644 --- a/libwhisper/WhisperHost.h +++ b/libwhisper/WhisperHost.h @@ -48,7 +48,7 @@ class WhisperHost: public HostCapability, public Interface, public friend class WhisperPeer; public: - WhisperHost(bool _useDB = false); + WhisperHost(bool _storeMessagesInDB = false); virtual ~WhisperHost(); unsigned protocolVersion() const { return WhisperProtocolVersion; } void cleanup(); ///< remove old messages @@ -63,7 +63,8 @@ public: virtual h256s checkWatch(unsigned _watchId) override; virtual h256s watchMessages(unsigned _watchId) override; ///< returns IDs of messages, which match specific watch criteria virtual Envelope envelope(h256 _m) const override { try { dev::ReadGuard l(x_messages); return m_messages.at(_m); } catch (...) { return Envelope(); } } - virtual void saveTopicsToDB(std::string const& _app, std::string const& _password); + + void exportFilters(dev::RLPStream& o_dst) const; protected: virtual void doWork() override; @@ -86,7 +87,7 @@ private: std::map m_watches; TopicBloomFilter m_bloom; - bool m_useDB; ///< needed for tests and other special cases + bool m_storeMessagesInDB; ///< needed for tests and other special cases }; } diff --git a/test/libwhisper/whisperDB.cpp b/test/libwhisper/whisperDB.cpp index b6fabfa07..aca6f6914 100644 --- a/test/libwhisper/whisperDB.cpp +++ b/test/libwhisper/whisperDB.cpp @@ -213,15 +213,15 @@ BOOST_AUTO_TEST_CASE(filters) { cnote << "Testing filters saving..."; VerbosityHolder setTemporaryLevel(2); - string const app("test suite whisperDB/filters"); - string const password("some pseudorandom stuff"); // todo: delete + string const password("some pseudorandom string"); { + WhisperFiltersDB db; p2p::Host h("Test"); auto wh = h.registerCapability(new WhisperHost()); wh->installWatch(BuildTopic("t1")); wh->installWatch(BuildTopic("t2")); - wh->saveTopicsToDB(app, password); + db.saveTopicsToDB(*wh, password); } short unsigned port1 = 30313; @@ -235,9 +235,11 @@ BOOST_AUTO_TEST_CASE(filters) host1.setIdealPeerCount(1); auto whost1 = host1.registerCapability(new WhisperHost()); host1.start(); - //auto ids = whost1->restoreTopicsFromDB(app, password); // todo: delete WhisperFiltersDB db; - auto ids = db.restoreTopicsFromDB(whost1.get(), app); + auto watches = db.restoreTopicsFromDB(whost1.get(), password); + auto zero = db.restoreTopicsFromDB(whost1.get(), password + "qwer"); + BOOST_REQUIRE(!watches.empty()); + BOOST_REQUIRE(zero.empty()); std::thread listener([&]() { @@ -249,7 +251,7 @@ BOOST_AUTO_TEST_CASE(filters) for (unsigned j = 0; j < 200 && messageCount < 2; ++j) { - for (unsigned id: ids) + for (unsigned id: watches) for (auto const& e: whost1->checkWatch(id)) { Message msg = whost1->envelope(e).open(whost1->fullTopics(id)); From 432e1988f94e659241899de70cfab6096d11864b Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Fri, 24 Jul 2015 18:22:18 +0200 Subject: [PATCH 09/10] refactoring --- libethereum/BlockChain.cpp | 2 ++ libp2p/Common.h | 20 +------------------- libwhisper/WhisperDB.cpp | 22 +++++++--------------- libwhisper/WhisperDB.h | 4 ++-- test/libwhisper/whisperDB.cpp | 8 ++++---- 5 files changed, 16 insertions(+), 40 deletions(-) diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 0baff8057..4e5386412 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -386,6 +386,7 @@ tuple BlockChain::sync(BlockQueue& _bq, OverlayDB c if (!badBlocks.empty()) badBlocks.push_back(block.verified.info.hash()); else + { do { try { @@ -427,6 +428,7 @@ tuple BlockChain::sync(BlockQueue& _bq, OverlayDB c badBlocks.push_back(block.verified.info.hash()); } } while (false); + } return make_tuple(ImportRoute{dead, fresh, goodTransactions}, _bq.doneDrain(badBlocks), count); } diff --git a/libp2p/Common.h b/libp2p/Common.h index 4c7fa837b..1b65d4759 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -243,25 +243,7 @@ public: void stop() { m_stopped = true; DEV_GUARDED(x_timers) m_timers.clear(); } protected: - void reap() - { - Guard l(x_timers); - std::vector::iterator t = m_timers.begin(); - while (t != m_timers.end()) - if (t->expired()) - { - t->wait(); - t = m_timers.erase(t); - } - else - t++; - - m_timers.emplace_back(m_io, m_reapIntervalMs, [this](boost::system::error_code const& ec) - { - if (!ec) - reap(); - }); - } + void reap(); private: ba::io_service& m_io; diff --git a/libwhisper/WhisperDB.cpp b/libwhisper/WhisperDB.cpp index 4e8f7cf92..84e9b1a31 100644 --- a/libwhisper/WhisperDB.cpp +++ b/libwhisper/WhisperDB.cpp @@ -158,17 +158,13 @@ void WhisperMessagesDB::saveSingleMessage(h256 const& _key, Envelope const& _e) } } -vector WhisperFiltersDB::restoreTopicsFromDB(WhisperHost* _host, string const& _password) +vector WhisperFiltersDB::restoreTopicsFromDB(WhisperHost* _host, h256 const& _id) { vector ret; - h256 s = sha3(_password); - h256 h = sha3(s); - string raw = lookup(h); + string raw = lookup(_id); if (!raw.empty()) { - bytes plain; - decryptSym(s, raw, plain); - RLP rlp(plain); + RLP rlp(raw); auto sz = rlp.itemCountStrict(); for (unsigned i = 0; i < sz; ++i) @@ -191,15 +187,11 @@ vector WhisperFiltersDB::restoreTopicsFromDB(WhisperHost* _host, strin return ret; } -void WhisperFiltersDB::saveTopicsToDB(WhisperHost const& _host, std::string const& _password) +void WhisperFiltersDB::saveTopicsToDB(WhisperHost const& _host, h256 const& _id) { - bytes plain; + bytes b; RLPStream rlp; _host.exportFilters(rlp); - rlp.swapOut(plain); - h256 s = sha3(_password); - h256 h = sha3(s); - bytes encrypted; - encryptSym(s, &plain, encrypted); - insert(h, encrypted); + rlp.swapOut(b); + insert(_id, b); } diff --git a/libwhisper/WhisperDB.h b/libwhisper/WhisperDB.h index 6ca79666f..bd6a22251 100644 --- a/libwhisper/WhisperDB.h +++ b/libwhisper/WhisperDB.h @@ -69,8 +69,8 @@ class WhisperFiltersDB: public WhisperDB public: WhisperFiltersDB(): WhisperDB("filters") {} virtual ~WhisperFiltersDB() {} - std::vector restoreTopicsFromDB(WhisperHost* _host, std::string const& _password); - void saveTopicsToDB(WhisperHost const& _host, std::string const& _password); + std::vector restoreTopicsFromDB(WhisperHost* _host, h256 const& _id); + void saveTopicsToDB(WhisperHost const& _host, h256 const& _id); }; } diff --git a/test/libwhisper/whisperDB.cpp b/test/libwhisper/whisperDB.cpp index aca6f6914..38486772d 100644 --- a/test/libwhisper/whisperDB.cpp +++ b/test/libwhisper/whisperDB.cpp @@ -213,7 +213,7 @@ BOOST_AUTO_TEST_CASE(filters) { cnote << "Testing filters saving..."; VerbosityHolder setTemporaryLevel(2); - string const password("some pseudorandom string"); + h256 persistID(0xC0FFEE); { WhisperFiltersDB db; @@ -221,7 +221,7 @@ BOOST_AUTO_TEST_CASE(filters) auto wh = h.registerCapability(new WhisperHost()); wh->installWatch(BuildTopic("t1")); wh->installWatch(BuildTopic("t2")); - db.saveTopicsToDB(*wh, password); + db.saveTopicsToDB(*wh, persistID); } short unsigned port1 = 30313; @@ -236,8 +236,8 @@ BOOST_AUTO_TEST_CASE(filters) auto whost1 = host1.registerCapability(new WhisperHost()); host1.start(); WhisperFiltersDB db; - auto watches = db.restoreTopicsFromDB(whost1.get(), password); - auto zero = db.restoreTopicsFromDB(whost1.get(), password + "qwer"); + auto watches = db.restoreTopicsFromDB(whost1.get(), persistID); + auto zero = db.restoreTopicsFromDB(whost1.get(), ++persistID); BOOST_REQUIRE(!watches.empty()); BOOST_REQUIRE(zero.empty()); From e19825f8d34ae09594b4ad3e1fe6fe0feeeeeb94 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Fri, 24 Jul 2015 20:44:27 +0200 Subject: [PATCH 10/10] test ports changed --- test/libp2p/peer.cpp | 14 ++++++------ test/libwhisper/whisperDB.cpp | 4 ++-- test/libwhisper/whisperTopic.cpp | 39 ++++++++++++++++++-------------- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/test/libp2p/peer.cpp b/test/libp2p/peer.cpp index acc420c96..fa04be5fd 100644 --- a/test/libp2p/peer.cpp +++ b/test/libp2p/peer.cpp @@ -44,8 +44,8 @@ BOOST_AUTO_TEST_CASE(host) return; VerbosityHolder setTemporaryLevel(10); - NetworkPreferences host1prefs("127.0.0.1", 30311, false); - NetworkPreferences host2prefs("127.0.0.1", 30312, false); + NetworkPreferences host1prefs("127.0.0.1", 30321, false); + NetworkPreferences host2prefs("127.0.0.1", 30322, false); Host host1("Test", host1prefs); Host host2("Test", host2prefs); host1.start(); @@ -97,7 +97,7 @@ BOOST_AUTO_TEST_CASE(saveNodes) for (unsigned i = 0; i < c_nodes; ++i) { - Host* h = new Host("Test", NetworkPreferences("127.0.0.1", 30300 + i, false)); + Host* h = new Host("Test", NetworkPreferences("127.0.0.1", 30325 + i, false)); h->setIdealPeerCount(10); // starting host is required so listenport is available h->start(); @@ -155,8 +155,8 @@ BOOST_AUTO_TEST_CASE(requirePeer) VerbosityHolder reduceVerbosity(10); const char* const localhost = "127.0.0.1"; - NetworkPreferences prefs1(localhost, 30301, false); - NetworkPreferences prefs2(localhost, 30302, false); + NetworkPreferences prefs1(localhost, 30323, false); + NetworkPreferences prefs2(localhost, 30324, false); Host host1("Test", prefs1); host1.start(); @@ -229,9 +229,9 @@ BOOST_AUTO_TEST_SUITE_END() int peerTest(int argc, char** argv) { Public remoteAlias; - short listenPort = 30303; + short listenPort = 30304; string remoteHost; - short remotePort = 30303; + short remotePort = 30304; for (int i = 1; i < argc; ++i) { diff --git a/test/libwhisper/whisperDB.cpp b/test/libwhisper/whisperDB.cpp index 38486772d..a62539bfd 100644 --- a/test/libwhisper/whisperDB.cpp +++ b/test/libwhisper/whisperDB.cpp @@ -224,7 +224,7 @@ BOOST_AUTO_TEST_CASE(filters) db.saveTopicsToDB(*wh, persistID); } - short unsigned port1 = 30313; + uint16_t port1 = 30308; unsigned const step = 10; bool host1Ready = false; bool sent = false; @@ -265,7 +265,7 @@ BOOST_AUTO_TEST_CASE(filters) } }); - Host host2("Test", NetworkPreferences("127.0.0.1", 30314, false)); + Host host2("Test", NetworkPreferences("127.0.0.1", 30309, false)); host2.setIdealPeerCount(1); auto whost2 = host2.registerCapability(new WhisperHost()); host2.start(); diff --git a/test/libwhisper/whisperTopic.cpp b/test/libwhisper/whisperTopic.cpp index 04f4f3f77..6a58be045 100644 --- a/test/libwhisper/whisperTopic.cpp +++ b/test/libwhisper/whisperTopic.cpp @@ -49,7 +49,8 @@ BOOST_AUTO_TEST_CASE(topic) cnote << "Testing Whisper..."; VerbosityHolder setTemporaryLevel(0); - Host host1("Test", NetworkPreferences("127.0.0.1", 30303, false)); + uint16_t port1 = 30311; + Host host1("Test", NetworkPreferences("127.0.0.1", port1, false)); host1.setIdealPeerCount(1); auto whost1 = host1.registerCapability(new WhisperHost()); host1.start(); @@ -80,14 +81,14 @@ BOOST_AUTO_TEST_CASE(topic) } }); - Host host2("Test", NetworkPreferences("127.0.0.1", 30300, false)); + Host host2("Test", NetworkPreferences("127.0.0.1", 30310, false)); host1.setIdealPeerCount(1); auto whost2 = host2.registerCapability(new WhisperHost()); host2.start(); while (!host1.haveNetwork()) this_thread::sleep_for(chrono::milliseconds(5)); - host2.addNode(host1.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), 30303, 30303)); + host2.addNode(host1.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), port1, port1)); // wait for nodes to connect this_thread::sleep_for(chrono::milliseconds(1000)); @@ -115,7 +116,8 @@ BOOST_AUTO_TEST_CASE(forwarding) VerbosityHolder setTemporaryLevel(0); // Host must be configured not to share peers. - Host host1("Listner", NetworkPreferences("127.0.0.1", 30303, false)); + uint16_t port1 = 30312; + Host host1("Listner", NetworkPreferences("127.0.0.1", port1, false)); host1.setIdealPeerCount(1); auto whost1 = host1.registerCapability(new WhisperHost()); host1.start(); @@ -150,7 +152,8 @@ BOOST_AUTO_TEST_CASE(forwarding) // Host must be configured not to share peers. - Host host2("Forwarder", NetworkPreferences("127.0.0.1", 30305, false)); + uint16_t port2 = 30313; + Host host2("Forwarder", NetworkPreferences("127.0.0.1", port2, false)); host2.setIdealPeerCount(1); auto whost2 = host2.registerCapability(new WhisperHost()); host2.start(); @@ -167,7 +170,7 @@ BOOST_AUTO_TEST_CASE(forwarding) this_thread::sleep_for(chrono::milliseconds(50)); this_thread::sleep_for(chrono::milliseconds(500)); - host2.addNode(host1.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), 30303, 30303)); + host2.addNode(host1.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), port1, port1)); startedForwarder = true; @@ -188,11 +191,11 @@ BOOST_AUTO_TEST_CASE(forwarding) while (!startedForwarder) this_thread::sleep_for(chrono::milliseconds(50)); - Host ph("Sender", NetworkPreferences("127.0.0.1", 30300, false)); + Host ph("Sender", NetworkPreferences("127.0.0.1", 30314, false)); ph.setIdealPeerCount(1); shared_ptr wh = ph.registerCapability(new WhisperHost()); ph.start(); - ph.addNode(host2.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), 30305, 30305)); + ph.addNode(host2.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), port2, port2)); while (!ph.haveNetwork()) this_thread::sleep_for(chrono::milliseconds(10)); @@ -221,7 +224,8 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) bool done = false; // Host must be configured not to share peers. - Host host1("Forwarder", NetworkPreferences("127.0.0.1", 30305, false)); + uint16_t port1 = 30315; + Host host1("Forwarder", NetworkPreferences("127.0.0.1", port1, false)); host1.setIdealPeerCount(1); auto whost1 = host1.registerCapability(new WhisperHost()); host1.start(); @@ -250,14 +254,14 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) this_thread::sleep_for(chrono::milliseconds(2)); { - Host host2("Sender", NetworkPreferences("127.0.0.1", 30300, false)); + Host host2("Sender", NetworkPreferences("127.0.0.1", 30316, false)); host2.setIdealPeerCount(1); shared_ptr whost2 = host2.registerCapability(new WhisperHost()); host2.start(); while (!host2.haveNetwork()) this_thread::sleep_for(chrono::milliseconds(2)); - host2.requirePeer(host1.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), 30305, 30305)); + host2.requirePeer(host1.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), port1, port1)); while (!host2.peerCount() || !host1.peerCount()) this_thread::sleep_for(chrono::milliseconds(5)); @@ -267,7 +271,7 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) } { - Host ph("Listener", NetworkPreferences("127.0.0.1", 30300, false)); + Host ph("Listener", NetworkPreferences("127.0.0.1", 30317, false)); ph.setIdealPeerCount(1); shared_ptr wh = ph.registerCapability(new WhisperHost()); ph.start(); @@ -275,7 +279,7 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) this_thread::sleep_for(chrono::milliseconds(2)); auto w = wh->installWatch(BuildTopicMask("test")); // only interested in odd packets - ph.requirePeer(host1.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), 30305, 30305)); + ph.requirePeer(host1.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), port1, port1)); for (int i = 0; i < 200 && !result; ++i) { @@ -303,14 +307,15 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) cnote << "Testing Topic Advertising..."; VerbosityHolder setTemporaryLevel(2); - Host host1("first", NetworkPreferences("127.0.0.1", 30303, false)); + Host host1("first", NetworkPreferences("127.0.0.1", 30319, false)); host1.setIdealPeerCount(1); auto whost1 = host1.registerCapability(new WhisperHost()); host1.start(); while (!host1.haveNetwork()) this_thread::sleep_for(chrono::milliseconds(10)); - Host host2("second", NetworkPreferences("127.0.0.1", 30305, false)); + uint16_t port2 = 30318; + Host host2("second", NetworkPreferences("127.0.0.1", port2, false)); host2.setIdealPeerCount(1); auto whost2 = host2.registerCapability(new WhisperHost()); unsigned w2 = whost2->installWatch(BuildTopicMask("test2")); @@ -319,7 +324,7 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) while (!host2.haveNetwork()) this_thread::sleep_for(chrono::milliseconds(10)); - host1.addNode(host2.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), 30305, 30305)); + host1.addNode(host2.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), port2, port2)); while (!host1.haveNetwork()) this_thread::sleep_for(chrono::milliseconds(10)); @@ -385,7 +390,7 @@ BOOST_AUTO_TEST_CASE(selfAddressed) char const* text = "deterministic pseudorandom test"; BuildTopicMask mask(text); - Host host("first", NetworkPreferences("127.0.0.1", 30305, false)); + Host host("first", NetworkPreferences("127.0.0.1", 30320, false)); auto wh = host.registerCapability(new WhisperHost()); auto watch = wh->installWatch(BuildTopicMask(text));