Merge branch 'whisperencryption' into develop

alethzero/MainWin.cpp

@@ -1597,7 +1597,7 @@ void Main::on_destination_currentTextChanged()
 //	updateFee();
 }
 
-static shh::Topic topicFromText(QString _s)
+static shh::FullTopic topicFromText(QString _s)
 {
 	shh::BuildTopic ret;
 	while (_s.size())
@@ -2414,10 +2414,10 @@ void Main::refreshWhispers()
 		shh::Envelope const& e = w.second;
 		shh::Message m;
 		for (pair<Public, Secret> const& i: m_server->ids())
-			if (!!(m = e.open(i.second)))
+			if (!!(m = e.open(shh::FullTopic(), i.second)))
 				break;
 		if (!m)
-			m = e.open();
+			m = e.open(shh::FullTopic());
 
 		QString msg;
 		if (m.from())

libdevcore/FixedHash.h

@@ -67,6 +67,9 @@ public:
 	/// Explicitly construct, copying from a byte array.
 	explicit FixedHash(bytes const& _b) { if (_b.size() == N) memcpy(m_data.data(), _b.data(), std::min<unsigned>(_b.size(), N)); }
 
+	/// Explicitly construct, copying from a byte array.
+	explicit FixedHash(bytesConstRef _b) { if (_b.size() == N) memcpy(m_data.data(), _b.data(), std::min<unsigned>(_b.size(), N)); }
+
 	/// Explicitly construct, copying from a bytes in memory with given pointer.
 	explicit FixedHash(byte const* _bs, ConstructFromPointerType) { memcpy(m_data.data(), _bs, N); }
 

libdevcrypto/Common.cpp

@@ -80,6 +80,18 @@ bool dev::decrypt(Secret const& _k, bytesConstRef _cipher, bytes& o_plaintext)
 	return true;
 }
 
+void dev::encryptSym(Secret const& _k, bytesConstRef _plain, bytes& o_cipher)
+{
+	// TOOD: @alex @subtly do this properly.
+	encrypt(KeyPair(_k).pub(), _plain, o_cipher);
+}
+
+bool dev::decryptSym(Secret const& _k, bytesConstRef _cipher, bytes& o_plain)
+{
+	// TODO: @alex @subtly do this properly.
+	return decrypt(_k, _cipher, o_plain);
+}
+
 Public dev::recover(Signature const& _sig, h256 const& _message)
 {
 	return s_secp256k1.recover(_sig, _message.ref());

libdevcrypto/Common.h

@@ -86,7 +86,13 @@ void encrypt(Public const& _k, bytesConstRef _plain, bytes& o_cipher);
 
 /// Decrypts cipher using Secret key.
 bool decrypt(Secret const& _k, bytesConstRef _cipher, bytes& o_plaintext);
-	
+
+/// Symmetric encryption.
+void encryptSym(Secret const& _k, bytesConstRef _plain, bytes& o_cipher);
+
+/// Symmetric decryption.
+bool decryptSym(Secret const& _k, bytesConstRef _cipher, bytes& o_plaintext);
+
 /// Recovers Public key from signed message hash.
 Public recover(Signature const& _sig, h256 const& _hash);
 	

libdevcrypto/CryptoPP.h

@@ -62,7 +62,7 @@ using namespace CryptoPP;
 inline ECP::Point publicToPoint(Public const& _p) { Integer x(_p.data(), 32); Integer y(_p.data() + 32, 32); return std::move(ECP::Point(x,y)); }
 
 inline Integer secretToExponent(Secret const& _s) { return std::move(Integer(_s.data(), Secret::size)); }
-	
+
 /**
  * CryptoPP secp256k1 algorithms.
  */

libweb3jsonrpc/WebThreeStubServerBase.cpp

@@ -173,9 +173,9 @@ static shh::Envelope toSealed(Json::Value const& _json, shh::Message const& _m,
 	return _m.seal(_from, bt, ttl, workToProve);
 }
 
-static pair<shh::TopicMask, Public> toWatch(Json::Value const& _json)
+static pair<shh::FullTopic, Public> toWatch(Json::Value const& _json)
 {
-	shh::BuildTopicMask bt;
+	shh::BuildTopic bt;
 	Public to;
 
 	if (_json["to"].isString())
@@ -190,7 +190,7 @@ static pair<shh::TopicMask, Public> toWatch(Json::Value const& _json)
 				if (i.isString())
 					bt.shift(jsToBytes(i.asString()));
 	}
-	return make_pair(bt.toTopicMask(), to);
+	return make_pair(bt, to);
 }
 
 static Json::Value toJson(h256 const& _h, shh::Envelope const& _e, shh::Message const& _m)
@@ -576,12 +576,12 @@ Json::Value WebThreeStubServerBase::shh_changed(int const& _id)
 			if (pub)
 			{
 				cwarn << "Silently decrypting message from identity" << pub.abridged() << ": User validation hook goes here.";
-				m = e.open(m_ids[pub]);
+				m = e.open(face()->fullTopic(_id), m_ids[pub]);
-				if (!m)
-					continue;
 			}
 			else
-				m = e.open();
+				m = e.open(face()->fullTopic(_id));
+			if (!m)
+				continue;
 			ret.append(toJson(h, e, m));
 		}
 	

libwhisper/Common.h

@@ -62,6 +62,7 @@ enum WhisperPacket
 using TopicPart = FixedHash<4>;
 
 using Topic = std::vector<TopicPart>;
+using FullTopic = h256s;
 
 class BuildTopic
 {
@@ -75,7 +76,9 @@ public:
 	BuildTopic& shiftRaw(h256 const& _part) { m_parts.push_back(_part); return *this; }
 
 	operator Topic() const { return toTopic(); }
+	operator FullTopic() const { return toFullTopic(); }
 	Topic toTopic() const;
+	FullTopic toFullTopic() const { return m_parts; }
 
 protected:
 	BuildTopic& shiftBytes(bytes const& _b);
@@ -90,6 +93,7 @@ class TopicFilter
 {
 public:
 	TopicFilter() {}
+	TopicFilter(FullTopic const& _m) { m_topicMasks.push_back(TopicMask()); for (auto const& h: _m) m_topicMasks.back().push_back(std::make_pair(TopicPart(h), h ? ~TopicPart() : TopicPart())); }
 	TopicFilter(TopicMask const& _m): m_topicMasks(1, _m) {}
 	TopicFilter(TopicMasks const& _m): m_topicMasks(_m) {}
 	TopicFilter(RLP const& _r)//: m_topicMasks(_r.toVector<std::vector<>>())
@@ -123,7 +127,9 @@ public:
 	template <class T> BuildTopicMask& operator()(T const& _t) { shift(_t); return *this; }
 
 	operator TopicMask() const { return toTopicMask(); }
+	operator FullTopic() const { return toFullTopic(); }
 	TopicMask toTopicMask() const;
+	FullTopic toFullTopic() const { return m_parts; }
 };
 
 }

libwhisper/Interface.cpp

@@ -34,7 +34,6 @@ using namespace dev::shh;
 #endif
 #define clogS(X) dev::LogOutputStream<X, true>(false) << "| " << std::setw(2) << session()->socketId() << "] "
 
-unsigned Interface::installWatch(TopicMask const& _mask)
+Interface::~Interface()
 {
-	return installWatch(TopicFilter(_mask));
 }

libwhisper/Interface.h

@@ -47,8 +47,9 @@ class Watch;
 
 struct InstalledFilter
 {
-	InstalledFilter(TopicFilter const& _f): filter(_f) {}
+	InstalledFilter(FullTopic const& _f): full(_f), filter(_f) {}
 
+	FullTopic full;
 	TopicFilter filter;
 	unsigned refCount = 1;
 };
@@ -65,12 +66,12 @@ struct ClientWatch
 class Interface
 {
 public:
-	virtual ~Interface() {}
+	virtual ~Interface();
 
 	virtual void inject(Envelope const& _m, WhisperPeer* _from = nullptr) = 0;
 
-	unsigned installWatch(TopicMask const& _mask);
+	virtual FullTopic const& fullTopic(unsigned _id) const = 0;
-	virtual unsigned installWatch(TopicFilter const& _filter) = 0;
+	virtual unsigned installWatch(FullTopic const& _mask) = 0;
 	virtual unsigned installWatchOnId(h256 _filterId) = 0;
 	virtual void uninstallWatch(unsigned _watchId) = 0;
 	virtual h256s peekWatch(unsigned _watchId) const = 0;
@@ -79,10 +80,10 @@ public:
 
 	virtual Envelope envelope(h256 _m) const = 0;
 
-	void post(bytes const& _payload, Topic _topic, unsigned _ttl = 50, unsigned _workToProve = 50) { inject(Message(_payload).seal(_topic, _ttl, _workToProve)); }
+	void post(bytes const& _payload, FullTopic _topic, unsigned _ttl = 50, unsigned _workToProve = 50) { inject(Message(_payload).seal(_topic, _ttl, _workToProve)); }
-	void post(Public _to, bytes const& _payload, Topic _topic, unsigned _ttl = 50, unsigned _workToProve = 50) { inject(Message(_payload).sealTo(_to, _topic, _ttl, _workToProve)); }
+	void post(Public _to, bytes const& _payload, FullTopic _topic, unsigned _ttl = 50, unsigned _workToProve = 50) { inject(Message(_payload).sealTo(_to, _topic, _ttl, _workToProve)); }
-	void post(Secret _from, bytes const& _payload, Topic _topic, unsigned _ttl = 50, unsigned _workToProve = 50) { inject(Message(_payload).seal(_from, _topic, _ttl, _workToProve)); }
+	void post(Secret _from, bytes const& _payload, FullTopic _topic, unsigned _ttl = 50, unsigned _workToProve = 50) { inject(Message(_payload).seal(_from, _topic, _ttl, _workToProve)); }
-	void post(Secret _from, Public _to, bytes const& _payload, Topic _topic, unsigned _ttl = 50, unsigned _workToProve = 50) { inject(Message(_payload).sealTo(_from, _to, _topic, _ttl, _workToProve)); }
+	void post(Secret _from, Public _to, bytes const& _payload, FullTopic _topic, unsigned _ttl = 50, unsigned _workToProve = 50) { inject(Message(_payload).sealTo(_from, _to, _topic, _ttl, _workToProve)); }
 };
 
 struct WatshhChannel: public dev::LogChannel { static const char* name() { return "shh"; } static const int verbosity = 1; };
@@ -104,8 +105,7 @@ class Watch: public boost::noncopyable
 
 public:
 	Watch() {}
-	Watch(Interface& _c, TopicMask const& _f): m_c(&_c), m_id(_c.installWatch(_f)) {}
+	Watch(Interface& _c, FullTopic const& _f): m_c(&_c), m_id(_c.installWatch(_f)) {}
-	Watch(Interface& _c, TopicFilter const& _tf): m_c(&_c), m_id(_c.installWatch(_tf)) {}
 	~Watch() { if (m_c) m_c->uninstallWatch(m_id); }
 
 	h256s check() { return m_c ? m_c->checkWatch(m_id) : h256s(); }

libwhisper/Message.cpp

@@ -26,7 +26,16 @@ using namespace dev;
 using namespace dev::p2p;
 using namespace dev::shh;
 
-Message::Message(Envelope const& _e, Secret const& _s)
+Topic collapse(FullTopic const& _fullTopic)
+{
+	Topic ret;
+	ret.reserve(_fullTopic.size());
+	for (auto const& ft: _fullTopic)
+		ret.push_back(TopicPart(ft));
+	return ret;
+}
+
+Message::Message(Envelope const& _e, FullTopic const& _fk, Secret const& _s)
 {
 	try
 	{
@@ -34,8 +43,39 @@ Message::Message(Envelope const& _e, Secret const& _s)
 		if (_s)
 			if (!decrypt(_s, &(_e.data()), b))
 				return;
-		if (populate(_s ? b : _e.data()))
+			else{}
-			m_to = KeyPair(_s).pub();
+		else
+		{
+			// public - need to get the key through combining with the topic/topicIndex we know.
+			unsigned topicIndex = 0;
+			Secret topicSecret;
+
+			// determine topicSecret/topicIndex from knowledge of the collapsed topics (which give the order) and our full-size filter topic.
+			Topic knownTopic = collapse(_fk);
+			for (unsigned ti = 0; ti < _fk.size() && !topicSecret; ++ti)
+				for (unsigned i = 0; i < _e.topic().size(); ++i)
+					if (_e.topic()[i] == knownTopic[ti])
+					{
+						topicSecret = _fk[ti];
+						topicIndex = i;
+						break;
+					}
+
+			if (_e.data().size() < _e.topic().size() * 32)
+				return;
+
+			// get key from decrypted topic key: just xor
+			h256 tk = h256(bytesConstRef(&(_e.data())).cropped(32 * topicIndex, 32));
+			bytesConstRef cipherText = bytesConstRef(&(_e.data())).cropped(32 * _e.topic().size());
+			cnote << "Decrypting(" << topicIndex << "): " << topicSecret << tk << (topicSecret ^ tk) << toHex(cipherText);
+			if (!decryptSym(topicSecret ^ tk, cipherText, b))
+				return;
+			cnote << "Got: " << toHex(b);
+		}
+
+		if (populate(b))
+			if (_s)
+				m_to = KeyPair(_s).pub();
 	}
 	catch (...)	// Invalid secret? TODO: replace ... with InvalidSecret
 	{
@@ -63,9 +103,10 @@ bool Message::populate(bytes const& _data)
 	return true;
 }
 
-Envelope Message::seal(Secret _from, Topic const& _topic, unsigned _ttl, unsigned _workToProve) const
+Envelope Message::seal(Secret _from, FullTopic const& _fullTopic, unsigned _ttl, unsigned _workToProve) const
 {
-	Envelope ret(time(0) + _ttl, _ttl, _topic);
+	Topic topic = collapse(_fullTopic);
+	Envelope ret(time(0) + _ttl, _ttl, topic);
 
 	bytes input(1 + m_payload.size());
 	input[0] = 0;
@@ -83,7 +124,25 @@ Envelope Message::seal(Secret _from, Topic const& _topic, unsigned _ttl, unsigne
 	if (m_to)
 		encrypt(m_to, &input, ret.m_data);
 	else
-		swap(ret.m_data, input);
+	{
+		// create the shared secret and encrypt
+		Secret s = Secret::random();
+		for (h256 const& t: _fullTopic)
+			ret.m_data += (t ^ s).asBytes();
+		bytes d;
+		encryptSym(s, &input, d);
+		ret.m_data += d;
+
+		for (unsigned i = 0; i < _fullTopic.size(); ++i)
+		{
+			bytes b;
+			h256 tk = h256(bytesConstRef(&(ret.m_data)).cropped(32 * i, 32));
+			bytesConstRef cipherText = bytesConstRef(&(ret.m_data)).cropped(32 * ret.topic().size());
+			cnote << "Test decrypting(" << i << "): " << _fullTopic[i] << tk << (_fullTopic[i] ^ tk) << toHex(cipherText);
+			assert(decryptSym(_fullTopic[i] ^ tk, cipherText, b));
+			cnote << "Got: " << toHex(b);
+		}
+	}
 
 	ret.proveWork(_workToProve);
 	return ret;
@@ -98,9 +157,9 @@ Envelope::Envelope(RLP const& _m)
 	m_nonce = _m[4].toInt<u256>();
 }
 
-Message Envelope::open(Secret const& _s) const
+Message Envelope::open(FullTopic const& _ft, Secret const& _s) const
 {
-	return Message(*this, _s);
+	return Message(*this, _ft, _s);
 }
 
 unsigned Envelope::workProved() const

libwhisper/Message.h

@@ -39,6 +39,16 @@ namespace shh
 
 class Message;
 
+static const unsigned Undefined = (unsigned)-1;
+
+struct FilterKey
+{
+	FilterKey() {}
+	FilterKey(unsigned _tI, Secret const& _k): topicIndex(_tI), key(_k) {}
+	unsigned topicIndex = Undefined;
+	Secret key;
+};
+
 enum IncludeNonce
 {
 	WithoutNonce = 0,
@@ -64,7 +74,7 @@ public:
 	Topic const& topic() const { return m_topic; }
 	bytes const& data() const { return m_data; }
 
-	Message open(Secret const& _s = Secret()) const;
+	Message open(FullTopic const& _ft, Secret const& _s = Secret()) const;
 
 	unsigned workProved() const;
 	void proveWork(unsigned _ms);
@@ -91,7 +101,7 @@ class Message
 {
 public:
 	Message() {}
-	Message(Envelope const& _e, Secret const& _s = Secret());
+	Message(Envelope const& _e, FullTopic const& _ft, Secret const& _s = Secret());
 	Message(bytes const& _payload): m_payload(_payload) {}
 	Message(bytesConstRef _payload): m_payload(_payload.toBytes()) {}
 	Message(bytes&& _payload) { std::swap(_payload, m_payload); }
@@ -108,11 +118,11 @@ public:
 	operator bool() const { return !!m_payload.size() || m_from || m_to; }
 
 	/// Turn this message into a ditributable Envelope.
-	Envelope seal(Secret _from, Topic const& _topic, unsigned _workToProve = 50, unsigned _ttl = 50) const;
+	Envelope seal(Secret _from, FullTopic const& _topic, unsigned _workToProve = 50, unsigned _ttl = 50) const;
 	// Overloads for skipping _from or specifying _to.
-	Envelope seal(Topic const& _topic, unsigned _ttl = 50, unsigned _workToProve = 50) const { return seal(Secret(), _topic, _workToProve, _ttl); }
+	Envelope seal(FullTopic const& _topic, unsigned _ttl = 50, unsigned _workToProve = 50) const { return seal(Secret(), _topic, _workToProve, _ttl); }
-	Envelope sealTo(Public _to, Topic const& _topic, unsigned _workToProve = 50, unsigned _ttl = 50) { m_to = _to; return seal(Secret(), _topic, _workToProve, _ttl); }
+	Envelope sealTo(Public _to, FullTopic const& _topic, unsigned _workToProve = 50, unsigned _ttl = 50) { m_to = _to; return seal(Secret(), _topic, _workToProve, _ttl); }
-	Envelope sealTo(Secret _from, Public _to, Topic const& _topic, unsigned _workToProve = 50, unsigned _ttl = 50) { m_to = _to; return seal(_from, _topic, _workToProve, _ttl); }
+	Envelope sealTo(Secret _from, Public _to, FullTopic const& _topic, unsigned _workToProve = 50, unsigned _ttl = 50) { m_to = _to; return seal(_from, _topic, _workToProve, _ttl); }
 
 private:
 	bool populate(bytes const& _data);

libwhisper/WhisperHost.cpp

@@ -56,7 +56,7 @@ void WhisperHost::streamMessage(h256 _m, RLPStream& _s) const
 
 void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p)
 {
-	cnote << "inject: " << _m.expiry() << _m.ttl() << _m.topic() << toHex(_m.data());
+	cnote << this << ": inject: " << _m.expiry() << _m.ttl() << _m.topic() << toHex(_m.data());
 
 	if (_m.expiry() <= time(0))
 		return;
@@ -104,14 +104,15 @@ unsigned WhisperHost::installWatchOnId(h256 _h)
 	return ret;
 }
 
-unsigned WhisperHost::installWatch(shh::TopicFilter const& _f)
+unsigned WhisperHost::installWatch(shh::FullTopic const& _ft)
 {
 	Guard l(m_filterLock);
 
-	h256 h = _f.sha3();
+	InstalledFilter f(_ft);
+	h256 h = f.filter.sha3();
 
 	if (!m_filters.count(h))
-		m_filters.insert(make_pair(h, _f));
+		m_filters.insert(make_pair(h, f));
 
 	return installWatchOnId(h);
 }

libwhisper/WhisperHost.h

@@ -39,6 +39,8 @@ namespace dev
 namespace shh
 {
 
+static const FullTopic EmptyFullTopic;
+
 class WhisperHost: public HostCapability<WhisperPeer>, public Interface, public Worker
 {
 	friend class WhisperPeer;
@@ -51,8 +53,8 @@ public:
 
 	virtual void inject(Envelope const& _e, WhisperPeer* _from = nullptr) override;
 
-	using Interface::installWatch;
+	virtual FullTopic const& fullTopic(unsigned _id) const { try { return m_filters.at(m_watches.at(_id).id).full; } catch (...) { return EmptyFullTopic; } }
-	virtual unsigned installWatch(TopicFilter const& _filter) override;
+	virtual unsigned installWatch(FullTopic const& _filter) override;
 	virtual unsigned installWatchOnId(h256 _filterId) override;
 	virtual void uninstallWatch(unsigned _watchId) override;
 	virtual h256s peekWatch(unsigned _watchId) const override { dev::Guard l(m_filterLock); try { return m_watches.at(_watchId).changes; } catch (...) { return h256s(); } }

libwhisper/WhisperPeer.cpp

@@ -106,7 +106,7 @@ void WhisperPeer::sendMessages()
 	}
 }
 
-void WhisperPeer::noteNewMessage(h256 _h, Message const& _m)
+void WhisperPeer::noteNewMessage(h256 _h, Envelope const& _m)
 {
 	Guard l(x_unseen);
 	m_unseen.insert(make_pair(rating(_m), _h));

libwhisper/WhisperPeer.h

@@ -63,8 +63,8 @@ private:
 
 	void sendMessages();
 
-	unsigned rating(Message const&) const { return 0; }	// TODO
+	unsigned rating(Envelope const&) const { return 0; }	// TODO
-	void noteNewMessage(h256 _h, Message const& _m);
+	void noteNewMessage(h256 _h, Envelope const& _m);
 
 	mutable dev::Mutex x_unseen;
 	std::multimap<unsigned, h256> m_unseen;	///< Rated according to what they want.

test/whisperTopic.cpp

@@ -46,16 +46,16 @@ BOOST_AUTO_TEST_CASE(topic)
 		auto wh = ph.registerCapability(new WhisperHost());
 		ph.start();
 
-		started = true;
-
 		/// Only interested in odd packets
 		auto w = wh->installWatch(BuildTopicMask("odd"));
 
-		for (int i = 0, last = 0; i < 200 && last < 81; ++i)
+		started = true;
+
+		for (int iterout = 0, last = 0; iterout < 200 && last < 81; ++iterout)
 		{
 			for (auto i: wh->checkWatch(w))
 			{
-				Message msg = wh->envelope(i).open();
+				Message msg = wh->envelope(i).open(wh->fullTopic(w));
 				last = RLP(msg.payload()).toInt<unsigned>();
 				cnote << "New message from:" << msg.from().abridged() << RLP(msg.payload()).toInt<unsigned>();
 				result += last;
@@ -116,7 +116,7 @@ BOOST_AUTO_TEST_CASE(forwarding)
 		{
 			for (auto i: wh->checkWatch(w))
 			{
-				Message msg = wh->envelope(i).open();
+				Message msg = wh->envelope(i).open(wh->fullTopic(w));
 				unsigned last = RLP(msg.payload()).toInt<unsigned>();
 				cnote << "New message from:" << msg.from().abridged() << RLP(msg.payload()).toInt<unsigned>();
 				result = last;
@@ -152,7 +152,7 @@ BOOST_AUTO_TEST_CASE(forwarding)
 		{
 			for (auto i: wh->checkWatch(w))
 			{
-				Message msg = wh->envelope(i).open();
+				Message msg = wh->envelope(i).open(wh->fullTopic(w));
 				cnote << "New message from:" << msg.from().abridged() << RLP(msg.payload()).toInt<unsigned>();
 			}
 			this_thread::sleep_for(chrono::milliseconds(50));
@@ -215,7 +215,7 @@ BOOST_AUTO_TEST_CASE(asyncforwarding)
 		{
 			for (auto i: wh->checkWatch(w))
 			{
-				Message msg = wh->envelope(i).open();
+				Message msg = wh->envelope(i).open(wh->fullTopic(w));
 				cnote << "New message from:" << msg.from().abridged() << RLP(msg.payload()).toInt<unsigned>();
 			}
 			this_thread::sleep_for(chrono::milliseconds(50));
@@ -255,7 +255,7 @@ BOOST_AUTO_TEST_CASE(asyncforwarding)
 		{
 			for (auto i: wh->checkWatch(w))
 			{
-				Message msg = wh->envelope(i).open();
+				Message msg = wh->envelope(i).open(wh->fullTopic(w));
 				unsigned last = RLP(msg.payload()).toInt<unsigned>();
 				cnote << "New message from:" << msg.from().abridged() << RLP(msg.payload()).toInt<unsigned>();
 				result = last;