From e82f8db2de4a3fa90c4710dfc40adc6a9114f342 Mon Sep 17 00:00:00 2001 From: subtly Date: Sat, 20 Jun 2015 07:24:03 -0400 Subject: [PATCH 01/59] Prep for #2179. Abstract parsing of frame header from Session into struct. Catch unhandled exceptions thrown by ASIO. --- libp2p/Common.h | 11 +- libp2p/Host.cpp | 14 +- libp2p/RLPXFrameCoder.cpp | 12 ++ libp2p/RLPXFrameCoder.h | 14 ++ libp2p/Session.cpp | 337 +++++++++++++++++++------------------- libp2p/Session.h | 10 +- test/libp2p/net.cpp | 10 +- 7 files changed, 225 insertions(+), 183 deletions(-) diff --git a/libp2p/Common.h b/libp2p/Common.h index 4a1b64b70..445ba0cca 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -149,14 +149,15 @@ using CapDescs = std::vector; */ struct PeerSessionInfo { - NodeId id; - std::string clientVersion; - std::string host; - unsigned short port; + NodeId const id; + std::string const clientVersion; + std::string const host; + unsigned short const port; std::chrono::steady_clock::duration lastPing; - std::set caps; + std::set const caps; unsigned socketId; std::map notes; + unsigned const protocolVersion; }; using PeerSessionInfos = std::vector; diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index feb116c4a..c9c2743ee 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -254,7 +254,7 @@ void Host::startPeerSession(Public const& _id, RLP const& _rlp, RLPXFrameCoder* clog(NetMessageSummary) << "Hello: " << clientVersion << "V[" << protocolVersion << "]" << _id << showbase << capslog.str() << dec << listenPort; // create session so disconnects are managed - auto ps = make_shared(this, _io, _s, p, PeerSessionInfo({_id, clientVersion, p->endpoint.address.to_string(), listenPort, chrono::steady_clock::duration(), _rlp[2].toSet(), 0, map()})); + auto ps = make_shared(this, _io, _s, p, PeerSessionInfo({_id, clientVersion, p->endpoint.address.to_string(), listenPort, chrono::steady_clock::duration(), _rlp[2].toSet(), 0, map(), protocolVersion})); if (protocolVersion < dev::p2p::c_protocolVersion - 1) { ps->disconnect(IncompatibleProtocol); @@ -724,8 +724,16 @@ void Host::startedWorking() void Host::doWork() { - if (m_run) - m_ioService.run(); + try + { + if (m_run) + m_ioService.run(); + } + catch (std::exception const& _e) + { + clog(NetP2PWarn) << "Exception in Network Thread:" << _e.what(); + clog(NetP2PWarn) << "Network Restart is Recommended."; + } } void Host::keepAlivePeers() diff --git a/libp2p/RLPXFrameCoder.cpp b/libp2p/RLPXFrameCoder.cpp index c4bb46814..193c45511 100644 --- a/libp2p/RLPXFrameCoder.cpp +++ b/libp2p/RLPXFrameCoder.cpp @@ -29,6 +29,18 @@ using namespace dev; using namespace dev::p2p; using namespace CryptoPP; +RLPXFrameInfo::RLPXFrameInfo(bytesConstRef _header) +{ + length = (_header[0] * 256 + _header[1]) * 256 + _header[2]; + padding = ((16 - (length % 16)) % 16); + RLP header(_header.cropped(3), RLP::ThrowOnFail | RLP::FailIfTooSmall); + auto itemCount = header.itemCount(); + protocolId = header[0].toInt(); + hasSequence = itemCount > 1; + sequenceId = hasSequence ? header[1].toInt() : 0; + totalLength = itemCount == 3 ? header[2].toInt() : 0; +} + RLPXFrameCoder::RLPXFrameCoder(RLPXHandshake const& _init) { // we need: diff --git a/libp2p/RLPXFrameCoder.h b/libp2p/RLPXFrameCoder.h index 7c5eedbff..3964326ff 100644 --- a/libp2p/RLPXFrameCoder.h +++ b/libp2p/RLPXFrameCoder.h @@ -32,7 +32,21 @@ namespace dev { namespace p2p { + +struct RLPXFrameInfo +{ + RLPXFrameInfo() = default; + /// Constructor. frame-size || protocol-type, [sequence-id[, total-packet-size]] + RLPXFrameInfo(bytesConstRef _frameHeader); + uint32_t length = 0; ///< Max: 2**24 + uint8_t padding = 0; + uint16_t protocolId = 0; + bool hasSequence = false; + uint16_t sequenceId = 0; + uint32_t totalLength = 0; +}; + class RLPXHandshake; /** diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index c3f0f2e35..95fd4a3d6 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -27,7 +27,6 @@ #include #include #include -#include "RLPxHandshake.h" #include "Host.h" #include "Capability.h" using namespace std; @@ -156,111 +155,25 @@ void Session::serviceNodesRequest() addNote("peers", "done"); } -bool Session::interpret(PacketType _t, RLP const& _r) +bool Session::frameReceived(uint16_t _capId, PacketType _t, RLP const& _r) { m_lastReceived = chrono::steady_clock::now(); - clog(NetRight) << _t << _r; try // Generic try-catch block designed to capture RLP format errors - TODO: give decent diagnostics, make a bit more specific over what is caught. { - switch (_t) - { - case DisconnectPacket: - { - string reason = "Unspecified"; - auto r = (DisconnectReason)_r[0].toInt(); - if (!_r[0].isInt()) - drop(BadProtocol); - else - { - reason = reasonOf(r); - clog(NetMessageSummary) << "Disconnect (reason: " << reason << ")"; - drop(DisconnectRequested); - } - break; - } - case PingPacket: - { - clog(NetTriviaSummary) << "Ping"; - RLPStream s; - sealAndSend(prep(s, PongPacket)); - break; - } - case PongPacket: - m_info.lastPing = std::chrono::steady_clock::now() - m_ping; - clog(NetTriviaSummary) << "Latency: " << chrono::duration_cast(m_info.lastPing).count() << " ms"; - break; - case GetPeersPacket: - // Disabled for interop testing. - // GetPeers/PeersPacket will be modified to only exchange new nodes which it's peers are interested in. - break; - - clog(NetTriviaSummary) << "GetPeers"; - m_theyRequestedNodes = true; - serviceNodesRequest(); - break; - case PeersPacket: - // Disabled for interop testing. - // GetPeers/PeersPacket will be modified to only exchange new nodes which it's peers are interested in. - break; - - clog(NetTriviaSummary) << "Peers (" << dec << (_r.itemCount() - 1) << " entries)"; - m_weRequestedNodes = false; - for (unsigned i = 0; i < _r.itemCount(); ++i) - { - bi::address peerAddress; - if (_r[i][0].size() == 16) - peerAddress = bi::address_v6(_r[i][0].toHash>().asArray()); - else if (_r[i][0].size() == 4) - peerAddress = bi::address_v4(_r[i][0].toHash>().asArray()); - else - { - cwarn << "Received bad peer packet:" << _r; - disconnect(BadProtocol); - return true; - } - auto ep = bi::tcp::endpoint(peerAddress, _r[i][1].toInt()); - NodeId id = _r[i][2].toHash(); - - clog(NetAllDetail) << "Checking: " << ep << "(" << id << ")"; - - if (!isPublicAddress(peerAddress)) - goto CONTINUE; // Private address. Ignore. - - if (!id) - goto LAMEPEER; // Null identity. Ignore. - - if (m_server->id() == id) - goto LAMEPEER; // Just our info - we already have that. - - if (id == this->id()) - goto LAMEPEER; // Just their info - we already have that. - - if (!ep.port()) - goto LAMEPEER; // Zero port? Don't think so. - - if (ep.port() >= /*49152*/32768) - goto LAMEPEER; // Private port according to IANA. - - // OK passed all our checks. Assume it's good. - addRating(1000); - m_server->addNode(id, NodeIPEndpoint(ep.address(), ep.port(), ep.port())); - clog(NetTriviaDetail) << "New peer: " << ep << "(" << id << ")"; - CONTINUE:; - LAMEPEER:; - } - break; - default: + // v4 frame headers are useless, offset packet type used + // v5 protocol type is in header, packet type not offset + if (_capId == 0 && _t < UserPacket) + return interpret(_t, _r); + if (m_info.protocolVersion >= 5) + for (auto const& i: m_capabilities) + if (_capId == (uint16_t)i.first.second) + return i.second->m_enabled ? i.second->interpret(_t, _r) : true; + if (m_info.protocolVersion <= 4) for (auto const& i: m_capabilities) if (_t >= (int)i.second->m_idOffset && _t - i.second->m_idOffset < i.second->hostCapability()->messageCount()) - { - if (i.second->m_enabled) - return i.second->interpret(_t - i.second->m_idOffset, _r); - else - return true; - } - return false; - } + return i.second->m_enabled ? i.second->interpret(_t - i.second->m_idOffset, _r) : true; + return false; } catch (std::exception const& _e) { @@ -271,6 +184,101 @@ bool Session::interpret(PacketType _t, RLP const& _r) return true; } +bool Session::interpret(PacketType _t, RLP const& _r) +{ + switch (_t) + { + case DisconnectPacket: + { + string reason = "Unspecified"; + auto r = (DisconnectReason)_r[0].toInt(); + if (!_r[0].isInt()) + drop(BadProtocol); + else + { + reason = reasonOf(r); + clog(NetMessageSummary) << "Disconnect (reason: " << reason << ")"; + drop(DisconnectRequested); + } + break; + } + case PingPacket: + { + clog(NetTriviaSummary) << "Ping"; + RLPStream s; + sealAndSend(prep(s, PongPacket)); + break; + } + case PongPacket: + m_info.lastPing = std::chrono::steady_clock::now() - m_ping; + clog(NetTriviaSummary) << "Latency: " << chrono::duration_cast(m_info.lastPing).count() << " ms"; + break; + case GetPeersPacket: + // Disabled for interop testing. + // GetPeers/PeersPacket will be modified to only exchange new nodes which it's peers are interested in. + break; + + clog(NetTriviaSummary) << "GetPeers"; + m_theyRequestedNodes = true; + serviceNodesRequest(); + break; + case PeersPacket: + // Disabled for interop testing. + // GetPeers/PeersPacket will be modified to only exchange new nodes which it's peers are interested in. + break; + + clog(NetTriviaSummary) << "Peers (" << dec << (_r.itemCount() - 1) << " entries)"; + m_weRequestedNodes = false; + for (unsigned i = 0; i < _r.itemCount(); ++i) + { + bi::address peerAddress; + if (_r[i][0].size() == 16) + peerAddress = bi::address_v6(_r[i][0].toHash>().asArray()); + else if (_r[i][0].size() == 4) + peerAddress = bi::address_v4(_r[i][0].toHash>().asArray()); + else + { + cwarn << "Received bad peer packet:" << _r; + disconnect(BadProtocol); + return true; + } + auto ep = bi::tcp::endpoint(peerAddress, _r[i][1].toInt()); + NodeId id = _r[i][2].toHash(); + + clog(NetAllDetail) << "Checking: " << ep << "(" << id << ")"; + + if (!isPublicAddress(peerAddress)) + goto CONTINUE; // Private address. Ignore. + + if (!id) + goto LAMEPEER; // Null identity. Ignore. + + if (m_server->id() == id) + goto LAMEPEER; // Just our info - we already have that. + + if (id == this->id()) + goto LAMEPEER; // Just their info - we already have that. + + if (!ep.port()) + goto LAMEPEER; // Zero port? Don't think so. + + if (ep.port() >= /*49152*/32768) + goto LAMEPEER; // Private port according to IANA. + + // OK passed all our checks. Assume it's good. + addRating(1000); + m_server->addNode(id, NodeIPEndpoint(ep.address(), ep.port(), ep.port())); + clog(NetTriviaDetail) << "New peer: " << ep << "(" << id << ")"; + CONTINUE:; + LAMEPEER:; + } + break; + default: + return false; + } + return true; +} + void Session::ping() { RLPStream s; @@ -292,12 +300,9 @@ void Session::sealAndSend(RLPStream& _s) bool Session::checkPacket(bytesConstRef _msg) { - if (_msg.size() < 2) + if (_msg[0] > 0x7f || _msg.size() < 2) return false; - if (_msg[0] > 0x7f) - return false; - RLP r(_msg.cropped(1)); - if (r.actualSize() + 1 != _msg.size()) + if (RLP(_msg.cropped(1)).actualSize() + 1 != _msg.size()) return false; return true; } @@ -413,82 +418,78 @@ void Session::doRead() { ThreadContext tc(info().id.abridged()); ThreadContext tc2(info().clientVersion); - if (ec && ec.category() != boost::asio::error::get_misc_category() && ec.value() != boost::asio::error::eof) + if (!checkRead(h256::size, ec, length)) + return; + else if (!m_io->authAndDecryptHeader(bytesRef(m_data.data(), length))) { - clog(NetWarn) << "Error reading: " << ec.message(); - drop(TCPError); + clog(NetWarn) << "header decrypt failed"; + drop(BadProtocol); // todo: better error + return; + } + + RLPXFrameInfo header; + try + { + header = RLPXFrameInfo(bytesConstRef(m_data.data(), length)); } - else if (ec && length == 0) + catch (std::exception const& _e) + { + clog(NetWarn) << "Exception decoding frame header RLP:" << bytesConstRef(m_data.data(), h128::size).cropped(3); + drop(BadProtocol); return; - else + } + + /// read padded frame and mac + auto tlen = header.length + header.padding + h128::size; + ba::async_read(m_socket->ref(), boost::asio::buffer(m_data, tlen), [this, self, header, tlen](boost::system::error_code ec, std::size_t length) { - /// authenticate and decrypt header - bytesRef header(m_data.data(), h256::size); - if (!m_io->authAndDecryptHeader(header)) + ThreadContext tc(info().id.abridged()); + ThreadContext tc2(info().clientVersion); + if (!checkRead(tlen, ec, length)) + return; + else if (!m_io->authAndDecryptFrame(bytesRef(m_data.data(), tlen))) { - clog(NetWarn) << "header decrypt failed"; + clog(NetWarn) << "frame decrypt failed"; drop(BadProtocol); // todo: better error return; } - /// check frame size - uint32_t frameSize = (m_data[0] * 256 + m_data[1]) * 256 + m_data[2]; - if (frameSize >= (uint32_t)1 << 24) + bytesConstRef frame(m_data.data(), header.length); + if (!checkPacket(frame)) { - clog(NetWarn) << "frame size too large"; - drop(BadProtocol); + cerr << "Received " << frame.size() << ": " << toHex(frame) << endl; + clog(NetWarn) << "INVALID MESSAGE RECEIVED"; + disconnect(BadProtocol); return; } - - /// rlp of header has protocol-type, sequence-id[, total-packet-size] - bytes headerRLP(13); - bytesConstRef(m_data.data(), h128::size).cropped(3).copyTo(&headerRLP); - - /// read padded frame and mac - auto tlen = frameSize + ((16 - (frameSize % 16)) % 16) + h128::size; - ba::async_read(m_socket->ref(), boost::asio::buffer(m_data, tlen), [this, self, headerRLP, frameSize, tlen](boost::system::error_code ec, std::size_t length) + else { - ThreadContext tc(info().id.abridged()); - ThreadContext tc2(info().clientVersion); - if (ec && ec.category() != boost::asio::error::get_misc_category() && ec.value() != boost::asio::error::eof) - { - clog(NetWarn) << "Error reading: " << ec.message(); - drop(TCPError); - } - else if (ec && length < tlen) - { - clog(NetWarn) << "Error reading - Abrupt peer disconnect: " << ec.message(); - repMan().noteRude(*this); - drop(TCPError); - return; - } - else - { - if (!m_io->authAndDecryptFrame(bytesRef(m_data.data(), tlen))) - { - clog(NetWarn) << "frame decrypt failed"; - drop(BadProtocol); // todo: better error - return; - } - - bytesConstRef frame(m_data.data(), frameSize); - if (!checkPacket(frame)) - { - cerr << "Received " << frame.size() << ": " << toHex(frame) << endl; - clog(NetWarn) << "INVALID MESSAGE RECEIVED"; - disconnect(BadProtocol); - return; - } - else - { - auto packetType = (PacketType)RLP(frame.cropped(0, 1)).toInt(); - RLP r(frame.cropped(1)); - if (!interpret(packetType, r)) - clog(NetWarn) << "Couldn't interpret packet." << RLP(r); - } - doRead(); - } - }); - } + auto packetType = (PacketType)RLP(frame.cropped(0, 1)).toInt(); + RLP r(frame.cropped(1)); + if (!frameReceived(header.protocolId, packetType, r)) + clog(NetWarn) << "Couldn't interpret packet." << RLP(r); + } + doRead(); + }); }); } + +bool Session::checkRead(std::size_t _expected, boost::system::error_code _ec, std::size_t _length) +{ + if (_ec && _ec.category() != boost::asio::error::get_misc_category() && _ec.value() != boost::asio::error::eof) + { + clog(NetConnect) << "Error reading: " << _ec.message(); + drop(TCPError); + return false; + } + else if (_ec && _length < _expected) + { + clog(NetWarn) << "Error reading - Abrupt peer disconnect: " << _ec.message(); + repMan().noteRude(*this); + drop(TCPError); + return false; + } + // If this fails then there's an unhandled asio error + assert(_expected == _length); + return true; +} diff --git a/libp2p/Session.h b/libp2p/Session.h index 6b45fe381..ec079cd1d 100644 --- a/libp2p/Session.h +++ b/libp2p/Session.h @@ -96,13 +96,19 @@ private: /// Perform a read on the socket. void doRead(); + + /// Check error code after reading and drop peer if error code. + bool checkRead(std::size_t _expected, boost::system::error_code _ec, std::size_t _length); /// Perform a single round of the write operation. This could end up calling itself asynchronously. void write(); - /// Interpret an incoming message. - bool interpret(PacketType _t, RLP const& _r); + /// Deliver RLPX packet to Session or Capability for interpretation. + bool frameReceived(uint16_t _capId, PacketType _t, RLP const& _r); + /// Interpret an incoming Session packet. + bool interpret(PacketType _t, RLP const& _r); + /// @returns true iff the _msg forms a valid message for sending or receiving on the network. static bool checkPacket(bytesConstRef _msg); diff --git a/test/libp2p/net.cpp b/test/libp2p/net.cpp index 8c652b479..1e3e2e15c 100644 --- a/test/libp2p/net.cpp +++ b/test/libp2p/net.cpp @@ -314,23 +314,23 @@ BOOST_AUTO_TEST_CASE(kademlia) node.nodeTable->discover(); // ideally, joining with empty node table logs warning we can check for node.setup(); node.populate(); - clog << "NodeTable:\n" << *node.nodeTable.get() << endl; +// clog << "NodeTable:\n" << *node.nodeTable.get() << endl; node.populateAll(); - clog << "NodeTable:\n" << *node.nodeTable.get() << endl; +// clog << "NodeTable:\n" << *node.nodeTable.get() << endl; auto nodes = node.nodeTable->nodes(); nodes.sort(); node.nodeTable->reset(); - clog << "NodeTable:\n" << *node.nodeTable.get() << endl; +// clog << "NodeTable:\n" << *node.nodeTable.get() << endl; node.populate(1); - clog << "NodeTable:\n" << *node.nodeTable.get() << endl; +// clog << "NodeTable:\n" << *node.nodeTable.get() << endl; node.nodeTable->discover(); this_thread::sleep_for(chrono::milliseconds(2000)); - clog << "NodeTable:\n" << *node.nodeTable.get() << endl; +// clog << "NodeTable:\n" << *node.nodeTable.get() << endl; BOOST_REQUIRE_EQUAL(node.nodeTable->count(), 8); From 78467d3ae0af55b3c8c96a9e3a779ae21bc2d0c3 Mon Sep 17 00:00:00 2001 From: subtly Date: Sat, 20 Jun 2015 07:31:20 -0400 Subject: [PATCH 02/59] small rename --- libp2p/Session.cpp | 4 ++-- libp2p/Session.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index 95fd4a3d6..f24377fb2 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -155,7 +155,7 @@ void Session::serviceNodesRequest() addNote("peers", "done"); } -bool Session::frameReceived(uint16_t _capId, PacketType _t, RLP const& _r) +bool Session::readPacket(uint16_t _capId, PacketType _t, RLP const& _r) { m_lastReceived = chrono::steady_clock::now(); clog(NetRight) << _t << _r; @@ -466,7 +466,7 @@ void Session::doRead() { auto packetType = (PacketType)RLP(frame.cropped(0, 1)).toInt(); RLP r(frame.cropped(1)); - if (!frameReceived(header.protocolId, packetType, r)) + if (!readPacket(header.protocolId, packetType, r)) clog(NetWarn) << "Couldn't interpret packet." << RLP(r); } doRead(); diff --git a/libp2p/Session.h b/libp2p/Session.h index ec079cd1d..ba46c5a16 100644 --- a/libp2p/Session.h +++ b/libp2p/Session.h @@ -104,7 +104,7 @@ private: void write(); /// Deliver RLPX packet to Session or Capability for interpretation. - bool frameReceived(uint16_t _capId, PacketType _t, RLP const& _r); + bool readPacket(uint16_t _capId, PacketType _t, RLP const& _r); /// Interpret an incoming Session packet. bool interpret(PacketType _t, RLP const& _r); From 9c51074e0b66dd66f2dd2cfcafee09c0d46a92ad Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Sun, 21 Jun 2015 20:19:15 +0200 Subject: [PATCH 03/59] updates --- libwhisper/Common.h | 4 ++-- libwhisper/WhisperPeer.h | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/libwhisper/Common.h b/libwhisper/Common.h index 71435603e..a7b3c7800 100644 --- a/libwhisper/Common.h +++ b/libwhisper/Common.h @@ -54,8 +54,8 @@ enum WhisperPacket { StatusPacket = 0, MessagesPacket, - AddFilterPacket, - RemoveFilterPacket, + AddTopicFilterPacket, + RemoveTopicFilterPacket, PacketCount }; diff --git a/libwhisper/WhisperPeer.h b/libwhisper/WhisperPeer.h index 5cb124568..ddaf70aca 100644 --- a/libwhisper/WhisperPeer.h +++ b/libwhisper/WhisperPeer.h @@ -60,6 +60,8 @@ public: WhisperHost* host() const; + void setBloom(FixedHash const& _b) { m_bloom = _b; } + private: virtual bool interpret(unsigned _id, RLP const&) override; @@ -72,6 +74,8 @@ private: std::multimap m_unseen; ///< Rated according to what they want. std::chrono::system_clock::time_point m_timer = std::chrono::system_clock::now(); + + FixedHash m_bloom; }; } From 43a43b03e39bd0bc0520a913eef66b715596d2d2 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Mon, 22 Jun 2015 14:15:32 +0200 Subject: [PATCH 04/59] bloom filter exchange protocol --- libwhisper/Common.h | 3 +-- libwhisper/WhisperHost.cpp | 15 ++++++++++++++- libwhisper/WhisperHost.h | 2 +- libwhisper/WhisperPeer.cpp | 13 +++++++++++++ libwhisper/WhisperPeer.h | 9 ++++++--- 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/libwhisper/Common.h b/libwhisper/Common.h index a7b3c7800..fb48f2df6 100644 --- a/libwhisper/Common.h +++ b/libwhisper/Common.h @@ -54,8 +54,7 @@ enum WhisperPacket { StatusPacket = 0, MessagesPacket, - AddTopicFilterPacket, - RemoveTopicFilterPacket, + UpdateTopicFilterPacket, PacketCount }; diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index d6759df6f..2cccb0151 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -85,6 +85,15 @@ void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p) } } +void WhisperHost::advertizeTopicsOfInterest() +{ + for (auto i: peerSessions()) + { + auto w = i.first->cap().get(); + w->advertizeTopicsOfInterest(m_bloom); + } +} + void WhisperHost::noteChanged(h256 _messageHash, h256 _filter) { for (auto& i: m_watches) @@ -114,6 +123,8 @@ unsigned WhisperHost::installWatch(shh::Topics const& _t) m_filters.insert(make_pair(h, f)); m_bloom.addRaw(f.filter.exportBloom()); + advertizeTopicsOfInterest(); + return installWatchOnId(h); } @@ -153,9 +164,11 @@ void WhisperHost::uninstallWatch(unsigned _i) auto fit = m_filters.find(id); if (fit != m_filters.end()) { - m_bloom.removeRaw(fit->second.filter.exportBloom()); if (!--fit->second.refCount) m_filters.erase(fit); + + m_bloom.removeRaw(fit->second.filter.exportBloom()); + advertizeTopicsOfInterest(); } } diff --git a/libwhisper/WhisperHost.h b/libwhisper/WhisperHost.h index 46c41f3a2..7020e3f63 100644 --- a/libwhisper/WhisperHost.h +++ b/libwhisper/WhisperHost.h @@ -77,8 +77,8 @@ private: virtual void onStopping() override { stopWorking(); } void streamMessage(h256 _m, RLPStream& _s) const; - void noteChanged(h256 _messageHash, h256 _filter); + void advertizeTopicsOfInterest(); mutable dev::SharedMutex x_messages; std::map m_messages; diff --git a/libwhisper/WhisperPeer.cpp b/libwhisper/WhisperPeer.cpp index 0be59592d..144de3ec6 100644 --- a/libwhisper/WhisperPeer.cpp +++ b/libwhisper/WhisperPeer.cpp @@ -70,6 +70,11 @@ bool WhisperPeer::interpret(unsigned _id, RLP const& _r) host()->inject(Envelope(i), this); break; } + case UpdateTopicFilterPacket: + { + m_bloom = (FixedHash)_r; + break; + } default: return false; } @@ -104,3 +109,11 @@ void WhisperPeer::noteNewMessage(h256 _h, Envelope const& _m) Guard l(x_unseen); m_unseen.insert(make_pair(rating(_m), _h)); } + +void WhisperPeer::advertizeTopicsOfInterest(FixedHash const& _bloom) +{ + RLPStream s; + prep(s, UpdateTopicFilterPacket, 1); + s << _bloom; + sealAndSend(s); +} diff --git a/libwhisper/WhisperPeer.h b/libwhisper/WhisperPeer.h index ddaf70aca..f934abb2a 100644 --- a/libwhisper/WhisperPeer.h +++ b/libwhisper/WhisperPeer.h @@ -60,7 +60,10 @@ public: WhisperHost* host() const; - void setBloom(FixedHash const& _b) { m_bloom = _b; } + FixedHash const& bloom() const { return m_bloom; } + + /// called by the host, sends our bloom filter to remote peer + void advertizeTopicsOfInterest(FixedHash const& _bloom); private: virtual bool interpret(unsigned _id, RLP const&) override; @@ -74,8 +77,8 @@ private: std::multimap m_unseen; ///< Rated according to what they want. std::chrono::system_clock::time_point m_timer = std::chrono::system_clock::now(); - - FixedHash m_bloom; + + FixedHash m_bloom; ///< Peer's topics of interest }; } From d254972f405be53da7f945f46a7baaf5ed9c00b1 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Mon, 22 Jun 2015 21:42:08 +0200 Subject: [PATCH 05/59] bloom filter exchage protocol --- libwhisper/WhisperHost.h | 10 ++--- libwhisper/WhisperPeer.cpp | 21 +++++---- libwhisper/WhisperPeer.h | 20 +++------ test/libwhisper/whisperTopic.cpp | 74 ++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 30 deletions(-) diff --git a/libwhisper/WhisperHost.h b/libwhisper/WhisperHost.h index 7020e3f63..b5e321dea 100644 --- a/libwhisper/WhisperHost.h +++ b/libwhisper/WhisperHost.h @@ -50,11 +50,12 @@ class WhisperHost: public HostCapability, public Interface, public public: WhisperHost(); virtual ~WhisperHost(); - unsigned protocolVersion() const { return 2; } + void cleanup(); + std::map all() const { ReadGuard l(x_messages); return m_messages; } + FixedHash const& bloom() const { return m_bloom; } virtual void inject(Envelope const& _e, WhisperPeer* _from = nullptr) override; - virtual Topics const& fullTopics(unsigned _id) const override { try { return m_filters.at(m_watches.at(_id).id).full; } catch (...) { return EmptyTopics; } } virtual unsigned installWatch(Topics const& _filter) override; virtual unsigned installWatchOnId(h256 _filterId) override; @@ -62,13 +63,8 @@ public: virtual h256s peekWatch(unsigned _watchId) const override { dev::Guard l(m_filterLock); try { return m_watches.at(_watchId).changes; } catch (...) { return h256s(); } } virtual h256s checkWatch(unsigned _watchId) override { cleanup(); dev::Guard l(m_filterLock); h256s ret; try { ret = m_watches.at(_watchId).changes; m_watches.at(_watchId).changes.clear(); } catch (...) {} return ret; } virtual h256s watchMessages(unsigned _watchId) override; /// returns IDs of messages, which match specific watch criteria - virtual Envelope envelope(h256 _m) const override { try { dev::ReadGuard l(x_messages); return m_messages.at(_m); } catch (...) { return Envelope(); } } - std::map all() const { ReadGuard l(x_messages); return m_messages; } - - void cleanup(); - protected: virtual void doWork() override; diff --git a/libwhisper/WhisperPeer.cpp b/libwhisper/WhisperPeer.cpp index 144de3ec6..826b7842d 100644 --- a/libwhisper/WhisperPeer.cpp +++ b/libwhisper/WhisperPeer.cpp @@ -19,11 +19,10 @@ * @date 2014 */ -#include "WhisperPeer.h" - #include #include #include "WhisperHost.h" + using namespace std; using namespace dev; using namespace dev::p2p; @@ -32,7 +31,10 @@ using namespace dev::shh; WhisperPeer::WhisperPeer(std::shared_ptr _s, HostCapabilityFace* _h, unsigned _i, CapDesc const&): Capability(_s, _h, _i) { RLPStream s; - sealAndSend(prep(s, StatusPacket, 1) << version()); + prep(s, StatusPacket, 2); + s << version(); + s << host()->bloom(); + sealAndSend(s); } WhisperPeer::~WhisperPeer() @@ -57,6 +59,9 @@ bool WhisperPeer::interpret(unsigned _id, RLP const& _r) if (protocolVersion != version()) disable("Invalid protocol version."); + if (_r.itemCount() > 1) // for backwards compatibility + m_bloom = (FixedHash)_r[1]; + for (auto const& m: host()->all()) m_unseen.insert(make_pair(0, m.first)); @@ -64,15 +69,15 @@ bool WhisperPeer::interpret(unsigned _id, RLP const& _r) sendMessages(); break; } - case MessagesPacket: + case UpdateTopicFilterPacket: { - for (auto i: _r) - host()->inject(Envelope(i), this); + m_bloom = (FixedHash)_r[0]; break; } - case UpdateTopicFilterPacket: + case MessagesPacket: { - m_bloom = (FixedHash)_r; + for (auto i: _r) + host()->inject(Envelope(i), this); break; } default: diff --git a/libwhisper/WhisperPeer.h b/libwhisper/WhisperPeer.h index f934abb2a..95b4a1952 100644 --- a/libwhisper/WhisperPeer.h +++ b/libwhisper/WhisperPeer.h @@ -44,8 +44,6 @@ using p2p::HostCapability; using p2p::Capability; using p2p::CapDesc; -/** - */ class WhisperPeer: public Capability { friend class WhisperHost; @@ -53,31 +51,23 @@ class WhisperPeer: public Capability public: WhisperPeer(std::shared_ptr _s, HostCapabilityFace* _h, unsigned _i, CapDesc const& _cap); virtual ~WhisperPeer(); - + WhisperHost* host() const; static std::string name() { return "shh"; } static u256 version() { return 2; } static unsigned messageCount() { return PacketCount; } - - WhisperHost* host() const; - - FixedHash const& bloom() const { return m_bloom; } - - /// called by the host, sends our bloom filter to remote peer - void advertizeTopicsOfInterest(FixedHash const& _bloom); + FixedHash const& bloom() const { return m_bloom; } + void advertizeTopicsOfInterest(FixedHash const& _bloom); ///< sends our bloom filter to remote peer + FixedHash const& bloom() { return m_bloom; } private: virtual bool interpret(unsigned _id, RLP const&) override; - void sendMessages(); - unsigned rating(Envelope const&) const { return 0; } // TODO void noteNewMessage(h256 _h, Envelope const& _m); mutable dev::Mutex x_unseen; std::multimap m_unseen; ///< Rated according to what they want. - - std::chrono::system_clock::time_point m_timer = std::chrono::system_clock::now(); - + std::chrono::system_clock::time_point m_timer = std::chrono::system_clock::now(); FixedHash m_bloom; ///< Peer's topics of interest }; diff --git a/test/libwhisper/whisperTopic.cpp b/test/libwhisper/whisperTopic.cpp index f09705ccb..56b1eea02 100644 --- a/test/libwhisper/whisperTopic.cpp +++ b/test/libwhisper/whisperTopic.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -301,4 +302,77 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) BOOST_REQUIRE_EQUAL(result, 1); } +BOOST_AUTO_TEST_CASE(topicAdvertising) +{ + if (test::Options::get().nonetwork) + return; + + cnote << "Testing Topic Advertising..."; + VerbosityHolder setTemporaryLevel(2); + + Host host1("first", NetworkPreferences("127.0.0.1", 30303, false)); + host1.setIdealPeerCount(1); + auto whost1 = host1.registerCapability(new WhisperHost()); + host1.start(); + while (!host1.haveNetwork()) + this_thread::sleep_for(chrono::milliseconds(10)); + + Host host2("second", NetworkPreferences("127.0.0.1", 30305, false)); + host2.setIdealPeerCount(1); + auto whost2 = host2.registerCapability(new WhisperHost()); + auto watch2 = whost2->installWatch(BuildTopicMask("test2")); + + host2.start(); + while (!host2.haveNetwork()) + this_thread::sleep_for(chrono::milliseconds(10)); + + host1.addNode(host2.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), 30305, 30305)); + while (!host1.haveNetwork()) + this_thread::sleep_for(chrono::milliseconds(10)); + + while (!host1.peerCount()) + this_thread::sleep_for(chrono::milliseconds(10)); + + while (!host2.peerCount()) + this_thread::sleep_for(chrono::milliseconds(10)); + + while (!whost1->peerSessions().size()) + this_thread::sleep_for(chrono::milliseconds(10)); + + for (int i = 0; i < 200; ++i) + { + this_thread::sleep_for(chrono::milliseconds(10)); + auto sessions1 = whost1->peerSessions(); + size_t x = sessions1.size(); + BOOST_REQUIRE(x > 0); + if (whost1->peerSessions()[x-1].first->cap()->bloom()) + break; + } + + BOOST_REQUIRE(whost1->peerSessions().size()); + FixedHash bf1 = whost1->peerSessions().back().first->cap()->bloom(); + FixedHash bf2 = whost2->bloom(); + BOOST_REQUIRE_EQUAL(bf1, bf2); + BOOST_REQUIRE(bf1); + BOOST_REQUIRE(!whost1->bloom()); + + auto watch1 = whost1->installWatch(BuildTopicMask("test1")); + + for (int i = 0; i < 300; ++i) + { + this_thread::sleep_for(chrono::milliseconds(10)); + if (whost2->peerSessions().back().first->cap()->bloom()) + break; + } + + auto sessions2 = whost2->peerSessions(); + BOOST_REQUIRE(sessions2.size()); + BOOST_REQUIRE_EQUAL(sessions2.back().second->id, host1.id()); + + bf2 = sessions2.back().first->cap()->bloom(); + bf1 = whost1->bloom(); + BOOST_REQUIRE_EQUAL(bf1, bf2); + BOOST_REQUIRE(bf1); +} + BOOST_AUTO_TEST_SUITE_END() From ec11b0490d51511cdfcbe65562310abefc6c40bf Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Mon, 22 Jun 2015 22:20:39 +0200 Subject: [PATCH 06/59] bugfix --- test/libwhisper/whisperTopic.cpp | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/test/libwhisper/whisperTopic.cpp b/test/libwhisper/whisperTopic.cpp index 56b1eea02..043e445ce 100644 --- a/test/libwhisper/whisperTopic.cpp +++ b/test/libwhisper/whisperTopic.cpp @@ -336,21 +336,19 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) while (!host2.peerCount()) this_thread::sleep_for(chrono::milliseconds(10)); - while (!whost1->peerSessions().size()) - this_thread::sleep_for(chrono::milliseconds(10)); + std::vector, std::shared_ptr>> sessions; - for (int i = 0; i < 200; ++i) + for (int i = 0; i < 300; ++i) { - this_thread::sleep_for(chrono::milliseconds(10)); - auto sessions1 = whost1->peerSessions(); - size_t x = sessions1.size(); - BOOST_REQUIRE(x > 0); - if (whost1->peerSessions()[x-1].first->cap()->bloom()) + sessions = whost1->peerSessions(); + if (!sessions.empty() && sessions.back().first->cap()->bloom()) break; + else + this_thread::sleep_for(chrono::milliseconds(10)); } - BOOST_REQUIRE(whost1->peerSessions().size()); - FixedHash bf1 = whost1->peerSessions().back().first->cap()->bloom(); + BOOST_REQUIRE(sessions.size()); + FixedHash bf1 = sessions.back().first->cap()->bloom(); FixedHash bf2 = whost2->bloom(); BOOST_REQUIRE_EQUAL(bf1, bf2); BOOST_REQUIRE(bf1); @@ -360,16 +358,17 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) for (int i = 0; i < 300; ++i) { - this_thread::sleep_for(chrono::milliseconds(10)); - if (whost2->peerSessions().back().first->cap()->bloom()) + sessions = whost2->peerSessions(); + if (!sessions.empty() && sessions.back().first->cap()->bloom()) break; + else + this_thread::sleep_for(chrono::milliseconds(10)); } - auto sessions2 = whost2->peerSessions(); - BOOST_REQUIRE(sessions2.size()); - BOOST_REQUIRE_EQUAL(sessions2.back().second->id, host1.id()); + BOOST_REQUIRE(sessions.size()); + BOOST_REQUIRE_EQUAL(sessions.back().second->id, host1.id()); - bf2 = sessions2.back().first->cap()->bloom(); + bf2 = sessions.back().first->cap()->bloom(); bf1 = whost1->bloom(); BOOST_REQUIRE_EQUAL(bf1, bf2); BOOST_REQUIRE(bf1); From aaec62722eaa5b1fca050ee1c9ac1d9e0ea49c4a Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Tue, 23 Jun 2015 13:56:34 +0200 Subject: [PATCH 07/59] protocol changed --- libwhisper/Common.cpp | 6 +++--- libwhisper/WhisperHost.cpp | 5 +---- libwhisper/WhisperPeer.cpp | 14 +++++++++----- libwhisper/WhisperPeer.h | 3 +-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/libwhisper/Common.cpp b/libwhisper/Common.cpp index 32acbdd8e..79c87b96f 100644 --- a/libwhisper/Common.cpp +++ b/libwhisper/Common.cpp @@ -95,12 +95,12 @@ TopicFilter::TopicFilter(RLP const& _r) } } -FixedHash TopicFilter::exportBloom() const +FixedHash TopicFilter::exportBloom() const { - FixedHash ret; + FixedHash ret; for (TopicMask const& t: m_topicMasks) for (auto const& i: t) - ret |= i.first.template bloomPart(); + ret |= i.first.template bloomPart(); return ret; } diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index 2cccb0151..cf33e2e81 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -88,10 +88,7 @@ void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p) void WhisperHost::advertizeTopicsOfInterest() { for (auto i: peerSessions()) - { - auto w = i.first->cap().get(); - w->advertizeTopicsOfInterest(m_bloom); - } + i.first->cap().get()->advertizeTopicsOfInterest(); } void WhisperHost::noteChanged(h256 _messageHash, h256 _filter) diff --git a/libwhisper/WhisperPeer.cpp b/libwhisper/WhisperPeer.cpp index 826b7842d..7ac08aa41 100644 --- a/libwhisper/WhisperPeer.cpp +++ b/libwhisper/WhisperPeer.cpp @@ -31,6 +31,7 @@ using namespace dev::shh; WhisperPeer::WhisperPeer(std::shared_ptr _s, HostCapabilityFace* _h, unsigned _i, CapDesc const&): Capability(_s, _h, _i) { RLPStream s; + //sealAndSend(prep(s, StatusPacket, 1) << version()); prep(s, StatusPacket, 2); s << version(); s << host()->bloom(); @@ -58,9 +59,12 @@ bool WhisperPeer::interpret(unsigned _id, RLP const& _r) if (protocolVersion != version()) disable("Invalid protocol version."); - - if (_r.itemCount() > 1) // for backwards compatibility - m_bloom = (FixedHash)_r[1]; + else + { + if (_r.itemCount() > 1) // for backwards compatibility + m_bloom = (FixedHash)_r[1]; + advertizeTopicsOfInterest(); + } for (auto const& m: host()->all()) m_unseen.insert(make_pair(0, m.first)); @@ -115,10 +119,10 @@ void WhisperPeer::noteNewMessage(h256 _h, Envelope const& _m) m_unseen.insert(make_pair(rating(_m), _h)); } -void WhisperPeer::advertizeTopicsOfInterest(FixedHash const& _bloom) +void WhisperPeer::advertizeTopicsOfInterest() { RLPStream s; prep(s, UpdateTopicFilterPacket, 1); - s << _bloom; + s << host()->bloom(); sealAndSend(s); } diff --git a/libwhisper/WhisperPeer.h b/libwhisper/WhisperPeer.h index 95b4a1952..3a2c879ee 100644 --- a/libwhisper/WhisperPeer.h +++ b/libwhisper/WhisperPeer.h @@ -56,8 +56,7 @@ public: static u256 version() { return 2; } static unsigned messageCount() { return PacketCount; } FixedHash const& bloom() const { return m_bloom; } - void advertizeTopicsOfInterest(FixedHash const& _bloom); ///< sends our bloom filter to remote peer - FixedHash const& bloom() { return m_bloom; } + void advertizeTopicsOfInterest(); ///< sends our bloom filter to remote peer private: virtual bool interpret(unsigned _id, RLP const&) override; From db959d5c6fe381bbddb14afba17b36b6dc5ffd87 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Tue, 23 Jun 2015 15:00:34 +0200 Subject: [PATCH 08/59] deleted unused variable --- test/libwhisper/whisperTopic.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/libwhisper/whisperTopic.cpp b/test/libwhisper/whisperTopic.cpp index 043e445ce..6fe132efc 100644 --- a/test/libwhisper/whisperTopic.cpp +++ b/test/libwhisper/whisperTopic.cpp @@ -320,7 +320,7 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) Host host2("second", NetworkPreferences("127.0.0.1", 30305, false)); host2.setIdealPeerCount(1); auto whost2 = host2.registerCapability(new WhisperHost()); - auto watch2 = whost2->installWatch(BuildTopicMask("test2")); + whost2->installWatch(BuildTopicMask("test2")); host2.start(); while (!host2.haveNetwork()) @@ -354,7 +354,7 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) BOOST_REQUIRE(bf1); BOOST_REQUIRE(!whost1->bloom()); - auto watch1 = whost1->installWatch(BuildTopicMask("test1")); + whost1->installWatch(BuildTopicMask("test1")); for (int i = 0; i < 300; ++i) { From eb995ebde54975065abbbcf7d3e44cf9b86c5b95 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Wed, 24 Jun 2015 13:56:07 +0200 Subject: [PATCH 09/59] BF size changed --- libwhisper/Common.h | 4 ++-- libwhisper/WhisperHost.cpp | 8 ++++---- libwhisper/WhisperHost.h | 2 +- libwhisper/WhisperPeer.cpp | 8 ++++---- libwhisper/WhisperPeer.h | 2 +- test/libwhisper/bloomFilter.cpp | 27 +++++++++++++++------------ 6 files changed, 27 insertions(+), 24 deletions(-) diff --git a/libwhisper/Common.h b/libwhisper/Common.h index fb48f2df6..4f5bda540 100644 --- a/libwhisper/Common.h +++ b/libwhisper/Common.h @@ -54,11 +54,11 @@ enum WhisperPacket { StatusPacket = 0, MessagesPacket, - UpdateTopicFilterPacket, + TopicFilterPacket, PacketCount }; -enum { TopicBloomFilterSize = 8 }; +enum { TopicBloomFilterSize = 64 }; using AbridgedTopic = FixedHash<4>; using Topic = h256; diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index cf33e2e81..a6eeed4e3 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -85,10 +85,10 @@ void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p) } } -void WhisperHost::advertizeTopicsOfInterest() +void WhisperHost::advertiseTopicsOfInterest() { for (auto i: peerSessions()) - i.first->cap().get()->advertizeTopicsOfInterest(); + i.first->cap().get()->advertiseTopicsOfInterest(); } void WhisperHost::noteChanged(h256 _messageHash, h256 _filter) @@ -120,7 +120,7 @@ unsigned WhisperHost::installWatch(shh::Topics const& _t) m_filters.insert(make_pair(h, f)); m_bloom.addRaw(f.filter.exportBloom()); - advertizeTopicsOfInterest(); + advertiseTopicsOfInterest(); return installWatchOnId(h); } @@ -165,7 +165,7 @@ void WhisperHost::uninstallWatch(unsigned _i) m_filters.erase(fit); m_bloom.removeRaw(fit->second.filter.exportBloom()); - advertizeTopicsOfInterest(); + advertiseTopicsOfInterest(); } } diff --git a/libwhisper/WhisperHost.h b/libwhisper/WhisperHost.h index b5e321dea..851031177 100644 --- a/libwhisper/WhisperHost.h +++ b/libwhisper/WhisperHost.h @@ -74,7 +74,7 @@ private: void streamMessage(h256 _m, RLPStream& _s) const; void noteChanged(h256 _messageHash, h256 _filter); - void advertizeTopicsOfInterest(); + void advertiseTopicsOfInterest(); mutable dev::SharedMutex x_messages; std::map m_messages; diff --git a/libwhisper/WhisperPeer.cpp b/libwhisper/WhisperPeer.cpp index 7ac08aa41..6fb03e9a0 100644 --- a/libwhisper/WhisperPeer.cpp +++ b/libwhisper/WhisperPeer.cpp @@ -63,7 +63,7 @@ bool WhisperPeer::interpret(unsigned _id, RLP const& _r) { if (_r.itemCount() > 1) // for backwards compatibility m_bloom = (FixedHash)_r[1]; - advertizeTopicsOfInterest(); + advertiseTopicsOfInterest(); } for (auto const& m: host()->all()) @@ -73,7 +73,7 @@ bool WhisperPeer::interpret(unsigned _id, RLP const& _r) sendMessages(); break; } - case UpdateTopicFilterPacket: + case TopicFilterPacket: { m_bloom = (FixedHash)_r[0]; break; @@ -119,10 +119,10 @@ void WhisperPeer::noteNewMessage(h256 _h, Envelope const& _m) m_unseen.insert(make_pair(rating(_m), _h)); } -void WhisperPeer::advertizeTopicsOfInterest() +void WhisperPeer::advertiseTopicsOfInterest() { RLPStream s; - prep(s, UpdateTopicFilterPacket, 1); + prep(s, TopicFilterPacket, 1); s << host()->bloom(); sealAndSend(s); } diff --git a/libwhisper/WhisperPeer.h b/libwhisper/WhisperPeer.h index 3a2c879ee..442b12eaa 100644 --- a/libwhisper/WhisperPeer.h +++ b/libwhisper/WhisperPeer.h @@ -56,7 +56,7 @@ public: static u256 version() { return 2; } static unsigned messageCount() { return PacketCount; } FixedHash const& bloom() const { return m_bloom; } - void advertizeTopicsOfInterest(); ///< sends our bloom filter to remote peer + void advertiseTopicsOfInterest(); ///< sends our bloom filter to remote peer private: virtual bool interpret(unsigned _id, RLP const&) override; diff --git a/test/libwhisper/bloomFilter.cpp b/test/libwhisper/bloomFilter.cpp index e49473bdc..7fe363033 100644 --- a/test/libwhisper/bloomFilter.cpp +++ b/test/libwhisper/bloomFilter.cpp @@ -28,8 +28,7 @@ using namespace dev; using namespace dev::shh; using TopicBloomFilterShort = TopicBloomFilterBase<4>; -using TopicBloomFilterLong = TopicBloomFilterBase<8>; -using TopicBloomFilterTest = TopicBloomFilterLong; +using TopicBloomFilterTest = TopicBloomFilterBase; void testAddNonExisting(TopicBloomFilterShort& _f, AbridgedTopic const& _h) { @@ -59,7 +58,7 @@ void testRemoveExistingBloom(TopicBloomFilterShort& _f, AbridgedTopic const& _h) BOOST_REQUIRE(!_f.containsBloom(_h)); } -int calculateExpected(TopicBloomFilterTest const& f, int const n) +double calculateExpected(TopicBloomFilterTest const& f, int const n) { int const m = f.size * 8; // number of bits in the bloom int const k = f.BitsPerBloom; // number of hash functions (e.g. bits set to 1 in every bloom) @@ -77,10 +76,10 @@ int calculateExpected(TopicBloomFilterTest const& f, int const n) for (int i = 0; i < k; ++i) kBitsSet *= single; - return static_cast(kBitsSet * 100 + 0.5); // in percents, rounded up + return kBitsSet; } -void testFalsePositiveRate(TopicBloomFilterTest const& f, int const inserted, Topic& x) +double testFalsePositiveRate(TopicBloomFilterTest const& f, int const inserted, Topic& x) { int const c_sampleSize = 1000; int falsePositive = 0; @@ -93,12 +92,14 @@ void testFalsePositiveRate(TopicBloomFilterTest const& f, int const inserted, To ++falsePositive; } - falsePositive /= (c_sampleSize / 100); // in percents - int expected = calculateExpected(f, inserted); - int allowed = expected + (expected / 5); // allow deviations ~20% + double res = double(falsePositive) / double(c_sampleSize); - //cnote << "Inserted: " << inserted << ", False Positive Rate: " << falsePositive << ", Expected: " << expected; - BOOST_REQUIRE(falsePositive <= allowed); + double expected = calculateExpected(f, inserted); + double allowed = expected * 1.2 + 0.05; // allow deviations ~25% + + //cnote << "Inserted: " << inserted << ", False Positive Rate: " << res << ", Expected: " << expected; + BOOST_REQUIRE(res <= allowed); + return expected; } BOOST_AUTO_TEST_SUITE(bloomFilter) @@ -111,11 +112,13 @@ BOOST_AUTO_TEST_CASE(falsePositiveRate) TopicBloomFilterTest f; Topic x(0xC0DEFEED); // deterministic pseudorandom value - for (int i = 1; i < 21; ++i) + double expectedRate = 0; + + for (int i = 1; i < 50 && expectedRate < 0.5; ++i) { x = sha3(x); f.addBloom(AbridgedTopic(x)); - testFalsePositiveRate(f, i, x); + expectedRate = testFalsePositiveRate(f, i, x); } } From 6a3060c7b60f65c8bb39b0c1aa53b94f125b5637 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Wed, 24 Jun 2015 15:55:11 +0200 Subject: [PATCH 10/59] minor clean up --- libwhisper/WhisperPeer.h | 4 ++-- test/libwhisper/whisperTopic.cpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libwhisper/WhisperPeer.h b/libwhisper/WhisperPeer.h index 442b12eaa..431349878 100644 --- a/libwhisper/WhisperPeer.h +++ b/libwhisper/WhisperPeer.h @@ -55,7 +55,7 @@ public: static std::string name() { return "shh"; } static u256 version() { return 2; } static unsigned messageCount() { return PacketCount; } - FixedHash const& bloom() const { return m_bloom; } + FixedHash const& bloom() const { return m_bloom; } void advertiseTopicsOfInterest(); ///< sends our bloom filter to remote peer private: @@ -66,7 +66,7 @@ private: mutable dev::Mutex x_unseen; std::multimap m_unseen; ///< Rated according to what they want. - std::chrono::system_clock::time_point m_timer = std::chrono::system_clock::now(); + std::chrono::system_clock::time_point m_timer = std::chrono::system_clock::now(); FixedHash m_bloom; ///< Peer's topics of interest }; diff --git a/test/libwhisper/whisperTopic.cpp b/test/libwhisper/whisperTopic.cpp index 6fe132efc..2940ebd6c 100644 --- a/test/libwhisper/whisperTopic.cpp +++ b/test/libwhisper/whisperTopic.cpp @@ -324,7 +324,7 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) host2.start(); while (!host2.haveNetwork()) - this_thread::sleep_for(chrono::milliseconds(10)); + this_thread::sleep_for(chrono::milliseconds(10)); host1.addNode(host2.id(), NodeIPEndpoint(bi::address::from_string("127.0.0.1"), 30305, 30305)); while (!host1.haveNetwork()) @@ -344,7 +344,7 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) if (!sessions.empty() && sessions.back().first->cap()->bloom()) break; else - this_thread::sleep_for(chrono::milliseconds(10)); + this_thread::sleep_for(chrono::milliseconds(10)); } BOOST_REQUIRE(sessions.size()); @@ -362,7 +362,7 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) if (!sessions.empty() && sessions.back().first->cap()->bloom()) break; else - this_thread::sleep_for(chrono::milliseconds(10)); + this_thread::sleep_for(chrono::milliseconds(10)); } BOOST_REQUIRE(sessions.size()); From 784a786a82c254339f41146fa051e05a53198da9 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Thu, 25 Jun 2015 12:40:19 +0200 Subject: [PATCH 11/59] Topics of interest advertising made asynchronous --- libwhisper/WhisperHost.cpp | 17 ++++++++++++----- libwhisper/WhisperHost.h | 11 ++++++----- libwhisper/WhisperPeer.cpp | 26 ++++++++++++-------------- libwhisper/WhisperPeer.h | 12 +++++++++--- test/libwhisper/whisperTopic.cpp | 4 ++-- 5 files changed, 41 insertions(+), 29 deletions(-) diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index a6eeed4e3..7d3a0d4e8 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -87,8 +87,10 @@ void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p) void WhisperHost::advertiseTopicsOfInterest() { + FixedHash b = bloom(); + for (auto i: peerSessions()) - i.first->cap().get()->advertiseTopicsOfInterest(); + i.first->cap().get()->sendTopicsOfInterest(b); } void WhisperHost::noteChanged(h256 _messageHash, h256 _filter) @@ -111,16 +113,15 @@ unsigned WhisperHost::installWatchOnId(h256 _h) unsigned WhisperHost::installWatch(shh::Topics const& _t) { - Guard l(m_filterLock); - InstalledFilter f(_t); h256 h = f.filter.sha3(); + Guard l(m_filterLock); if (!m_filters.count(h)) m_filters.insert(make_pair(h, f)); m_bloom.addRaw(f.filter.exportBloom()); - advertiseTopicsOfInterest(); + noteAdvertiseTopicsOfInterest(); return installWatchOnId(h); } @@ -165,7 +166,7 @@ void WhisperHost::uninstallWatch(unsigned _i) m_filters.erase(fit); m_bloom.removeRaw(fit->second.filter.exportBloom()); - advertiseTopicsOfInterest(); + noteAdvertiseTopicsOfInterest(); } } @@ -185,3 +186,9 @@ void WhisperHost::cleanup() for (auto it = m_expiryQueue.begin(); it != m_expiryQueue.end() && it->first <= now; it = m_expiryQueue.erase(it)) m_messages.erase(it->second); } + +void WhisperHost::noteAdvertiseTopicsOfInterest() +{ + for (auto i: peerSessions()) + i.first->cap().get()->noteAdvertiseTopicsOfInterest(); +} diff --git a/libwhisper/WhisperHost.h b/libwhisper/WhisperHost.h index 851031177..69a1280aa 100644 --- a/libwhisper/WhisperHost.h +++ b/libwhisper/WhisperHost.h @@ -51,9 +51,9 @@ public: WhisperHost(); virtual ~WhisperHost(); unsigned protocolVersion() const { return 2; } - void cleanup(); - std::map all() const { ReadGuard l(x_messages); return m_messages; } - FixedHash const& bloom() const { return m_bloom; } + void cleanup(); ///< remove old messages + std::map all() const { dev::ReadGuard l(x_messages); return m_messages; } + FixedHash bloom() const { dev::Guard l(m_filterLock); return m_bloom; } virtual void inject(Envelope const& _e, WhisperPeer* _from = nullptr) override; virtual Topics const& fullTopics(unsigned _id) const override { try { return m_filters.at(m_watches.at(_id).id).full; } catch (...) { return EmptyTopics; } } @@ -62,11 +62,12 @@ public: virtual void uninstallWatch(unsigned _watchId) override; virtual h256s peekWatch(unsigned _watchId) const override { dev::Guard l(m_filterLock); try { return m_watches.at(_watchId).changes; } catch (...) { return h256s(); } } virtual h256s checkWatch(unsigned _watchId) override { cleanup(); dev::Guard l(m_filterLock); h256s ret; try { ret = m_watches.at(_watchId).changes; m_watches.at(_watchId).changes.clear(); } catch (...) {} return ret; } - virtual h256s watchMessages(unsigned _watchId) override; /// returns IDs of messages, which match specific watch criteria + virtual h256s watchMessages(unsigned _watchId) override; ///< returns IDs of messages, which match specific watch criteria virtual Envelope envelope(h256 _m) const override { try { dev::ReadGuard l(x_messages); return m_messages.at(_m); } catch (...) { return Envelope(); } } protected: virtual void doWork() override; + void noteAdvertiseTopicsOfInterest(); private: virtual void onStarting() override { startWorking(); } @@ -74,7 +75,7 @@ private: void streamMessage(h256 _m, RLPStream& _s) const; void noteChanged(h256 _messageHash, h256 _filter); - void advertiseTopicsOfInterest(); + void advertiseTopicsOfInterest(); ///< send our bloom filter to remote peers mutable dev::SharedMutex x_messages; std::map m_messages; diff --git a/libwhisper/WhisperPeer.cpp b/libwhisper/WhisperPeer.cpp index 6fb03e9a0..ff73732ce 100644 --- a/libwhisper/WhisperPeer.cpp +++ b/libwhisper/WhisperPeer.cpp @@ -31,11 +31,8 @@ using namespace dev::shh; WhisperPeer::WhisperPeer(std::shared_ptr _s, HostCapabilityFace* _h, unsigned _i, CapDesc const&): Capability(_s, _h, _i) { RLPStream s; - //sealAndSend(prep(s, StatusPacket, 1) << version()); - prep(s, StatusPacket, 2); - s << version(); - s << host()->bloom(); - sealAndSend(s); + sealAndSend(prep(s, StatusPacket, 1) << version()); + noteAdvertiseTopicsOfInterest(); } WhisperPeer::~WhisperPeer() @@ -59,23 +56,19 @@ bool WhisperPeer::interpret(unsigned _id, RLP const& _r) if (protocolVersion != version()) disable("Invalid protocol version."); - else - { - if (_r.itemCount() > 1) // for backwards compatibility - m_bloom = (FixedHash)_r[1]; - advertiseTopicsOfInterest(); - } for (auto const& m: host()->all()) m_unseen.insert(make_pair(0, m.first)); if (session()->id() < host()->host()->id()) sendMessages(); + + noteAdvertiseTopicsOfInterest(); break; } case TopicFilterPacket: { - m_bloom = (FixedHash)_r[0]; + setBloom((FixedHash)_r[0]); break; } case MessagesPacket: @@ -92,6 +85,9 @@ bool WhisperPeer::interpret(unsigned _id, RLP const& _r) void WhisperPeer::sendMessages() { + if (m_advertiseTopicsOfInterest) + sendTopicsOfInterest(host()->bloom()); + RLPStream amalg; unsigned msgCount = 0; { @@ -119,10 +115,12 @@ void WhisperPeer::noteNewMessage(h256 _h, Envelope const& _m) m_unseen.insert(make_pair(rating(_m), _h)); } -void WhisperPeer::advertiseTopicsOfInterest() +void WhisperPeer::sendTopicsOfInterest(FixedHash const& _bloom) { RLPStream s; prep(s, TopicFilterPacket, 1); - s << host()->bloom(); + s << _bloom; sealAndSend(s); + + m_advertiseTopicsOfInterest = false; } diff --git a/libwhisper/WhisperPeer.h b/libwhisper/WhisperPeer.h index 431349878..9c242dc1f 100644 --- a/libwhisper/WhisperPeer.h +++ b/libwhisper/WhisperPeer.h @@ -53,21 +53,27 @@ public: virtual ~WhisperPeer(); WhisperHost* host() const; static std::string name() { return "shh"; } - static u256 version() { return 2; } + static u256 version() { return 3; } static unsigned messageCount() { return PacketCount; } - FixedHash const& bloom() const { return m_bloom; } - void advertiseTopicsOfInterest(); ///< sends our bloom filter to remote peer + FixedHash bloom() const { dev::Guard g(x_bloom); return m_bloom; } + void sendTopicsOfInterest(FixedHash const& _bloom); ///< sends our bloom filter to remote peer + void noteAdvertiseTopicsOfInterest() { m_advertiseTopicsOfInterest = true; } private: virtual bool interpret(unsigned _id, RLP const&) override; void sendMessages(); unsigned rating(Envelope const&) const { return 0; } // TODO void noteNewMessage(h256 _h, Envelope const& _m); + void setBloom(FixedHash const& _b) { dev::Guard g(x_bloom); m_bloom = _b; } mutable dev::Mutex x_unseen; std::multimap m_unseen; ///< Rated according to what they want. std::chrono::system_clock::time_point m_timer = std::chrono::system_clock::now(); + + mutable dev::Mutex x_bloom; FixedHash m_bloom; ///< Peer's topics of interest + + bool m_advertiseTopicsOfInterest; }; } diff --git a/test/libwhisper/whisperTopic.cpp b/test/libwhisper/whisperTopic.cpp index 2940ebd6c..a152f756e 100644 --- a/test/libwhisper/whisperTopic.cpp +++ b/test/libwhisper/whisperTopic.cpp @@ -338,7 +338,7 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) std::vector, std::shared_ptr>> sessions; - for (int i = 0; i < 300; ++i) + for (int i = 0; i < 600; ++i) { sessions = whost1->peerSessions(); if (!sessions.empty() && sessions.back().first->cap()->bloom()) @@ -356,7 +356,7 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) whost1->installWatch(BuildTopicMask("test1")); - for (int i = 0; i < 300; ++i) + for (int i = 0; i < 600; ++i) { sessions = whost2->peerSessions(); if (!sessions.empty() && sessions.back().first->cap()->bloom()) From 9ba48b4628b20e78ef28bd2f3521f2a1a325ad1a Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Thu, 25 Jun 2015 16:57:25 +0200 Subject: [PATCH 12/59] mutex added --- libwhisper/Common.h | 1 + libwhisper/WhisperHost.h | 2 +- libwhisper/WhisperPeer.cpp | 3 ++- libwhisper/WhisperPeer.h | 5 +++-- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/libwhisper/Common.h b/libwhisper/Common.h index 4f5bda540..ca84469ae 100644 --- a/libwhisper/Common.h +++ b/libwhisper/Common.h @@ -59,6 +59,7 @@ enum WhisperPacket }; enum { TopicBloomFilterSize = 64 }; +enum { WhisperProtocolVersion = 2 }; using AbridgedTopic = FixedHash<4>; using Topic = h256; diff --git a/libwhisper/WhisperHost.h b/libwhisper/WhisperHost.h index 69a1280aa..75ee15887 100644 --- a/libwhisper/WhisperHost.h +++ b/libwhisper/WhisperHost.h @@ -50,7 +50,7 @@ class WhisperHost: public HostCapability, public Interface, public public: WhisperHost(); virtual ~WhisperHost(); - unsigned protocolVersion() const { return 2; } + unsigned protocolVersion() const { return WhisperProtocolVersion; } void cleanup(); ///< remove old messages std::map all() const { dev::ReadGuard l(x_messages); return m_messages; } FixedHash bloom() const { dev::Guard l(m_filterLock); return m_bloom; } diff --git a/libwhisper/WhisperPeer.cpp b/libwhisper/WhisperPeer.cpp index ff73732ce..97b533bdf 100644 --- a/libwhisper/WhisperPeer.cpp +++ b/libwhisper/WhisperPeer.cpp @@ -117,10 +117,11 @@ void WhisperPeer::noteNewMessage(h256 _h, Envelope const& _m) void WhisperPeer::sendTopicsOfInterest(FixedHash const& _bloom) { + Guard g(x_advertiseTopicsOfInterest); RLPStream s; prep(s, TopicFilterPacket, 1); s << _bloom; sealAndSend(s); - m_advertiseTopicsOfInterest = false; } + diff --git a/libwhisper/WhisperPeer.h b/libwhisper/WhisperPeer.h index 9c242dc1f..1be2df97e 100644 --- a/libwhisper/WhisperPeer.h +++ b/libwhisper/WhisperPeer.h @@ -53,11 +53,11 @@ public: virtual ~WhisperPeer(); WhisperHost* host() const; static std::string name() { return "shh"; } - static u256 version() { return 3; } + static u256 version() { return WhisperProtocolVersion; } static unsigned messageCount() { return PacketCount; } FixedHash bloom() const { dev::Guard g(x_bloom); return m_bloom; } void sendTopicsOfInterest(FixedHash const& _bloom); ///< sends our bloom filter to remote peer - void noteAdvertiseTopicsOfInterest() { m_advertiseTopicsOfInterest = true; } + void noteAdvertiseTopicsOfInterest() { dev::Guard g(x_advertiseTopicsOfInterest); m_advertiseTopicsOfInterest = true; } private: virtual bool interpret(unsigned _id, RLP const&) override; @@ -73,6 +73,7 @@ private: mutable dev::Mutex x_bloom; FixedHash m_bloom; ///< Peer's topics of interest + mutable dev::Mutex x_advertiseTopicsOfInterest; bool m_advertiseTopicsOfInterest; }; From 52cbbda15dc17656cab0897a6a328b41f300f2e2 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Fri, 26 Jun 2015 00:58:23 +0200 Subject: [PATCH 13/59] deleted method InstallWatchOnID() --- libwhisper/Common.h | 2 +- libwhisper/Interface.h | 2 +- libwhisper/WhisperHost.cpp | 66 +++++++++++++++++--------------------- libwhisper/WhisperHost.h | 2 -- 4 files changed, 32 insertions(+), 40 deletions(-) diff --git a/libwhisper/Common.h b/libwhisper/Common.h index ca84469ae..deb13d111 100644 --- a/libwhisper/Common.h +++ b/libwhisper/Common.h @@ -59,7 +59,7 @@ enum WhisperPacket }; enum { TopicBloomFilterSize = 64 }; -enum { WhisperProtocolVersion = 2 }; +enum { WhisperProtocolVersion = 3 }; using AbridgedTopic = FixedHash<4>; using Topic = h256; diff --git a/libwhisper/Interface.h b/libwhisper/Interface.h index ff16c7e53..941cf3434 100644 --- a/libwhisper/Interface.h +++ b/libwhisper/Interface.h @@ -67,7 +67,7 @@ public: virtual Topics const& fullTopics(unsigned _id) const = 0; virtual unsigned installWatch(Topics const& _filter) = 0; - virtual unsigned installWatchOnId(h256 _filterId) = 0; +// virtual unsigned installWatchOnId(h256 _filterId) = 0; virtual void uninstallWatch(unsigned _watchId) = 0; virtual h256s peekWatch(unsigned _watchId) const = 0; virtual h256s checkWatch(unsigned _watchId) = 0; diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index 7d3a0d4e8..6670af723 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -85,16 +85,9 @@ void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p) } } -void WhisperHost::advertiseTopicsOfInterest() -{ - FixedHash b = bloom(); - - for (auto i: peerSessions()) - i.first->cap().get()->sendTopicsOfInterest(b); -} - void WhisperHost::noteChanged(h256 _messageHash, h256 _filter) { + Guard l(m_filterLock); for (auto& i: m_watches) if (i.second.id == _filter) { @@ -103,27 +96,26 @@ void WhisperHost::noteChanged(h256 _messageHash, h256 _filter) } } -unsigned WhisperHost::installWatchOnId(h256 _h) -{ - auto ret = m_watches.size() ? m_watches.rbegin()->first + 1 : 0; - m_watches[ret] = ClientWatch(_h); - cwatshh << "+++" << ret << _h; - return ret; -} - unsigned WhisperHost::installWatch(shh::Topics const& _t) { InstalledFilter f(_t); h256 h = f.filter.sha3(); + unsigned ret = 0; - Guard l(m_filterLock); - if (!m_filters.count(h)) - m_filters.insert(make_pair(h, f)); + DEV_GUARDED(m_filterLock) + { + if (!m_filters.count(h)) + m_filters.insert(make_pair(h, f)); - m_bloom.addRaw(f.filter.exportBloom()); - noteAdvertiseTopicsOfInterest(); + ret = m_watches.size() ? m_watches.rbegin()->first + 1 : 0; + m_watches[ret] = ClientWatch(h); + cwatshh << "+++" << ret << h; - return installWatchOnId(h); + m_bloom.addRaw(f.filter.exportBloom()); + } + + noteAdvertiseTopicsOfInterest(); + return ret; } h256s WhisperHost::watchMessages(unsigned _watchId) @@ -151,23 +143,25 @@ void WhisperHost::uninstallWatch(unsigned _i) { cwatshh << "XXX" << _i; - Guard l(m_filterLock); - - auto it = m_watches.find(_i); - if (it == m_watches.end()) - return; - auto id = it->second.id; - m_watches.erase(it); - - auto fit = m_filters.find(id); - if (fit != m_filters.end()) + DEV_GUARDED(m_filterLock) { - if (!--fit->second.refCount) - m_filters.erase(fit); + auto it = m_watches.find(_i); + if (it == m_watches.end()) + return; + auto id = it->second.id; + m_watches.erase(it); + + auto fit = m_filters.find(id); + if (fit != m_filters.end()) + { + if (!--fit->second.refCount) + m_filters.erase(fit); - m_bloom.removeRaw(fit->second.filter.exportBloom()); - noteAdvertiseTopicsOfInterest(); + m_bloom.removeRaw(fit->second.filter.exportBloom()); + } } + + noteAdvertiseTopicsOfInterest(); } void WhisperHost::doWork() diff --git a/libwhisper/WhisperHost.h b/libwhisper/WhisperHost.h index 75ee15887..6ec75af96 100644 --- a/libwhisper/WhisperHost.h +++ b/libwhisper/WhisperHost.h @@ -58,7 +58,6 @@ public: virtual void inject(Envelope const& _e, WhisperPeer* _from = nullptr) override; virtual Topics const& fullTopics(unsigned _id) const override { try { return m_filters.at(m_watches.at(_id).id).full; } catch (...) { return EmptyTopics; } } virtual unsigned installWatch(Topics const& _filter) override; - virtual unsigned installWatchOnId(h256 _filterId) override; virtual void uninstallWatch(unsigned _watchId) override; virtual h256s peekWatch(unsigned _watchId) const override { dev::Guard l(m_filterLock); try { return m_watches.at(_watchId).changes; } catch (...) { return h256s(); } } virtual h256s checkWatch(unsigned _watchId) override { cleanup(); dev::Guard l(m_filterLock); h256s ret; try { ret = m_watches.at(_watchId).changes; m_watches.at(_watchId).changes.clear(); } catch (...) {} return ret; } @@ -75,7 +74,6 @@ private: void streamMessage(h256 _m, RLPStream& _s) const; void noteChanged(h256 _messageHash, h256 _filter); - void advertiseTopicsOfInterest(); ///< send our bloom filter to remote peers mutable dev::SharedMutex x_messages; std::map m_messages; From 7b5c9710237446524bb33180735661b7814c5ad2 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 26 Jun 2015 11:26:41 +0200 Subject: [PATCH 14/59] More exception information when no block info available. Favour more expensive transactions. --- libethcore/Common.h | 1 + libethereum/Client.cpp | 3 ++- libethereum/TransactionQueue.cpp | 20 ++++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/libethcore/Common.h b/libethcore/Common.h index 25a6a8e1d..732d09981 100644 --- a/libethcore/Common.h +++ b/libethcore/Common.h @@ -112,6 +112,7 @@ enum class ImportResult AlreadyInChain, AlreadyKnown, Malformed, + OverbidGasPrice, BadChain }; diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index ad551c2ec..2b0a64491 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -105,7 +105,8 @@ void Client::onBadBlock(Exception& _ex) const bytes const* block = boost::get_error_info(_ex); if (!block) { - cwarn << "ODD: onBadBlock called but exception has no block in it."; + cwarn << "ODD: onBadBlock called but exception (" << _ex.what() << ") has no block in it."; + cwarn << boost::diagnostic_information(_ex, true); return; } diff --git a/libethereum/TransactionQueue.cpp b/libethereum/TransactionQueue.cpp index a86f6abf3..8931ee218 100644 --- a/libethereum/TransactionQueue.cpp +++ b/libethereum/TransactionQueue.cpp @@ -113,6 +113,26 @@ ImportResult TransactionQueue::manageImport_WITH_LOCK(h256 const& _h, Transactio // If it doesn't work, the signature is bad. // The transaction's nonce may yet be invalid (or, it could be "valid" but we may be missing a marginally older transaction). + auto r = m_senders.equal_range(_transaction.from()); + for (auto it = r.first; it != r.second; ++it) + if (m_current.count(it->second) && m_current[it->second].nonce() == _transaction.nonce()) + if (_transaction.gasPrice() < m_current[it->second].gasPrice()) + return ImportResult::OverbidGasPrice; + else + { + remove_WITH_LOCK(it->second); + break; + } + else if (m_future.count(it->second) && m_future[it->second].nonce() == _transaction.nonce()) + if (_transaction.gasPrice() < m_future[it->second].gasPrice()) + return ImportResult::OverbidGasPrice; + else + { + remove_WITH_LOCK(it->second); + break; + } + else {} + // If valid, append to blocks. insertCurrent_WITH_LOCK(make_pair(_h, _transaction)); m_known.insert(_h); From a51f37d8dae957c525273342e4a0b483dd05aec9 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 26 Jun 2015 11:29:47 +0200 Subject: [PATCH 15/59] Use onBad in case of bad uncle. --- libethereum/BlockChain.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 7abf9316e..ed4ec738a 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -1127,6 +1127,8 @@ VerifiedBlockRef BlockChain::verifyBlock(bytes const& _block, function Date: Fri, 26 Jun 2015 11:36:41 +0200 Subject: [PATCH 16/59] No need to add block info again - verifyBlock is guarantee to do it for us. --- libethereum/BlockChain.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index ed4ec738a..9bf89665a 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -400,7 +400,6 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import // clog(BlockChainNote) << " Malformed block: " << diagnostic_information(ex); ex << errinfo_phase(2); ex << errinfo_now(time(0)); - ex << errinfo_block(_block); throw; } #endif From 84dd9dd35df907b17dd1b052c75bb8f312751270 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 26 Jun 2015 11:43:39 +0200 Subject: [PATCH 17/59] More diagnostics for batch size evolving. --- libethash-cl/ethash_cl_miner.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/libethash-cl/ethash_cl_miner.cpp b/libethash-cl/ethash_cl_miner.cpp index 79024e2ad..d8952d6db 100644 --- a/libethash-cl/ethash_cl_miner.cpp +++ b/libethash-cl/ethash_cl_miner.cpp @@ -24,13 +24,13 @@ #include #include +#include #include #include #include #include #include #include -#include #include #include #include @@ -462,13 +462,21 @@ void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook m_searchKernel.setArg(6, start_nonce); // execute it! - boost::timer t; + chrono::high_resolution_clock::time_point t = chrono::high_resolution_clock::now(); m_queue.enqueueNDRangeKernel(m_searchKernel, cl::NullRange, m_batchSize, m_workgroupSize); - unsigned ms = t.elapsed() * 1000; - if (ms > _msPerBatch * 1.1) + chrono::high_resolution_clock::duration d = chrono::high_resolution_clock::now() - t; + if (d > chrono::milliseconds(_msPerBatch * 10 / 9)) + { + cerr << "Batch of" << m_batchSize << "took" << chrono::duration_cast(d).count() << "ms, >>" << _msPerBatch << "ms."; m_batchSize = max(128, m_batchSize * 9 / 10); - else if (ms < _msPerBatch * 0.9) + cerr << "New batch size" << m_batchSize; + } + else if (d < chrono::milliseconds(_msPerBatch * 9 / 10)) + { + cerr << "Batch of" << m_batchSize << "took" << chrono::duration_cast(d).count() << "ms, <<" << _msPerBatch << "ms."; m_batchSize = m_batchSize * 10 / 9; + cerr << "New batch size" << m_batchSize; + } pending.push({ start_nonce, buf }); buf = (buf + 1) % c_bufferCount; From a142e968d0dbd85c12a87bc388e7f139a1293998 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 26 Jun 2015 11:44:54 +0200 Subject: [PATCH 18/59] Greater timing scope. --- libethash-cl/ethash_cl_miner.cpp | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/libethash-cl/ethash_cl_miner.cpp b/libethash-cl/ethash_cl_miner.cpp index d8952d6db..bd268ec56 100644 --- a/libethash-cl/ethash_cl_miner.cpp +++ b/libethash-cl/ethash_cl_miner.cpp @@ -464,19 +464,6 @@ void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook // execute it! chrono::high_resolution_clock::time_point t = chrono::high_resolution_clock::now(); m_queue.enqueueNDRangeKernel(m_searchKernel, cl::NullRange, m_batchSize, m_workgroupSize); - chrono::high_resolution_clock::duration d = chrono::high_resolution_clock::now() - t; - if (d > chrono::milliseconds(_msPerBatch * 10 / 9)) - { - cerr << "Batch of" << m_batchSize << "took" << chrono::duration_cast(d).count() << "ms, >>" << _msPerBatch << "ms."; - m_batchSize = max(128, m_batchSize * 9 / 10); - cerr << "New batch size" << m_batchSize; - } - else if (d < chrono::milliseconds(_msPerBatch * 9 / 10)) - { - cerr << "Batch of" << m_batchSize << "took" << chrono::duration_cast(d).count() << "ms, <<" << _msPerBatch << "ms."; - m_batchSize = m_batchSize * 10 / 9; - cerr << "New batch size" << m_batchSize; - } pending.push({ start_nonce, buf }); buf = (buf + 1) % c_bufferCount; @@ -506,6 +493,20 @@ void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook pending.pop(); } + + chrono::high_resolution_clock::duration d = chrono::high_resolution_clock::now() - t; + if (d > chrono::milliseconds(_msPerBatch * 10 / 9)) + { + cerr << "Batch of" << m_batchSize << "took" << chrono::duration_cast(d).count() << "ms, >>" << _msPerBatch << "ms."; + m_batchSize = max(128, m_batchSize * 9 / 10); + cerr << "New batch size" << m_batchSize; + } + else if (d < chrono::milliseconds(_msPerBatch * 9 / 10)) + { + cerr << "Batch of" << m_batchSize << "took" << chrono::duration_cast(d).count() << "ms, <<" << _msPerBatch << "ms."; + m_batchSize = m_batchSize * 10 / 9; + cerr << "New batch size" << m_batchSize; + } } // not safe to return until this is ready From 59d88742c2dadbc7aec669a33e49a7d1a8191dc7 Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 26 Jun 2015 06:46:49 -0400 Subject: [PATCH 19/59] Prep for supporting both cryptopp and secp256k1. Link (old) secp256k1 library to devcrypto. Rename cryptopp Secp256k1 to Secp256k1PP. Update toPublic to use secp256k1 library and add test. --- CMakeLists.txt | 5 +++- alethzero/DappLoader.cpp | 2 +- libdevcrypto/CMakeLists.txt | 4 +++ libdevcrypto/Common.cpp | 49 +++++++++++++++++++++++++----------- libdevcrypto/Common.h | 2 ++ libdevcrypto/CryptoPP.cpp | 30 +++++++++++----------- libdevcrypto/CryptoPP.h | 8 +++--- libdevcrypto/ECDHE.cpp | 2 +- mix/FileIo.cpp | 2 +- test/libdevcrypto/crypto.cpp | 14 ++++++++++- test/libp2p/rlpx.cpp | 2 +- 11 files changed, 80 insertions(+), 40 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e3b6a53c..d7e598678 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -397,7 +397,10 @@ if (JSCONSOLE) add_subdirectory(ethconsole) endif () -add_subdirectory(secp256k1) +if (NOT WIN32) + add_subdirectory(secp256k1) +endif () + add_subdirectory(libscrypt) add_subdirectory(libdevcrypto) diff --git a/alethzero/DappLoader.cpp b/alethzero/DappLoader.cpp index 5ef784e3a..a91531f89 100644 --- a/alethzero/DappLoader.cpp +++ b/alethzero/DappLoader.cpp @@ -129,7 +129,7 @@ void DappLoader::downloadComplete(QNetworkReply* _reply) h256 expected = m_uriHashes[requestUrl]; bytes package(reinterpret_cast(data.constData()), reinterpret_cast(data.constData() + data.size())); - Secp256k1 dec; + Secp256k1PP dec; dec.decrypt(expected, package); h256 got = sha3(package); if (got != expected) diff --git a/libdevcrypto/CMakeLists.txt b/libdevcrypto/CMakeLists.txt index d30aa7bc1..4244d95ca 100644 --- a/libdevcrypto/CMakeLists.txt +++ b/libdevcrypto/CMakeLists.txt @@ -24,6 +24,10 @@ target_link_libraries(${EXECUTABLE} ${DB_LIBRARIES}) target_link_libraries(${EXECUTABLE} ${CRYPTOPP_LIBRARIES}) target_link_libraries(${EXECUTABLE} scrypt) target_link_libraries(${EXECUTABLE} devcore) +if (NOT WIN32) + add_definitions(-DETH_HAVE_SECP256K1) + target_link_libraries(${EXECUTABLE} secp256k1) +endif () install( TARGETS ${EXECUTABLE} RUNTIME DESTINATION bin ARCHIVE DESTINATION lib LIBRARY DESTINATION lib ) install( FILES ${HEADERS} DESTINATION include/${EXECUTABLE} ) diff --git a/libdevcrypto/Common.cpp b/libdevcrypto/Common.cpp index af61a1dd5..de1eb645a 100644 --- a/libdevcrypto/Common.cpp +++ b/libdevcrypto/Common.cpp @@ -29,6 +29,9 @@ #include #include #include +#if ETH_HAVE_SECP256K1 +#include +#endif #include "AES.h" #include "CryptoPP.h" #include "Exceptions.h" @@ -36,7 +39,17 @@ using namespace std; using namespace dev; using namespace dev::crypto; -static Secp256k1 s_secp256k1; +#ifdef ETH_HAVE_SECP256K1 +struct Secp256k1Context +{ + Secp256k1Context() { secp256k1_start(); } + ~Secp256k1Context() { secp256k1_stop(); } +}; +static Secp256k1Context s_secp256k1; +void dev::crypto::secp256k1Init() { (void)s_secp256k1; } +#endif + +static Secp256k1PP s_secp256k1pp; bool dev::SignatureStruct::isValid() const noexcept { @@ -53,34 +66,42 @@ Address dev::ZeroAddress = Address(); Public dev::toPublic(Secret const& _secret) { +#ifdef ETH_HAVE_SECP256K1 + bytes o(65); + int pubkeylen; + if (!secp256k1_ecdsa_pubkey_create(o.data(), &pubkeylen, _secret.data(), false)) + return Public(); + return FixedHash<64>(o.data()+1, Public::ConstructFromPointer); +#else Public p; - s_secp256k1.toPublic(_secret, p); + s_secp256k1pp.toPublic(_secret, p); return p; +#endif } Address dev::toAddress(Public const& _public) { - return s_secp256k1.toAddress(_public); + return right160(sha3(_public.ref())); } Address dev::toAddress(Secret const& _secret) { Public p; - s_secp256k1.toPublic(_secret, p); - return s_secp256k1.toAddress(p); + s_secp256k1pp.toPublic(_secret, p); + return toAddress(p); } void dev::encrypt(Public const& _k, bytesConstRef _plain, bytes& o_cipher) { bytes io = _plain.toBytes(); - s_secp256k1.encrypt(_k, io); + s_secp256k1pp.encrypt(_k, io); o_cipher = std::move(io); } bool dev::decrypt(Secret const& _k, bytesConstRef _cipher, bytes& o_plaintext) { bytes io = _cipher.toBytes(); - s_secp256k1.decrypt(_k, io); + s_secp256k1pp.decrypt(_k, io); if (io.empty()) return false; o_plaintext = std::move(io); @@ -90,14 +111,14 @@ bool dev::decrypt(Secret const& _k, bytesConstRef _cipher, bytes& o_plaintext) void dev::encryptECIES(Public const& _k, bytesConstRef _plain, bytes& o_cipher) { bytes io = _plain.toBytes(); - s_secp256k1.encryptECIES(_k, io); + s_secp256k1pp.encryptECIES(_k, io); o_cipher = std::move(io); } bool dev::decryptECIES(Secret const& _k, bytesConstRef _cipher, bytes& o_plaintext) { bytes io = _cipher.toBytes(); - if (!s_secp256k1.decryptECIES(_k, io)) + if (!s_secp256k1pp.decryptECIES(_k, io)) return false; o_plaintext = std::move(io); return true; @@ -163,17 +184,17 @@ bytes dev::decryptAES128CTR(bytesConstRef _k, h128 const& _iv, bytesConstRef _ci Public dev::recover(Signature const& _sig, h256 const& _message) { - return s_secp256k1.recover(_sig, _message.ref()); + return s_secp256k1pp.recover(_sig, _message.ref()); } Signature dev::sign(Secret const& _k, h256 const& _hash) { - return s_secp256k1.sign(_k, _hash); + return s_secp256k1pp.sign(_k, _hash); } bool dev::verify(Public const& _p, Signature const& _s, h256 const& _hash) { - return s_secp256k1.verify(_p, _s, _hash.ref(), true); + return s_secp256k1pp.verify(_p, _s, _hash.ref(), true); } bytes dev::pbkdf2(string const& _pass, bytes const& _salt, unsigned _iterations, unsigned _dkLen) @@ -232,8 +253,8 @@ KeyPair KeyPair::create() KeyPair::KeyPair(h256 _sec): m_secret(_sec) { - if (s_secp256k1.verifySecret(m_secret, m_public)) - m_address = s_secp256k1.toAddress(m_public); + if (s_secp256k1pp.verifySecret(m_secret, m_public)) + m_address = toAddress(m_public); } KeyPair KeyPair::fromEncryptedSeed(bytesConstRef _seed, std::string const& _password) diff --git a/libdevcrypto/Common.h b/libdevcrypto/Common.h index b3d2649b8..7bb46c3ea 100644 --- a/libdevcrypto/Common.h +++ b/libdevcrypto/Common.h @@ -177,6 +177,8 @@ namespace crypto { struct InvalidState: public dev::Exception {}; +void secp256k1Init(); + /// Key derivation h256 kdf(Secret const& _priv, h256 const& _hash); diff --git a/libdevcrypto/CryptoPP.cpp b/libdevcrypto/CryptoPP.cpp index 40eae10f1..270367de5 100644 --- a/libdevcrypto/CryptoPP.cpp +++ b/libdevcrypto/CryptoPP.cpp @@ -33,7 +33,7 @@ static_assert(dev::Secret::size == 32, "Secret key must be 32 bytes."); static_assert(dev::Public::size == 64, "Public key must be 64 bytes."); static_assert(dev::Signature::size == 65, "Signature must be 65 bytes."); -bytes Secp256k1::eciesKDF(Secret _z, bytes _s1, unsigned kdByteLen) +bytes Secp256k1PP::eciesKDF(Secret _z, bytes _s1, unsigned kdByteLen) { // interop w/go ecies implementation @@ -64,7 +64,7 @@ bytes Secp256k1::eciesKDF(Secret _z, bytes _s1, unsigned kdByteLen) return k; } -void Secp256k1::encryptECIES(Public const& _k, bytes& io_cipher) +void Secp256k1PP::encryptECIES(Public const& _k, bytes& io_cipher) { // interop w/go ecies implementation auto r = KeyPair::create(); @@ -98,7 +98,7 @@ void Secp256k1::encryptECIES(Public const& _k, bytes& io_cipher) io_cipher.swap(msg); } -bool Secp256k1::decryptECIES(Secret const& _k, bytes& io_text) +bool Secp256k1PP::decryptECIES(Secret const& _k, bytes& io_text) { // interop w/go ecies implementation @@ -145,7 +145,7 @@ bool Secp256k1::decryptECIES(Secret const& _k, bytes& io_text) return true; } -void Secp256k1::encrypt(Public const& _k, bytes& io_cipher) +void Secp256k1PP::encrypt(Public const& _k, bytes& io_cipher) { ECIES::Encryptor e; initializeDLScheme(_k, e); @@ -163,7 +163,7 @@ void Secp256k1::encrypt(Public const& _k, bytes& io_cipher) io_cipher = std::move(ciphertext); } -void Secp256k1::decrypt(Secret const& _k, bytes& io_text) +void Secp256k1PP::decrypt(Secret const& _k, bytes& io_text) { CryptoPP::ECIES::Decryptor d; initializeDLScheme(_k, d); @@ -194,12 +194,12 @@ void Secp256k1::decrypt(Secret const& _k, bytes& io_text) io_text = std::move(plain); } -Signature Secp256k1::sign(Secret const& _k, bytesConstRef _message) +Signature Secp256k1PP::sign(Secret const& _k, bytesConstRef _message) { return sign(_k, sha3(_message)); } -Signature Secp256k1::sign(Secret const& _key, h256 const& _hash) +Signature Secp256k1PP::sign(Secret const& _key, h256 const& _hash) { // assumption made by signing alogrithm asserts(m_q == m_qs); @@ -240,18 +240,18 @@ Signature Secp256k1::sign(Secret const& _key, h256 const& _hash) return sig; } -bool Secp256k1::verify(Signature const& _signature, bytesConstRef _message) +bool Secp256k1PP::verify(Signature const& _signature, bytesConstRef _message) { return !!recover(_signature, _message); } -bool Secp256k1::verify(Public const& _p, Signature const& _sig, bytesConstRef _message, bool _hashed) +bool Secp256k1PP::verify(Public const& _p, Signature const& _sig, bytesConstRef _message, bool _hashed) { // todo: verify w/o recovery (if faster) - return (bool)_p == _hashed ? (bool)recover(_sig, _message) : (bool)recover(_sig, sha3(_message).ref()); + return _p == (_hashed ? recover(_sig, _message) : recover(_sig, sha3(_message).ref())); } -Public Secp256k1::recover(Signature _signature, bytesConstRef _message) +Public Secp256k1PP::recover(Signature _signature, bytesConstRef _message) { Public recovered; @@ -293,7 +293,7 @@ Public Secp256k1::recover(Signature _signature, bytesConstRef _message) return recovered; } -bool Secp256k1::verifySecret(Secret const& _s, Public& _p) +bool Secp256k1PP::verifySecret(Secret const& _s, Public& _p) { DL_PrivateKey_EC k; k.Initialize(m_params, secretToExponent(_s)); @@ -309,7 +309,7 @@ bool Secp256k1::verifySecret(Secret const& _s, Public& _p) return true; } -void Secp256k1::agree(Secret const& _s, Public const& _r, h256& o_s) +void Secp256k1PP::agree(Secret const& _s, Public const& _r, h256& o_s) { // TODO: mutex ASN1::secp256k1() singleton // Creating Domain is non-const for m_oid and m_oid is not thread-safe @@ -320,7 +320,7 @@ void Secp256k1::agree(Secret const& _s, Public const& _r, h256& o_s) d.Agree(o_s.data(), _s.data(), remote); } -void Secp256k1::exportPublicKey(CryptoPP::DL_PublicKey_EC const& _k, Public& o_p) +void Secp256k1PP::exportPublicKey(CryptoPP::DL_PublicKey_EC const& _k, Public& o_p) { bytes prefixedKey(_k.GetGroupParameters().GetEncodedElementSize(true)); @@ -333,7 +333,7 @@ void Secp256k1::exportPublicKey(CryptoPP::DL_PublicKey_EC const& memcpy(o_p.data(), &prefixedKey[1], Public::size); } -void Secp256k1::exponentToPublic(Integer const& _e, Public& o_p) +void Secp256k1PP::exponentToPublic(Integer const& _e, Public& o_p) { CryptoPP::DL_PublicKey_EC pk; diff --git a/libdevcrypto/CryptoPP.h b/libdevcrypto/CryptoPP.h index 377da8754..6a0453330 100644 --- a/libdevcrypto/CryptoPP.h +++ b/libdevcrypto/CryptoPP.h @@ -67,13 +67,11 @@ inline Integer secretToExponent(Secret const& _s) { return std::move(Integer(_s. * CryptoPP secp256k1 algorithms. * @todo Collect ECIES methods into class. */ -class Secp256k1 +class Secp256k1PP { public: - Secp256k1(): m_oid(ASN1::secp256k1()), m_params(m_oid), m_curve(m_params.GetCurve()), m_q(m_params.GetGroupOrder()), m_qs(m_params.GetSubgroupOrder()) {} - - Address toAddress(Public const& _p) { return right160(sha3(_p.ref())); } - + Secp256k1PP(): m_oid(ASN1::secp256k1()), m_params(m_oid), m_curve(m_params.GetCurve()), m_q(m_params.GetGroupOrder()), m_qs(m_params.GetSubgroupOrder()) {} + void toPublic(Secret const& _s, Public& o_public) { exponentToPublic(Integer(_s.data(), sizeof(_s)), o_public); } /// Encrypts text (replace input). (ECIES w/XOR-SHA1) diff --git a/libdevcrypto/ECDHE.cpp b/libdevcrypto/ECDHE.cpp index a5aaf3984..f9e55f676 100644 --- a/libdevcrypto/ECDHE.cpp +++ b/libdevcrypto/ECDHE.cpp @@ -27,7 +27,7 @@ using namespace std; using namespace dev; using namespace dev::crypto; -static Secp256k1 s_secp256k1; +static Secp256k1PP s_secp256k1; void dev::crypto::ecdh::agree(Secret const& _s, Public const& _r, h256& o_s) { diff --git a/mix/FileIo.cpp b/mix/FileIo.cpp index cf8300677..96a1ff446 100644 --- a/mix/FileIo.cpp +++ b/mix/FileIo.cpp @@ -180,7 +180,7 @@ QStringList FileIo::makePackage(QString const& _deploymentFolder) dev::h256 dappHash = dev::sha3(dapp); //encrypt KeyPair key(dappHash); - Secp256k1 enc; + Secp256k1PP enc; enc.encrypt(key.pub(), dapp); QUrl url(_deploymentFolder + "package.dapp"); diff --git a/test/libdevcrypto/crypto.cpp b/test/libdevcrypto/crypto.cpp index 1a0537ccd..776c0a88e 100644 --- a/test/libdevcrypto/crypto.cpp +++ b/test/libdevcrypto/crypto.cpp @@ -43,7 +43,7 @@ BOOST_GLOBAL_FIXTURE( MoveNonceToTempDir ) BOOST_AUTO_TEST_SUITE(devcrypto) -static Secp256k1 s_secp256k1; +static Secp256k1PP s_secp256k1; static CryptoPP::AutoSeededRandomPool s_rng; static CryptoPP::OID s_curveOID(CryptoPP::ASN1::secp256k1()); static CryptoPP::DL_GroupParameters_EC s_params(s_curveOID); @@ -64,6 +64,16 @@ BOOST_AUTO_TEST_CASE(emptySHA3Types) BOOST_REQUIRE_EQUAL(emptyListSHA3, EmptyListSHA3); } +BOOST_AUTO_TEST_CASE(secp256k1lib) +{ + secp256k1Init(); + KeyPair k = KeyPair::create(); + BOOST_REQUIRE(!!k.sec()); + BOOST_REQUIRE(!!k.pub()); + Public test = toPublic(k.sec()); + BOOST_REQUIRE(k.pub() == test); +} + BOOST_AUTO_TEST_CASE(cryptopp_patch) { KeyPair k = KeyPair::create(); @@ -156,6 +166,7 @@ BOOST_AUTO_TEST_CASE(cryptopp_cryptopp_secp256k1libport) BOOST_AUTO_TEST_CASE(cryptopp_ecdsa_sipaseckp256k1) { +#ifdef CRYPTOPPNOTBROKEN secp256k1_start(); // cryptopp integer encoding @@ -237,6 +248,7 @@ BOOST_AUTO_TEST_CASE(cryptopp_ecdsa_sipaseckp256k1) BOOST_CHECK(cssz <= 72); BOOST_REQUIRE(1 == secp256k1_ecdsa_verify(hm.data(), sizeof(hm), dersig, cssz, encpub, 65)); } +#endif } BOOST_AUTO_TEST_CASE(sha3_norestart) diff --git a/test/libp2p/rlpx.cpp b/test/libp2p/rlpx.cpp index 620ddd952..6d5e59733 100644 --- a/test/libp2p/rlpx.cpp +++ b/test/libp2p/rlpx.cpp @@ -39,7 +39,7 @@ using namespace CryptoPP; BOOST_AUTO_TEST_SUITE(rlpx) -static Secp256k1 s_secp256k1; +static Secp256k1PP s_secp256k1; static CryptoPP::AutoSeededRandomPool s_rng; static CryptoPP::OID s_curveOID(CryptoPP::ASN1::secp256k1()); static CryptoPP::DL_GroupParameters_EC s_params(s_curveOID); From b4dd16b8ba77806b2a2e04c5972162de72c9579c Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Fri, 26 Jun 2015 13:07:30 +0200 Subject: [PATCH 20/59] mutex usage changed --- libwhisper/WhisperPeer.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libwhisper/WhisperPeer.cpp b/libwhisper/WhisperPeer.cpp index 97b533bdf..13892a87b 100644 --- a/libwhisper/WhisperPeer.cpp +++ b/libwhisper/WhisperPeer.cpp @@ -117,11 +117,12 @@ void WhisperPeer::noteNewMessage(h256 _h, Envelope const& _m) void WhisperPeer::sendTopicsOfInterest(FixedHash const& _bloom) { - Guard g(x_advertiseTopicsOfInterest); + DEV_GUARDED(x_advertiseTopicsOfInterest) + m_advertiseTopicsOfInterest = false; + RLPStream s; prep(s, TopicFilterPacket, 1); s << _bloom; sealAndSend(s); - m_advertiseTopicsOfInterest = false; } From 67b9135b283de15e87c135bdbeb88918e464c0ad Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 26 Jun 2015 07:41:42 -0400 Subject: [PATCH 21/59] Don't link secp256k1 on windows. --- alethzero/CMakeLists.txt | 1 - libethereum/CMakeLists.txt | 1 - libethereum/State.cpp | 1 - libethereumx/CMakeLists.txt | 1 - libweb3jsonrpc/CMakeLists.txt | 1 - libwebthree/CMakeLists.txt | 1 - libwhisper/CMakeLists.txt | 1 - neth/CMakeLists.txt | 1 - test/CMakeLists.txt | 4 +++- test/libethereum/stateOriginal.cpp | 1 - third/CMakeLists.txt | 1 - 11 files changed, 3 insertions(+), 11 deletions(-) diff --git a/alethzero/CMakeLists.txt b/alethzero/CMakeLists.txt index fe1f2f82f..93609e54c 100644 --- a/alethzero/CMakeLists.txt +++ b/alethzero/CMakeLists.txt @@ -52,7 +52,6 @@ target_link_libraries(${EXECUTABLE} ethereum) target_link_libraries(${EXECUTABLE} evm) target_link_libraries(${EXECUTABLE} ethcore) target_link_libraries(${EXECUTABLE} devcrypto) -target_link_libraries(${EXECUTABLE} secp256k1) target_link_libraries(${EXECUTABLE} lll) if (SOLIDITY) target_link_libraries(${EXECUTABLE} solidity) diff --git a/libethereum/CMakeLists.txt b/libethereum/CMakeLists.txt index e896c712b..48cc69646 100644 --- a/libethereum/CMakeLists.txt +++ b/libethereum/CMakeLists.txt @@ -32,7 +32,6 @@ target_link_libraries(${EXECUTABLE} p2p) target_link_libraries(${EXECUTABLE} devcrypto) target_link_libraries(${EXECUTABLE} ethcore) target_link_libraries(${EXECUTABLE} ${Boost_REGEX_LIBRARIES}) -target_link_libraries(${EXECUTABLE} secp256k1) if (JSONRPC) target_link_libraries(${EXECUTABLE} ${JSON_RPC_CPP_CLIENT_LIBRARIES}) target_link_libraries(${EXECUTABLE} ${CURL_LIBRARIES}) diff --git a/libethereum/State.cpp b/libethereum/State.cpp index ead72f496..a91411391 100644 --- a/libethereum/State.cpp +++ b/libethereum/State.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include diff --git a/libethereumx/CMakeLists.txt b/libethereumx/CMakeLists.txt index 42b551c64..e0f386f1f 100644 --- a/libethereumx/CMakeLists.txt +++ b/libethereumx/CMakeLists.txt @@ -24,7 +24,6 @@ target_link_libraries(${EXECUTABLE} ethereum) target_link_libraries(${EXECUTABLE} evm) target_link_libraries(${EXECUTABLE} lll) target_link_libraries(${EXECUTABLE} ethcore) -target_link_libraries(${EXECUTABLE} secp256k1) install( TARGETS ${EXECUTABLE} RUNTIME DESTINATION bin ARCHIVE DESTINATION lib LIBRARY DESTINATION lib ) install( FILES ${HEADERS} DESTINATION include/${EXECUTABLE} ) diff --git a/libweb3jsonrpc/CMakeLists.txt b/libweb3jsonrpc/CMakeLists.txt index c65efd39b..09cb0c4dd 100644 --- a/libweb3jsonrpc/CMakeLists.txt +++ b/libweb3jsonrpc/CMakeLists.txt @@ -27,7 +27,6 @@ target_link_libraries(${EXECUTABLE} ${JSON_RPC_CPP_SERVER_LIBRARIES}) target_link_libraries(${EXECUTABLE} ${MHD_LIBRARIES}) target_link_libraries(${EXECUTABLE} webthree) -target_link_libraries(${EXECUTABLE} secp256k1) if (SOLIDITY) target_link_libraries(${EXECUTABLE} solidity) diff --git a/libwebthree/CMakeLists.txt b/libwebthree/CMakeLists.txt index 015c28f46..f960ac00c 100644 --- a/libwebthree/CMakeLists.txt +++ b/libwebthree/CMakeLists.txt @@ -28,7 +28,6 @@ target_link_libraries(${EXECUTABLE} whisper) target_link_libraries(${EXECUTABLE} p2p) target_link_libraries(${EXECUTABLE} ethcore) target_link_libraries(${EXECUTABLE} devcrypto) -target_link_libraries(${EXECUTABLE} secp256k1) install( TARGETS ${EXECUTABLE} RUNTIME DESTINATION bin ARCHIVE DESTINATION lib LIBRARY DESTINATION lib ) install( FILES ${HEADERS} DESTINATION include/${EXECUTABLE} ) diff --git a/libwhisper/CMakeLists.txt b/libwhisper/CMakeLists.txt index 8a1439c8b..5af43d0b2 100644 --- a/libwhisper/CMakeLists.txt +++ b/libwhisper/CMakeLists.txt @@ -25,7 +25,6 @@ target_link_libraries(${EXECUTABLE} ethcore) target_link_libraries(${EXECUTABLE} devcrypto) target_link_libraries(${EXECUTABLE} devcore) target_link_libraries(${EXECUTABLE} p2p) -target_link_libraries(${EXECUTABLE} secp256k1) install( TARGETS ${EXECUTABLE} RUNTIME DESTINATION bin ARCHIVE DESTINATION lib LIBRARY DESTINATION lib ) install( FILES ${HEADERS} DESTINATION include/${EXECUTABLE} ) diff --git a/neth/CMakeLists.txt b/neth/CMakeLists.txt index 48b6408dc..68de36368 100644 --- a/neth/CMakeLists.txt +++ b/neth/CMakeLists.txt @@ -19,7 +19,6 @@ endif() target_link_libraries(${EXECUTABLE} webthree) target_link_libraries(${EXECUTABLE} ethereum) -target_link_libraries(${EXECUTABLE} secp256k1) target_link_libraries(${EXECUTABLE} ncurses) target_link_libraries(${EXECUTABLE} form) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d39a5cca1..952b1674f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -74,7 +74,9 @@ target_link_libraries(testeth ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES}) target_link_libraries(testeth ${CURL_LIBRARIES}) target_link_libraries(testeth ethereum) target_link_libraries(testeth ethcore) -target_link_libraries(testeth secp256k1) +if (NOT WIN32) + target_link_libraries(testeth secp256k1) +endif() if (JSCONSOLE) target_link_libraries(testeth jsengine) diff --git a/test/libethereum/stateOriginal.cpp b/test/libethereum/stateOriginal.cpp index ac2f893de..452163061 100644 --- a/test/libethereum/stateOriginal.cpp +++ b/test/libethereum/stateOriginal.cpp @@ -22,7 +22,6 @@ #include #include -#include #include #include #include diff --git a/third/CMakeLists.txt b/third/CMakeLists.txt index a173d10fd..df526049a 100644 --- a/third/CMakeLists.txt +++ b/third/CMakeLists.txt @@ -41,7 +41,6 @@ target_link_libraries(${EXECUTABLE} webthree) target_link_libraries(${EXECUTABLE} ethereum) target_link_libraries(${EXECUTABLE} evm) target_link_libraries(${EXECUTABLE} ethcore) -target_link_libraries(${EXECUTABLE} secp256k1) if (NOT ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")) target_link_libraries(${EXECUTABLE} serpent) endif() From d0dd66ac187732ee8646e0f395eb423a4b625878 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 26 Jun 2015 13:46:12 +0200 Subject: [PATCH 22/59] One more tryout. --- libethash-cl/ethash_cl_miner.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libethash-cl/ethash_cl_miner.cpp b/libethash-cl/ethash_cl_miner.cpp index bd268ec56..866af88cf 100644 --- a/libethash-cl/ethash_cl_miner.cpp +++ b/libethash-cl/ethash_cl_miner.cpp @@ -454,6 +454,8 @@ void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook uint64_t start_nonce = uniform_int_distribution()(engine); for (;; start_nonce += m_batchSize) { + chrono::high_resolution_clock::time_point t = chrono::high_resolution_clock::now(); + // supply output buffer to kernel m_searchKernel.setArg(0, m_searchBuffer[buf]); if (m_dagChunksCount == 1) @@ -462,7 +464,6 @@ void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook m_searchKernel.setArg(6, start_nonce); // execute it! - chrono::high_resolution_clock::time_point t = chrono::high_resolution_clock::now(); m_queue.enqueueNDRangeKernel(m_searchKernel, cl::NullRange, m_batchSize, m_workgroupSize); pending.push({ start_nonce, buf }); From a37d37a9c2a28521d701029b5f9c9a17593d9b91 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 26 Jun 2015 13:55:43 +0200 Subject: [PATCH 23/59] don't try to alter the batch size until we can make timing work. --- libethash-cl/ethash_cl_miner.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libethash-cl/ethash_cl_miner.cpp b/libethash-cl/ethash_cl_miner.cpp index 866af88cf..ec400f7a7 100644 --- a/libethash-cl/ethash_cl_miner.cpp +++ b/libethash-cl/ethash_cl_miner.cpp @@ -454,7 +454,7 @@ void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook uint64_t start_nonce = uniform_int_distribution()(engine); for (;; start_nonce += m_batchSize) { - chrono::high_resolution_clock::time_point t = chrono::high_resolution_clock::now(); +// chrono::high_resolution_clock::time_point t = chrono::high_resolution_clock::now(); // supply output buffer to kernel m_searchKernel.setArg(0, m_searchBuffer[buf]); @@ -495,7 +495,7 @@ void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook pending.pop(); } - chrono::high_resolution_clock::duration d = chrono::high_resolution_clock::now() - t; +/* chrono::high_resolution_clock::duration d = chrono::high_resolution_clock::now() - t; if (d > chrono::milliseconds(_msPerBatch * 10 / 9)) { cerr << "Batch of" << m_batchSize << "took" << chrono::duration_cast(d).count() << "ms, >>" << _msPerBatch << "ms."; @@ -507,7 +507,7 @@ void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook cerr << "Batch of" << m_batchSize << "took" << chrono::duration_cast(d).count() << "ms, <<" << _msPerBatch << "ms."; m_batchSize = m_batchSize * 10 / 9; cerr << "New batch size" << m_batchSize; - } + }*/ } // not safe to return until this is ready From d3809ea2f8159d12f8e79f447622770b9038292f Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Fri, 26 Jun 2015 14:36:40 +0200 Subject: [PATCH 24/59] bugfix: WhisperHost::noteChanged() deleted --- libp2p/Session.cpp | 2 +- libwhisper/WhisperHost.cpp | 18 ++++-------------- libwhisper/WhisperHost.h | 2 -- libwhisper/WhisperPeer.cpp | 10 +++++----- 4 files changed, 10 insertions(+), 22 deletions(-) diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index 95e7be598..3d0bd0b6a 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -268,7 +268,7 @@ bool Session::interpret(PacketType _t, RLP const& _r) } catch (std::exception const& _e) { - clog(NetWarn) << "Peer causing an exception:" << _e.what() << _r; + clog(NetWarn) << "Exception caught in p2p::Session::interpret(): " << _e.what() << ". PacketType: " << _t << ". RLP: " << _r; disconnect(BadProtocol); return true; } diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index 6670af723..1a47a56d1 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -66,12 +66,13 @@ void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p) m_expiryQueue.insert(make_pair(_m.expiry(), h)); } -// if (_p) + DEV_GUARDED(m_filterLock) { - Guard l(m_filterLock); for (auto const& f: m_filters) if (f.second.filter.matches(_m)) - noteChanged(h, f.first); + for (auto& i: m_watches) + if (i.second.id == f.first) + i.second.changes.push_back(h); } // TODO p2p: capability-based rating @@ -85,17 +86,6 @@ void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p) } } -void WhisperHost::noteChanged(h256 _messageHash, h256 _filter) -{ - Guard l(m_filterLock); - for (auto& i: m_watches) - if (i.second.id == _filter) - { - cwatshh << "!!!" << i.first << i.second.id; - i.second.changes.push_back(_messageHash); - } -} - unsigned WhisperHost::installWatch(shh::Topics const& _t) { InstalledFilter f(_t); diff --git a/libwhisper/WhisperHost.h b/libwhisper/WhisperHost.h index 6ec75af96..386153ac5 100644 --- a/libwhisper/WhisperHost.h +++ b/libwhisper/WhisperHost.h @@ -71,9 +71,7 @@ protected: private: virtual void onStarting() override { startWorking(); } virtual void onStopping() override { stopWorking(); } - void streamMessage(h256 _m, RLPStream& _s) const; - void noteChanged(h256 _messageHash, h256 _filter); mutable dev::SharedMutex x_messages; std::map m_messages; diff --git a/libwhisper/WhisperPeer.cpp b/libwhisper/WhisperPeer.cpp index 13892a87b..665364f49 100644 --- a/libwhisper/WhisperPeer.cpp +++ b/libwhisper/WhisperPeer.cpp @@ -66,17 +66,17 @@ bool WhisperPeer::interpret(unsigned _id, RLP const& _r) noteAdvertiseTopicsOfInterest(); break; } - case TopicFilterPacket: - { - setBloom((FixedHash)_r[0]); - break; - } case MessagesPacket: { for (auto i: _r) host()->inject(Envelope(i), this); break; } + case TopicFilterPacket: + { + setBloom((FixedHash)_r[0]); + break; + } default: return false; } From 6afff6d2761a3c09c1f62e06c677aeee30a4440d Mon Sep 17 00:00:00 2001 From: chriseth Date: Fri, 26 Jun 2015 16:54:20 +0200 Subject: [PATCH 25/59] Add memory address in memory dump. --- mix/DebuggingStateWrapper.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mix/DebuggingStateWrapper.cpp b/mix/DebuggingStateWrapper.cpp index b8fbdef30..42c429224 100644 --- a/mix/DebuggingStateWrapper.cpp +++ b/mix/DebuggingStateWrapper.cpp @@ -38,13 +38,17 @@ using namespace dev::mix; namespace { - static QVariantList memDumpToList(bytes const& _bytes, unsigned _width) + static QVariantList memDumpToList(bytes const& _bytes, unsigned _width, bool _includeAddress = false) { QVariantList dumpList; for (unsigned i = 0; i < _bytes.size(); i += _width) { std::stringstream ret; - + if (_includeAddress) + { + ret << std::setfill('0') << std::setw(6) << std::hex << i << " "; + ret << " "; + } for (unsigned j = i; j < i + _width; ++j) if (j < _bytes.size()) if (_bytes[j] >= 32 && _bytes[j] < 127) @@ -137,7 +141,7 @@ QStringList QMachineState::debugStorage() QVariantList QMachineState::debugMemory() { - return memDumpToList(m_state.memory, 16); + return memDumpToList(m_state.memory, 16, true); } QCallData* QMachineState::getDebugCallData(QObject* _owner, bytes const& _data) From f94ff7b41c68807a51650b18a3e3e031136b542e Mon Sep 17 00:00:00 2001 From: chriseth Date: Fri, 26 Jun 2015 16:52:30 +0200 Subject: [PATCH 26/59] Fixed and simplified external type computation. --- libsolidity/AST.cpp | 4 +++- libsolidity/Types.cpp | 17 ++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/libsolidity/AST.cpp b/libsolidity/AST.cpp index 7333c024a..09af49c67 100644 --- a/libsolidity/AST.cpp +++ b/libsolidity/AST.cpp @@ -340,8 +340,10 @@ vector, FunctionTypePointer>> const& ContractDefinition::getIn { for (ASTPointer const& f: contract->getDefinedFunctions()) { + if (!f->isPartOfExternalInterface()) + continue; string functionSignature = f->externalSignature(); - if (f->isPartOfExternalInterface() && signaturesSeen.count(functionSignature) == 0) + if (signaturesSeen.count(functionSignature) == 0) { functionsSeen.insert(f->getName()); signaturesSeen.insert(functionSignature); diff --git a/libsolidity/Types.cpp b/libsolidity/Types.cpp index 01876b5a7..15c367421 100644 --- a/libsolidity/Types.cpp +++ b/libsolidity/Types.cpp @@ -827,15 +827,16 @@ TypePointer ArrayType::externalType() const { if (m_arrayKind != ArrayKind::Ordinary) return this->copyForLocation(DataLocation::Memory, true); - if (!m_baseType->externalType()) + TypePointer baseExt = m_baseType->externalType(); + if (!baseExt) return TypePointer(); if (m_baseType->getCategory() == Category::Array && m_baseType->isDynamicallySized()) return TypePointer(); if (isDynamicallySized()) - return std::make_shared(DataLocation::Memory, m_baseType->externalType()); + return std::make_shared(DataLocation::Memory, baseExt); else - return std::make_shared(DataLocation::Memory, m_baseType->externalType(), m_length); + return std::make_shared(DataLocation::Memory, baseExt, m_length); } TypePointer ArrayType::copyForLocation(DataLocation _location, bool _isPointer) const @@ -1268,15 +1269,17 @@ FunctionTypePointer FunctionType::externalFunctionType() const for (auto type: m_parameterTypes) { - if (!type->externalType()) + if (auto ext = type->externalType()) + paramTypes.push_back(ext); + else return FunctionTypePointer(); - paramTypes.push_back(type->externalType()); } for (auto type: m_returnParameterTypes) { - if (!type->externalType()) + if (auto ext = type->externalType()) + retParamTypes.push_back(ext); + else return FunctionTypePointer(); - retParamTypes.push_back(type->externalType()); } return make_shared(paramTypes, retParamTypes, m_parameterNames, m_returnParameterNames, m_location, m_arbitraryParameters); } From 79375056db44bc891202b350aeb95a78c5cf4a1b Mon Sep 17 00:00:00 2001 From: chriseth Date: Thu, 25 Jun 2015 17:51:01 +0200 Subject: [PATCH 27/59] Do not copy reference types to memory in-place. --- libsolidity/ArrayUtils.cpp | 110 ++++++++----- libsolidity/ArrayUtils.h | 15 +- libsolidity/Compiler.cpp | 2 +- libsolidity/CompilerUtils.cpp | 182 +++++++++++++++------- libsolidity/CompilerUtils.h | 11 +- libsolidity/ExpressionCompiler.cpp | 76 +++------ libsolidity/ExpressionCompiler.h | 7 - libsolidity/LValue.cpp | 56 +++++++ libsolidity/LValue.h | 23 +++ libsolidity/Types.h | 5 + test/libsolidity/SolidityEndToEndTest.cpp | 47 ++++++ 11 files changed, 369 insertions(+), 165 deletions(-) diff --git a/libsolidity/ArrayUtils.cpp b/libsolidity/ArrayUtils.cpp index 3be12af72..74bde70af 100644 --- a/libsolidity/ArrayUtils.cpp +++ b/libsolidity/ArrayUtils.cpp @@ -39,6 +39,11 @@ void ArrayUtils::copyArrayToStorage(ArrayType const& _targetType, ArrayType cons // stack layout: [source_ref] [source_byte_off] [source length] target_ref target_byte_off (top) solAssert(_targetType.location() == DataLocation::Storage, ""); + if (_sourceType.location() == DataLocation::Memory) + solAssert( + _sourceType.getBaseType()->isValueType(), + "Copying arrays of non-value-types to storage not yet implemented." + ); IntegerType uint256(256); Type const* targetBaseType = _targetType.isByteArray() ? &uint256 : &(*_targetType.getBaseType()); @@ -235,8 +240,9 @@ void ArrayUtils::copyArrayToMemory(const ArrayType& _sourceType, bool _padToWord { solAssert( _sourceType.getBaseType()->getCalldataEncodedSize() > 0, - "Nested arrays not yet implemented here." + "Nested dynamic arrays not implemented here." ); + CompilerUtils utils(m_context); unsigned baseSize = 1; if (!_sourceType.isByteArray()) // We always pad the elements, regardless of _padToWordBoundaries. @@ -246,7 +252,7 @@ void ArrayUtils::copyArrayToMemory(const ArrayType& _sourceType, bool _padToWord { if (!_sourceType.isDynamicallySized()) m_context << _sourceType.getLength(); - if (_sourceType.getBaseType()->getCalldataEncodedSize() > 1) + if (baseSize > 1) m_context << u256(baseSize) << eth::Instruction::MUL; // stack: target source_offset source_len m_context << eth::Instruction::DUP1 << eth::Instruction::DUP3 << eth::Instruction::DUP5; @@ -257,8 +263,36 @@ void ArrayUtils::copyArrayToMemory(const ArrayType& _sourceType, bool _padToWord } else if (_sourceType.location() == DataLocation::Memory) { - // memcpy using the built-in contract retrieveLength(_sourceType); + // stack: target source length + if (!_sourceType.getBaseType()->isValueType()) + { + // copy using a loop + m_context << u256(0) << eth::Instruction::SWAP3; + // stack: counter source length target + auto repeat = m_context.newTag(); + m_context << repeat; + m_context << eth::Instruction::DUP2 << eth::Instruction::DUP5; + m_context << eth::Instruction::LT << eth::Instruction::ISZERO; + auto loopEnd = m_context.appendConditionalJump(); + m_context << eth::Instruction::DUP3 << eth::Instruction::DUP5; + accessIndex(_sourceType, false); + MemoryItem(m_context, *_sourceType.getBaseType(), true).retrieveValue(SourceLocation(), true); + if (auto baseArray = dynamic_cast(_sourceType.getBaseType().get())) + copyArrayToMemory(*baseArray, _padToWordBoundaries); + else + utils.storeInMemoryDynamic(*_sourceType.getBaseType()); + m_context << eth::Instruction::SWAP3 << u256(1) << eth::Instruction::ADD; + m_context << eth::Instruction::SWAP3; + m_context.appendJumpTo(repeat); + m_context << loopEnd; + m_context << eth::Instruction::SWAP3; + utils.popStackSlots(3); + // stack: updated_target_pos + return; + } + + // memcpy using the built-in contract if (_sourceType.isDynamicallySized()) { // change pointer to data part @@ -271,7 +305,7 @@ void ArrayUtils::copyArrayToMemory(const ArrayType& _sourceType, bool _padToWord // stack: //@TODO do not use ::CALL if less than 32 bytes? m_context << eth::Instruction::DUP1 << eth::Instruction::DUP4 << eth::Instruction::DUP4; - CompilerUtils(m_context).memoryCopy(); + utils.memoryCopy(); m_context << eth::Instruction::SWAP1 << eth::Instruction::POP; // stack: @@ -345,7 +379,7 @@ void ArrayUtils::copyArrayToMemory(const ArrayType& _sourceType, bool _padToWord { // actual array data is stored at SHA3(storage_offset) m_context << eth::Instruction::SWAP1; - CompilerUtils(m_context).computeHashStatic(); + utils.computeHashStatic(); m_context << eth::Instruction::SWAP1; } @@ -375,7 +409,10 @@ void ArrayUtils::copyArrayToMemory(const ArrayType& _sourceType, bool _padToWord else m_context << eth::Instruction::DUP2 << u256(0); StorageItem(m_context, *_sourceType.getBaseType()).retrieveValue(SourceLocation(), true); - CompilerUtils(m_context).storeInMemoryDynamic(*_sourceType.getBaseType()); + if (auto baseArray = dynamic_cast(_sourceType.getBaseType().get())) + copyArrayToMemory(*baseArray, _padToWordBoundaries); + else + utils.storeInMemoryDynamic(*_sourceType.getBaseType()); // increment storage_data_offset and byte offset if (haveByteOffset) incrementByteOffset(storageBytes, 2, 3); @@ -387,7 +424,8 @@ void ArrayUtils::copyArrayToMemory(const ArrayType& _sourceType, bool _padToWord } } // check for loop condition - m_context << eth::Instruction::DUP1 << eth::dupInstruction(haveByteOffset ? 5 : 4) << eth::Instruction::GT; + m_context << eth::Instruction::DUP1 << eth::dupInstruction(haveByteOffset ? 5 : 4); + m_context << eth::Instruction::GT; m_context.appendConditionalJumpTo(loopStart); // stack here: memory_end_offset storage_data_offset [storage_byte_offset] memory_offset if (haveByteOffset) @@ -597,12 +635,14 @@ void ArrayUtils::convertLengthToSize(ArrayType const& _arrayType, bool _pad) con } else { - solAssert( - _arrayType.getBaseType()->getCalldataEncodedSize() > 0, - "Copying nested dynamic arrays not yet implemented." - ); if (!_arrayType.isByteArray()) - m_context << _arrayType.getBaseType()->getCalldataEncodedSize() << eth::Instruction::MUL; + { + if (_arrayType.location() == DataLocation::Memory) + m_context << _arrayType.getBaseType()->memoryHeadSize(); + else + m_context << _arrayType.getBaseType()->getCalldataEncodedSize(); + m_context << eth::Instruction::MUL; + } else if (_pad) m_context << u256(31) << eth::Instruction::ADD << u256(32) << eth::Instruction::DUP1 @@ -632,7 +672,7 @@ void ArrayUtils::retrieveLength(ArrayType const& _arrayType) const } } -void ArrayUtils::accessIndex(ArrayType const& _arrayType) const +void ArrayUtils::accessIndex(ArrayType const& _arrayType, bool _doBoundsCheck) const { DataLocation location = _arrayType.location(); eth::Instruction load = @@ -640,19 +680,22 @@ void ArrayUtils::accessIndex(ArrayType const& _arrayType) const location == DataLocation::Memory ? eth::Instruction::MLOAD : eth::Instruction::CALLDATALOAD; - // retrieve length - if (!_arrayType.isDynamicallySized()) - m_context << _arrayType.getLength(); - else if (location == DataLocation::CallData) - // length is stored on the stack - m_context << eth::Instruction::SWAP1; - else - m_context << eth::Instruction::DUP2 << load; - // stack: - // check out-of-bounds access - m_context << eth::Instruction::DUP2 << eth::Instruction::LT << eth::Instruction::ISZERO; - // out-of-bounds access throws exception - m_context.appendConditionalJumpTo(m_context.errorTag()); + if (_doBoundsCheck) + { + // retrieve length + if (!_arrayType.isDynamicallySized()) + m_context << _arrayType.getLength(); + else if (location == DataLocation::CallData) + // length is stored on the stack + m_context << eth::Instruction::SWAP1; + else + m_context << eth::Instruction::DUP2 << load; + // stack: + // check out-of-bounds access + m_context << eth::Instruction::DUP2 << eth::Instruction::LT << eth::Instruction::ISZERO; + // out-of-bounds access throws exception + m_context.appendConditionalJumpTo(m_context.errorTag()); + } // stack: m_context << eth::Instruction::SWAP1; @@ -671,18 +714,13 @@ void ArrayUtils::accessIndex(ArrayType const& _arrayType) const if (!_arrayType.isByteArray()) { m_context << eth::Instruction::SWAP1; - m_context << _arrayType.getBaseType()->getCalldataEncodedSize() << eth::Instruction::MUL; + if (location == DataLocation::CallData) + m_context << _arrayType.getBaseType()->getCalldataEncodedSize(); + else + m_context << u256(_arrayType.memoryHeadSize()); + m_context << eth::Instruction::MUL; } m_context << eth::Instruction::ADD; - //@todo we should also load if it is a reference type of dynamic length - // but we should apply special logic if we load from calldata. - if (_arrayType.getBaseType()->isValueType()) - CompilerUtils(m_context).loadFromMemoryDynamic( - *_arrayType.getBaseType(), - location == DataLocation::CallData, - !_arrayType.isByteArray(), - false - ); break; case DataLocation::Storage: m_context << eth::Instruction::SWAP1; diff --git a/libsolidity/ArrayUtils.h b/libsolidity/ArrayUtils.h index 8d56f3c8f..c047fdcc0 100644 --- a/libsolidity/ArrayUtils.h +++ b/libsolidity/ArrayUtils.h @@ -44,7 +44,11 @@ public: /// Stack pre: source_reference [source_byte_offset/source_length] target_reference target_byte_offset /// Stack post: target_reference target_byte_offset void copyArrayToStorage(ArrayType const& _targetType, ArrayType const& _sourceType) const; - /// Copies an array (which cannot be dynamically nested) from anywhere to memory. + /// Copies the data part of an array (which cannot be dynamically nested) from anywhere + /// to a given position in memory. + /// This always copies contained data as is (i.e. structs and fixed-size arrays are copied in + /// place as required by the ABI encoding). Use CompilerUtils::convertType if you want real + /// memory copies of nested arrays. /// Stack pre: memory_offset source_item /// Stack post: memory_offest + length(padded) void copyArrayToMemory(ArrayType const& _sourceType, bool _padToWordBoundaries = true) const; @@ -74,12 +78,11 @@ public: /// Stack pre: reference (excludes byte offset for dynamic storage arrays) /// Stack post: reference length void retrieveLength(ArrayType const& _arrayType) const; - /// Retrieves the value at a specific index. If the location is storage, only retrieves the - /// position. + /// Performs bounds checking and returns a reference on the stack. /// Stack pre: reference [length] index - /// Stack post for storage: slot byte_offset - /// Stack post for calldata: value - void accessIndex(ArrayType const& _arrayType) const; + /// Stack post (storage): storage_slot byte_offset + /// Stack post: memory/calldata_offset + void accessIndex(ArrayType const& _arrayType, bool _doBoundsCheck = true) const; private: /// Adds the given number of bytes to a storage byte offset counter and also increments diff --git a/libsolidity/Compiler.cpp b/libsolidity/Compiler.cpp index b05a7a9b1..788e07ef7 100644 --- a/libsolidity/Compiler.cpp +++ b/libsolidity/Compiler.cpp @@ -657,7 +657,7 @@ void Compiler::appendStackVariableInitialisation(VariableDeclaration const& _var { CompilerContext::LocationSetter location(m_context, _variable); m_context.addVariable(_variable); - ExpressionCompiler(m_context).appendStackVariableInitialisation(*_variable.getType()); + CompilerUtils(m_context).pushZeroValue(*_variable.getType()); } void Compiler::compileExpression(Expression const& _expression, TypePointer const& _targetType) diff --git a/libsolidity/CompilerUtils.cpp b/libsolidity/CompilerUtils.cpp index 47a9a3542..b07e14c61 100644 --- a/libsolidity/CompilerUtils.cpp +++ b/libsolidity/CompilerUtils.cpp @@ -54,6 +54,13 @@ void CompilerUtils::storeFreeMemoryPointer() m_context << u256(freeMemoryPointer) << eth::Instruction::MSTORE; } +void CompilerUtils::allocateMemory() +{ + fetchFreeMemoryPointer(); + m_context << eth::Instruction::SWAP1 << eth::Instruction::DUP2 << eth::Instruction::ADD; + storeFreeMemoryPointer(); +} + void CompilerUtils::toSizeAfterFreeMemoryPointer() { fetchFreeMemoryPointer(); @@ -101,17 +108,20 @@ void CompilerUtils::storeInMemory(unsigned _offset) void CompilerUtils::storeInMemoryDynamic(Type const& _type, bool _padToWordBoundaries) { - if (_type.getCategory() == Type::Category::Array) - ArrayUtils(m_context).copyArrayToMemory( - dynamic_cast(_type), - _padToWordBoundaries - ); + if (auto ref = dynamic_cast(&_type)) + { + solAssert(ref->location() == DataLocation::Memory, ""); + storeInMemoryDynamic(IntegerType(256), _padToWordBoundaries); + } else { unsigned numBytes = prepareMemoryStore(_type, _padToWordBoundaries); if (numBytes > 0) { - solAssert(_type.getSizeOnStack() == 1, "Memory store of types with stack size != 1 not implemented."); + solAssert( + _type.getSizeOnStack() == 1, + "Memory store of types with stack size != 1 not implemented." + ); m_context << eth::Instruction::DUP2 << eth::Instruction::MSTORE; m_context << u256(numBytes) << eth::Instruction::ADD; } @@ -164,7 +174,10 @@ void CompilerUtils::encodeToMemory( type = _givenTypes[i]; // delay conversion else convertType(*_givenTypes[i], *targetType, true); - storeInMemoryDynamic(*type, _padToWordBoundaries); + if (auto arrayType = dynamic_cast(type.get())) + ArrayUtils(m_context).copyArrayToMemory(*arrayType, _padToWordBoundaries); + else + storeInMemoryDynamic(*type, _padToWordBoundaries); } stackPos += _givenTypes[i]->getSizeOnStack(); } @@ -207,7 +220,7 @@ void CompilerUtils::encodeToMemory( m_context << eth::swapInstruction(arrayType.getSizeOnStack() + 1) << eth::Instruction::POP; // stack: ... // copy data part - storeInMemoryDynamic(arrayType, true); + ArrayUtils(m_context).copyArrayToMemory(arrayType, _padToWordBoundaries); // stack: ... thisDynPointer++; @@ -349,63 +362,67 @@ void CompilerUtils::convertType(Type const& _typeOnStack, Type const& _targetTyp { // stack: (variably sized) unsigned stackSize = typeOnStack.getSizeOnStack(); - fetchFreeMemoryPointer(); - moveIntoStack(stackSize); - // stack: (variably sized) - if (targetType.isDynamicallySized()) + bool fromStorage = (typeOnStack.location() == DataLocation::Storage); + if (fromStorage) { - bool fromStorage = (typeOnStack.location() == DataLocation::Storage); - // store length - if (fromStorage) - { - stackSize--; - // remove storage offset, as requested by ArrayUtils::retrieveLength - m_context << eth::Instruction::POP; - } - ArrayUtils(m_context).retrieveLength(typeOnStack); - // Stack: - m_context << eth::dupInstruction(2 + stackSize) << eth::Instruction::MSTORE; - m_context << eth::dupInstruction(1 + stackSize) << u256(0x20); - m_context << eth::Instruction::ADD; - moveIntoStack(stackSize); - if (fromStorage) - { - m_context << u256(0); - stackSize++; - } + stackSize--; + // remove storage offset, as requested by ArrayUtils::retrieveLength + m_context << eth::Instruction::POP; } - else + ArrayUtils(m_context).retrieveLength(typeOnStack); + + // allocate memory + // stack: (variably sized) + m_context << eth::Instruction::DUP1; + ArrayUtils(m_context).convertLengthToSize(targetType, true); + // stack: (variably sized) + if (targetType.isDynamicallySized()) + m_context << u256(0x20) << eth::Instruction::ADD; + allocateMemory(); + // stack: (variably sized) + m_context << eth::Instruction::DUP1; + moveIntoStack(2 + stackSize); + if (targetType.isDynamicallySized()) { - m_context << eth::dupInstruction(1 + stackSize); - moveIntoStack(stackSize); + m_context << eth::Instruction::DUP2; + storeInMemoryDynamic(IntegerType(256)); } - // Stack: - // Store data part. - storeInMemoryDynamic(typeOnStack); - // Stack - storeFreeMemoryPointer(); - } - else if (typeOnStack.location() == DataLocation::CallData) - { - // Stack: [] - // length is present if dynamically sized - fetchFreeMemoryPointer(); - moveIntoStack(typeOnStack.getSizeOnStack()); - // stack: memptr calldataoffset [] - if (typeOnStack.isDynamicallySized()) + // stack: (variably sized) + if (targetType.getBaseType()->isValueType()) { - solAssert(targetType.isDynamicallySized(), ""); - m_context << eth::Instruction::DUP3 << eth::Instruction::DUP2; - storeInMemoryDynamic(IntegerType(256)); - moveIntoStack(typeOnStack.getSizeOnStack()); + copyToStackTop(2 + stackSize, stackSize); + if (fromStorage) + m_context << u256(0); // add byte offset again + ArrayUtils(m_context).copyArrayToMemory(typeOnStack); } else - m_context << eth::Instruction::DUP2 << eth::Instruction::SWAP1; - // stack: mem_ptr mem_data_ptr calldataoffset [] - storeInMemoryDynamic(typeOnStack); - storeFreeMemoryPointer(); + { + m_context << u256(0) << eth::Instruction::SWAP1; + // stack: (variably sized) + auto repeat = m_context.newTag(); + m_context << repeat; + m_context << eth::Instruction::DUP3 << eth::Instruction::DUP3; + m_context << eth::Instruction::LT << eth::Instruction::ISZERO; + auto loopEnd = m_context.appendConditionalJump(); + copyToStackTop(3 + stackSize, stackSize); + copyToStackTop(2 + stackSize, 1); + ArrayUtils(m_context).accessIndex(typeOnStack); + MemoryItem(m_context, *typeOnStack.getBaseType(), true).retrieveValue( + SourceLocation(), + true + ); + convertType(*typeOnStack.getBaseType(), *targetType.getBaseType(), _cleanupNeeded); + storeInMemoryDynamic(*targetType.getBaseType(), true); + m_context << eth::Instruction::SWAP1 << u256(1) << eth::Instruction::ADD; + m_context << eth::Instruction::SWAP1; + m_context.appendJumpTo(repeat); + m_context << loopEnd; + m_context << eth::Instruction::POP; + } + // stack: (variably sized) + popStackSlots(2 + stackSize); + // Stack: } - // nothing to do for memory to memory break; } default: @@ -444,6 +461,57 @@ void CompilerUtils::convertType(Type const& _typeOnStack, Type const& _targetTyp } } +void CompilerUtils::pushZeroValue(const Type& _type) +{ + auto const* referenceType = dynamic_cast(&_type); + if (!referenceType || referenceType->location() == DataLocation::Storage) + { + for (size_t i = 0; i < _type.getSizeOnStack(); ++i) + m_context << u256(0); + return; + } + solAssert(referenceType->location() == DataLocation::Memory, ""); + + m_context << u256(max(32u, _type.getCalldataEncodedSize())); + allocateMemory(); + m_context << eth::Instruction::DUP1; + + if (auto structType = dynamic_cast(&_type)) + for (auto const& member: structType->getMembers()) + { + pushZeroValue(*member.type); + storeInMemoryDynamic(*member.type); + } + else if (auto arrayType = dynamic_cast(&_type)) + { + if (arrayType->isDynamicallySized()) + { + // zero length + m_context << u256(0); + storeInMemoryDynamic(IntegerType(256)); + } + else if (arrayType->getLength() > 0) + { + m_context << arrayType->getLength() << eth::Instruction::SWAP1; + // stack: items_to_do memory_pos + auto repeat = m_context.newTag(); + m_context << repeat; + pushZeroValue(*arrayType->getBaseType()); + storeInMemoryDynamic(*arrayType->getBaseType()); + m_context << eth::Instruction::SWAP1 << u256(1) << eth::Instruction::SWAP1; + m_context << eth::Instruction::SUB << eth::Instruction::SWAP1; + m_context << eth::Instruction::DUP2; + m_context.appendConditionalJumpTo(repeat); + m_context << eth::Instruction::SWAP1 << eth::Instruction::POP; + } + } + else + solAssert(false, "Requested initialisation for unknown type: " + _type.toString()); + + // remove the updated memory pointer + m_context << eth::Instruction::POP; +} + void CompilerUtils::moveToStackVariable(VariableDeclaration const& _variable) { unsigned const stackPosition = m_context.baseToCurrentStackOffset(m_context.getBaseStackOffsetOfVariable(_variable)); diff --git a/libsolidity/CompilerUtils.h b/libsolidity/CompilerUtils.h index a9e07f74f..7dd44da8f 100644 --- a/libsolidity/CompilerUtils.h +++ b/libsolidity/CompilerUtils.h @@ -41,6 +41,10 @@ public: void fetchFreeMemoryPointer(); /// Stores the free memory pointer from the stack. void storeFreeMemoryPointer(); + /// Allocates a number of bytes in memory as given on the stack. + /// Stack pre: + /// Stack post: + void allocateMemory(); /// Appends code that transforms memptr to (memptr - free_memptr) memptr void toSizeAfterFreeMemoryPointer(); @@ -70,7 +74,8 @@ public: /// @param _type type of the data on the stack void storeInMemory(unsigned _offset); /// Dynamic version of @see storeInMemory, expects the memory offset below the value on the stack - /// and also updates that. For arrays, only copies the data part. + /// and also updates that. For reference types, only copies the data pointer. Fails for + /// non-memory-references. /// @param _padToWordBoundaries if true, adds zeros to pad to multiple of 32 bytes. Array elements /// are always padded (except for byte arrays), regardless of this parameter. /// Stack pre: memory_offset value... @@ -107,6 +112,10 @@ public: /// necessary. void convertType(Type const& _typeOnStack, Type const& _targetType, bool _cleanupNeeded = false); + /// Creates a zero-value for the given type and puts it onto the stack. This might allocate + /// memory for memory references. + void pushZeroValue(Type const& _type); + /// Moves the value that is at the top of the stack to a stack variable. void moveToStackVariable(VariableDeclaration const& _variable); /// Copies an item that occupies @a _itemSize stack slots from a stack depth of @a _stackDepth diff --git a/libsolidity/ExpressionCompiler.cpp b/libsolidity/ExpressionCompiler.cpp index fb10eb83b..7ddcc0318 100644 --- a/libsolidity/ExpressionCompiler.cpp +++ b/libsolidity/ExpressionCompiler.cpp @@ -56,62 +56,6 @@ void ExpressionCompiler::appendStateVariableInitialization(VariableDeclaration c StorageItem(m_context, _varDecl).storeValue(*_varDecl.getType(), _varDecl.getLocation(), true); } -void ExpressionCompiler::appendStackVariableInitialisation(Type const& _type, bool _toMemory) -{ - CompilerUtils utils(m_context); - auto const* referenceType = dynamic_cast(&_type); - if (!referenceType || referenceType->location() == DataLocation::Storage) - { - for (size_t i = 0; i < _type.getSizeOnStack(); ++i) - m_context << u256(0); - if (_toMemory) - utils.storeInMemoryDynamic(_type); - return; - } - solAssert(referenceType->location() == DataLocation::Memory, ""); - if (!_toMemory) - { - // allocate memory - utils.fetchFreeMemoryPointer(); - m_context << eth::Instruction::DUP1 << u256(max(32u, _type.getCalldataEncodedSize())); - m_context << eth::Instruction::ADD; - utils.storeFreeMemoryPointer(); - m_context << eth::Instruction::DUP1; - } - - if (auto structType = dynamic_cast(&_type)) - for (auto const& member: structType->getMembers()) - appendStackVariableInitialisation(*member.type, true); - else if (auto arrayType = dynamic_cast(&_type)) - { - if (arrayType->isDynamicallySized()) - { - // zero length - m_context << u256(0); - CompilerUtils(m_context).storeInMemoryDynamic(IntegerType(256)); - } - else if (arrayType->getLength() > 0) - { - m_context << arrayType->getLength() << eth::Instruction::SWAP1; - // stack: items_to_do memory_pos - auto repeat = m_context.newTag(); - m_context << repeat; - appendStackVariableInitialisation(*arrayType->getBaseType(), true); - m_context << eth::Instruction::SWAP1 << u256(1) << eth::Instruction::SWAP1; - m_context << eth::Instruction::SUB << eth::Instruction::SWAP1; - m_context << eth::Instruction::DUP2; - m_context.appendConditionalJumpTo(repeat); - m_context << eth::Instruction::SWAP1 << eth::Instruction::POP; - } - } - else - solAssert(false, "Requested initialisation for unknown type: " + _type.toString()); - - if (!_toMemory) - // remove the updated memory pointer - m_context << eth::Instruction::POP; -} - void ExpressionCompiler::appendStateVariableAccessor(VariableDeclaration const& _varDecl) { CompilerContext::LocationSetter locationSetter(m_context, _varDecl); @@ -211,6 +155,8 @@ bool ExpressionCompiler::visit(Assignment const& _assignment) TypePointer type = _assignment.getRightHandSide().getType(); if (!_assignment.getType()->dataStoredIn(DataLocation::Storage)) { + //@todo we should delay conversion here if RHS is not in memory, LHS is a MemoryItem + // and not dynamically-sized. utils().convertType(*type, *_assignment.getType()); type = _assignment.getType(); } @@ -827,8 +773,9 @@ bool ExpressionCompiler::visit(IndexAccess const& _indexAccess) _indexAccess.getIndexExpression()->accept(*this); // stack layout: [] ArrayUtils(m_context).accessIndex(arrayType); - if (arrayType.location() == DataLocation::Storage) + switch (arrayType.location()) { + case DataLocation::Storage: if (arrayType.isByteArray()) { solAssert(!arrayType.isString(), "Index access to string is not allowed."); @@ -836,6 +783,21 @@ bool ExpressionCompiler::visit(IndexAccess const& _indexAccess) } else setLValueToStorageItem(_indexAccess); + break; + case DataLocation::Memory: + setLValue(_indexAccess, *_indexAccess.getType(), !arrayType.isByteArray()); + break; + case DataLocation::CallData: + //@todo if we implement this, the value in calldata has to be added to the base offset + solAssert(!arrayType.getBaseType()->isDynamicallySized(), "Nested arrays not yet implemented."); + if (arrayType.getBaseType()->isValueType()) + CompilerUtils(m_context).loadFromMemoryDynamic( + *arrayType.getBaseType(), + true, + !arrayType.isByteArray(), + false + ); + break; } } else diff --git a/libsolidity/ExpressionCompiler.h b/libsolidity/ExpressionCompiler.h index 747e241ef..642560c64 100644 --- a/libsolidity/ExpressionCompiler.h +++ b/libsolidity/ExpressionCompiler.h @@ -64,13 +64,6 @@ public: /// Appends code to set a state variable to its initial value/expression. void appendStateVariableInitialization(VariableDeclaration const& _varDecl); - /// Appends code to initialise a local variable. - /// If @a _toMemory is false, leaves the value on the stack. For memory references, this - /// allocates new memory. - /// If @a _toMemory is true, directly stores the data in the memory pos on the stack and - /// updates it. - void appendStackVariableInitialisation(Type const& _type, bool _toMemory = false); - /// Appends code for a State Variable accessor function void appendStateVariableAccessor(VariableDeclaration const& _varDecl); diff --git a/libsolidity/LValue.cpp b/libsolidity/LValue.cpp index 1acf0a3e8..cf640e910 100644 --- a/libsolidity/LValue.cpp +++ b/libsolidity/LValue.cpp @@ -82,6 +82,62 @@ void StackVariable::setToZero(SourceLocation const& _location, bool) const << eth::Instruction::POP; } +MemoryItem::MemoryItem(CompilerContext& _compilerContext, Type const& _type, bool _padded): + LValue(_compilerContext, _type), + m_padded(_padded) +{ +} + +void MemoryItem::retrieveValue(SourceLocation const&, bool _remove) const +{ + if (m_dataType.isValueType()) + { + if (!_remove) + m_context << eth::Instruction::DUP1; + CompilerUtils(m_context).loadFromMemoryDynamic(m_dataType, false, m_padded, false); + } + else + m_context << eth::Instruction::MLOAD; +} + +void MemoryItem::storeValue(Type const& _sourceType, SourceLocation const&, bool _move) const +{ + CompilerUtils utils(m_context); + if (m_dataType.isValueType()) + { + solAssert(_sourceType.isValueType(), ""); + utils.moveIntoStack(_sourceType.getSizeOnStack()); + utils.convertType(_sourceType, m_dataType, true); + if (!_move) + { + utils.moveToStackTop(m_dataType.getSizeOnStack()); + utils.copyToStackTop(2, m_dataType.getSizeOnStack()); + } + utils.storeInMemoryDynamic(m_dataType, m_padded); + m_context << eth::Instruction::POP; + } + else + { + solAssert(_sourceType == m_dataType, "Conversion not implemented for assignment to memory."); + + solAssert(m_dataType.getSizeOnStack() == 1, ""); + if (!_move) + m_context << eth::Instruction::DUP2 << eth::Instruction::SWAP1; + // stack: [value] value lvalue + // only store the reference + m_context << eth::Instruction::MSTORE; + } +} + +void MemoryItem::setToZero(SourceLocation const&, bool _removeReference) const +{ + CompilerUtils utils(m_context); + if (!_removeReference) + m_context << eth::Instruction::DUP1; + utils.pushZeroValue(m_dataType); + utils.storeInMemoryDynamic(m_dataType, m_padded); + m_context << eth::Instruction::POP; +} StorageItem::StorageItem(CompilerContext& _compilerContext, Declaration const& _declaration): StorageItem(_compilerContext, *_declaration.getType()) diff --git a/libsolidity/LValue.h b/libsolidity/LValue.h index 726d63328..882b3626e 100644 --- a/libsolidity/LValue.h +++ b/libsolidity/LValue.h @@ -97,6 +97,29 @@ private: unsigned m_size; }; +/** + * Reference to some item in memory. + */ +class MemoryItem: public LValue +{ +public: + MemoryItem(CompilerContext& _compilerContext, Type const& _type, bool _padded); + virtual unsigned sizeOnStack() const override { return 1; } + virtual void retrieveValue(SourceLocation const& _location, bool _remove = false) const override; + virtual void storeValue( + Type const& _sourceType, + SourceLocation const& _location = SourceLocation(), + bool _move = false + ) const override; + virtual void setToZero( + SourceLocation const& _location = SourceLocation(), + bool _removeReference = true + ) const override; +private: + /// Special flag to deal with byte array elements. + bool m_padded = false; +}; + /** * Reference to some item in storage. On the stack this is , * where 0 <= offset_inside_value < 32 and an offset of i means that the value is multiplied diff --git a/libsolidity/Types.h b/libsolidity/Types.h index 9d412cd68..9d9aaaba7 100644 --- a/libsolidity/Types.h +++ b/libsolidity/Types.h @@ -179,6 +179,9 @@ public: /// is not a simple big-endian encoding or the type cannot be stored in calldata. /// If @a _padded then it is assumed that each element is padded to a multiple of 32 bytes. virtual unsigned getCalldataEncodedSize(bool _padded) const { (void)_padded; return 0; } + /// @returns the size of this data type in bytes when stored in memory. For memory-reference + /// types, this is the size of the memory pointer. + virtual unsigned memoryHeadSize() const { return getCalldataEncodedSize(); } /// Convenience version of @see getCalldataEncodedSize(bool) unsigned getCalldataEncodedSize() const { return getCalldataEncodedSize(true); } /// @returns true if the type is dynamically encoded in calldata @@ -373,6 +376,8 @@ public: explicit ReferenceType(DataLocation _location): m_location(_location) {} DataLocation location() const { return m_location; } + virtual unsigned memoryHeadSize() const override { return 32; } + /// @returns a copy of this type with location (recursively) changed to @a _location, /// whereas isPointer is only shallowly changed - the deep copy is always a bound reference. virtual TypePointer copyForLocation(DataLocation _location, bool _isPointer) const = 0; diff --git a/test/libsolidity/SolidityEndToEndTest.cpp b/test/libsolidity/SolidityEndToEndTest.cpp index 75793abf7..a01e98cf8 100644 --- a/test/libsolidity/SolidityEndToEndTest.cpp +++ b/test/libsolidity/SolidityEndToEndTest.cpp @@ -4691,6 +4691,53 @@ BOOST_AUTO_TEST_CASE(memory_types_initialisation) BOOST_CHECK(callContractFunction("nestedStat()") == encodeArgs(vector(3 * 7))); } +BOOST_AUTO_TEST_CASE(memory_arrays_index_access_write) +{ + char const* sourceCode = R"( + contract Test { + function set(uint24[3][4] x) { + x[2][2] = 1; + x[3][2] = 7; + } + function f() returns (uint24[3][4]){ + uint24[3][4] memory data; + set(data); + return data; + } + } + )"; + compileAndRun(sourceCode, 0, "Test"); + + vector data(3 * 4); + data[3 * 2 + 2] = 1; + data[3 * 3 + 2] = 7; + BOOST_CHECK(callContractFunction("f()") == encodeArgs(data)); +} + +BOOST_AUTO_TEST_CASE(memory_arrays_dynamic_index_access_write) +{ + char const* sourceCode = R"( + contract Test { + uint24[3][][4] data; + function set(uint24[3][][4] x) internal returns (uint24[3][][4]) { + x[1][2][2] = 1; + x[1][3][2] = 7; + return x; + } + function f() returns (uint24[3][]) { + data[1].length = 4; + return set(data)[1]; + } + } + )"; + compileAndRun(sourceCode, 0, "Test"); + + vector data(3 * 4); + data[3 * 2 + 2] = 1; + data[3 * 3 + 2] = 7; + BOOST_CHECK(callContractFunction("f()") == encodeArgs(u256(0x20), u256(4), data)); +} + BOOST_AUTO_TEST_SUITE_END() } From 766c3ee37b5ddc10b5a199c8731ef32e2d81bb42 Mon Sep 17 00:00:00 2001 From: chriseth Date: Fri, 26 Jun 2015 20:27:56 +0200 Subject: [PATCH 28/59] Some fixes for calldata arrays. --- libsolidity/ArrayUtils.cpp | 27 ++++++++++++++++----------- libsolidity/Compiler.cpp | 9 ++++++++- libsolidity/CompilerUtils.cpp | 7 ++----- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/libsolidity/ArrayUtils.cpp b/libsolidity/ArrayUtils.cpp index 74bde70af..f13b28173 100644 --- a/libsolidity/ArrayUtils.cpp +++ b/libsolidity/ArrayUtils.cpp @@ -39,11 +39,6 @@ void ArrayUtils::copyArrayToStorage(ArrayType const& _targetType, ArrayType cons // stack layout: [source_ref] [source_byte_off] [source length] target_ref target_byte_off (top) solAssert(_targetType.location() == DataLocation::Storage, ""); - if (_sourceType.location() == DataLocation::Memory) - solAssert( - _sourceType.getBaseType()->isValueType(), - "Copying arrays of non-value-types to storage not yet implemented." - ); IntegerType uint256(256); Type const* targetBaseType = _targetType.isByteArray() ? &uint256 : &(*_targetType.getBaseType()); @@ -139,14 +134,14 @@ void ArrayUtils::copyArrayToStorage(ArrayType const& _targetType, ArrayType cons if (sourceBaseType->getCategory() == Type::Category::Array) { solAssert(byteOffsetSize == 0, "Byte offset for array as base type."); + auto const& sourceBaseArrayType = dynamic_cast(*sourceBaseType); m_context << eth::Instruction::DUP3; if (sourceIsStorage) m_context << u256(0); + else if (sourceBaseArrayType.location() == DataLocation::Memory) + m_context << eth::Instruction::MLOAD; m_context << eth::dupInstruction(sourceIsStorage ? 4 : 3) << u256(0); - copyArrayToStorage( - dynamic_cast(*targetBaseType), - dynamic_cast(*sourceBaseType) - ); + copyArrayToStorage(dynamic_cast(*targetBaseType), sourceBaseArrayType); m_context << eth::Instruction::POP << eth::Instruction::POP; } else if (directCopy) @@ -193,11 +188,18 @@ void ArrayUtils::copyArrayToStorage(ArrayType const& _targetType, ArrayType cons if (haveByteOffsetSource) incrementByteOffset(sourceBaseType->getStorageBytes(), 1, haveByteOffsetTarget ? 5 : 4); else + { + m_context << eth::swapInstruction(2 + byteOffsetSize); + if (sourceIsStorage) + m_context << sourceBaseType->getStorageSize(); + else if (_sourceType.location() == DataLocation::Memory) + m_context << sourceBaseType->memoryHeadSize(); + else + m_context << sourceBaseType->getCalldataEncodedSize(true); m_context - << eth::swapInstruction(2 + byteOffsetSize) - << (sourceIsStorage ? sourceBaseType->getStorageSize() : sourceBaseType->getCalldataEncodedSize()) << eth::Instruction::ADD << eth::swapInstruction(2 + byteOffsetSize); + } // increment target if (haveByteOffsetTarget) incrementByteOffset(targetBaseType->getStorageBytes(), byteOffsetSize, byteOffsetSize + 2); @@ -696,6 +698,9 @@ void ArrayUtils::accessIndex(ArrayType const& _arrayType, bool _doBoundsCheck) c // out-of-bounds access throws exception m_context.appendConditionalJumpTo(m_context.errorTag()); } + else if (location == DataLocation::CallData && _arrayType.isDynamicallySized()) + // remove length if present + m_context << eth::Instruction::SWAP1 << eth::Instruction::POP; // stack: m_context << eth::Instruction::SWAP1; diff --git a/libsolidity/Compiler.cpp b/libsolidity/Compiler.cpp index 788e07ef7..6ed6480f2 100644 --- a/libsolidity/Compiler.cpp +++ b/libsolidity/Compiler.cpp @@ -261,7 +261,7 @@ void Compiler::appendCalldataUnpacker( { // We do not check the calldata size, everything is zero-paddedd - //@todo this does not yet support nested arrays + //@todo this does not yet support nested dynamic arrays if (_startOffset == u256(-1)) _startOffset = u256(CompilerUtils::dataStartOffset); @@ -279,6 +279,12 @@ void Compiler::appendCalldataUnpacker( solAssert(!arrayType.getBaseType()->isDynamicallySized(), "Nested arrays not yet implemented."); if (_fromMemory) { + solAssert( + arrayType.getBaseType()->isValueType(), + "Nested memory arrays not yet implemented here." + ); + // @todo If base type is an array or struct, it is still calldata-style encoded, so + // we would have to convert it like below. solAssert(arrayType.location() == DataLocation::Memory, ""); // compute data pointer m_context << eth::Instruction::DUP1 << eth::Instruction::MLOAD; @@ -311,6 +317,7 @@ void Compiler::appendCalldataUnpacker( } if (arrayType.location() == DataLocation::Memory) { + // stack: calldata_ref [length] next_calldata // copy to memory // move calldata type up again CompilerUtils(m_context).moveIntoStack(calldataType->getSizeOnStack()); diff --git a/libsolidity/CompilerUtils.cpp b/libsolidity/CompilerUtils.cpp index b07e14c61..208d7cecc 100644 --- a/libsolidity/CompilerUtils.cpp +++ b/libsolidity/CompilerUtils.cpp @@ -390,6 +390,7 @@ void CompilerUtils::convertType(Type const& _typeOnStack, Type const& _targetTyp // stack: (variably sized) if (targetType.getBaseType()->isValueType()) { + solAssert(typeOnStack.getBaseType()->isValueType(), ""); copyToStackTop(2 + stackSize, stackSize); if (fromStorage) m_context << u256(0); // add byte offset again @@ -406,11 +407,7 @@ void CompilerUtils::convertType(Type const& _typeOnStack, Type const& _targetTyp auto loopEnd = m_context.appendConditionalJump(); copyToStackTop(3 + stackSize, stackSize); copyToStackTop(2 + stackSize, 1); - ArrayUtils(m_context).accessIndex(typeOnStack); - MemoryItem(m_context, *typeOnStack.getBaseType(), true).retrieveValue( - SourceLocation(), - true - ); + ArrayUtils(m_context).accessIndex(typeOnStack, false); convertType(*typeOnStack.getBaseType(), *targetType.getBaseType(), _cleanupNeeded); storeInMemoryDynamic(*targetType.getBaseType(), true); m_context << eth::Instruction::SWAP1 << u256(1) << eth::Instruction::ADD; From 9935fe75da7f784d7abd5f97ef72a7f069d18904 Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Fri, 26 Jun 2015 21:22:33 +0200 Subject: [PATCH 29/59] Fix a warning in ethash_cl_miner --- libethash-cl/ethash_cl_miner.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libethash-cl/ethash_cl_miner.cpp b/libethash-cl/ethash_cl_miner.cpp index ec400f7a7..b12e4d9f8 100644 --- a/libethash-cl/ethash_cl_miner.cpp +++ b/libethash-cl/ethash_cl_miner.cpp @@ -416,6 +416,7 @@ bool ethash_cl_miner::init( void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook& hook, unsigned _msPerBatch) { + (void)_msPerBatch; try { struct pending_batch From f95baf2cb9adfb342cf6eeb2d1d68019413c3855 Mon Sep 17 00:00:00 2001 From: chriseth Date: Fri, 26 Jun 2015 18:35:43 +0200 Subject: [PATCH 30/59] Delete for memory objects. --- libsolidity/ExpressionCompiler.cpp | 22 +++++++++++--- libsolidity/LValue.cpp | 13 ++------- test/libsolidity/SolidityEndToEndTest.cpp | 29 +++++++++++++++++++ .../SolidityNameAndTypeResolution.cpp | 26 +++++++++++++++++ 4 files changed, 75 insertions(+), 15 deletions(-) diff --git a/libsolidity/ExpressionCompiler.cpp b/libsolidity/ExpressionCompiler.cpp index 7ddcc0318..59907b14b 100644 --- a/libsolidity/ExpressionCompiler.cpp +++ b/libsolidity/ExpressionCompiler.cpp @@ -679,10 +679,24 @@ void ExpressionCompiler::endVisit(MemberAccess const& _memberAccess) case Type::Category::Struct: { StructType const& type = dynamic_cast(*_memberAccess.getExpression().getType()); - m_context << eth::Instruction::POP; // structs always align to new slot - pair const& offsets = type.getStorageOffsetsOfMember(member); - m_context << offsets.first << eth::Instruction::ADD << u256(offsets.second); - setLValueToStorageItem(_memberAccess); + switch (type.location()) + { + case DataLocation::Storage: + { + m_context << eth::Instruction::POP; // structs always align to new slot + pair const& offsets = type.getStorageOffsetsOfMember(member); + m_context << offsets.first << eth::Instruction::ADD << u256(offsets.second); + setLValueToStorageItem(_memberAccess); + break; + } + case DataLocation::Memory: + { + solAssert(false, "Member access for memory structs not yet implemented."); + break; + } + default: + solAssert(false, "Illegal data location for struct."); + } break; } case Type::Category::Enum: diff --git a/libsolidity/LValue.cpp b/libsolidity/LValue.cpp index cf640e910..7eec478b3 100644 --- a/libsolidity/LValue.cpp +++ b/libsolidity/LValue.cpp @@ -69,17 +69,8 @@ void StackVariable::storeValue(Type const&, SourceLocation const& _location, boo void StackVariable::setToZero(SourceLocation const& _location, bool) const { - unsigned stackDiff = m_context.baseToCurrentStackOffset(m_baseStackOffset); - if (stackDiff > 16) - BOOST_THROW_EXCEPTION( - CompilerError() << - errinfo_sourceLocation(_location) << - errinfo_comment("Stack too deep, try removing local variables.") - ); - solAssert(stackDiff >= m_size - 1, ""); - for (unsigned i = 0; i < m_size; ++i) - m_context << u256(0) << eth::swapInstruction(stackDiff + 1 - i) - << eth::Instruction::POP; + CompilerUtils(m_context).pushZeroValue(m_dataType); + storeValue(m_dataType, _location, true); } MemoryItem::MemoryItem(CompilerContext& _compilerContext, Type const& _type, bool _padded): diff --git a/test/libsolidity/SolidityEndToEndTest.cpp b/test/libsolidity/SolidityEndToEndTest.cpp index a01e98cf8..9451c8cf2 100644 --- a/test/libsolidity/SolidityEndToEndTest.cpp +++ b/test/libsolidity/SolidityEndToEndTest.cpp @@ -4691,6 +4691,35 @@ BOOST_AUTO_TEST_CASE(memory_types_initialisation) BOOST_CHECK(callContractFunction("nestedStat()") == encodeArgs(vector(3 * 7))); } +BOOST_AUTO_TEST_CASE(memory_arrays_delete) +{ + char const* sourceCode = R"( + contract Test { + function del() returns (uint24[3][4]) { + uint24[3][4] memory x; + for (uint24 i = 0; i < x.length; i ++) + for (uint24 j = 0; j < x[i].length; j ++) + x[i][j] = i * 0x10 + j; + delete x[1]; + delete x[3][2]; + return x; + } + } + )"; + compileAndRun(sourceCode, 0, "Test"); + + vector data(3 * 4); + for (unsigned i = 0; i < 4; i++) + for (unsigned j = 0; j < 3; j++) + { + u256 v = 0; + if (!(i == 1 || (i == 3 && j == 2))) + v = i * 0x10 + j; + data[i * 3 + j] = v; + } + BOOST_CHECK(callContractFunction("del()") == encodeArgs(data)); +} + BOOST_AUTO_TEST_CASE(memory_arrays_index_access_write) { char const* sourceCode = R"( diff --git a/test/libsolidity/SolidityNameAndTypeResolution.cpp b/test/libsolidity/SolidityNameAndTypeResolution.cpp index 765593c59..f24930ba5 100644 --- a/test/libsolidity/SolidityNameAndTypeResolution.cpp +++ b/test/libsolidity/SolidityNameAndTypeResolution.cpp @@ -1900,6 +1900,18 @@ BOOST_AUTO_TEST_CASE(storage_location_local_variables) BOOST_CHECK_NO_THROW(parseTextAndResolveNames(sourceCode)); } +BOOST_AUTO_TEST_CASE(no_mappings_in_memory_array) +{ + char const* sourceCode = R"( + contract C { + function f() { + mapping(uint=>uint)[] memory x; + } + } + )"; + BOOST_CHECK_THROW(parseTextAndResolveNames(sourceCode), TypeError); +} + BOOST_AUTO_TEST_CASE(assignment_mem_to_local_storage_variable) { char const* sourceCode = R"( @@ -1931,6 +1943,20 @@ BOOST_AUTO_TEST_CASE(storage_assign_to_different_local_variable) BOOST_CHECK_THROW(parseTextAndResolveNames(sourceCode), TypeError); } +BOOST_AUTO_TEST_CASE(no_delete_on_storage_pointers) +{ + char const* sourceCode = R"( + contract C { + uint[] data; + function f() { + var x = data; + delete x; + } + } + )"; + BOOST_CHECK_THROW(parseTextAndResolveNames(sourceCode), TypeError); +} + BOOST_AUTO_TEST_CASE(assignment_mem_storage_variable_directly) { char const* sourceCode = R"( From 97180a1305951d6451537a5f3defcbbafb690831 Mon Sep 17 00:00:00 2001 From: chriseth Date: Fri, 26 Jun 2015 19:14:26 +0200 Subject: [PATCH 31/59] No delete on storage pointers. --- libsolidity/Types.cpp | 29 +++++++++++++++++------------ libsolidity/Types.h | 3 +-- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/libsolidity/Types.cpp b/libsolidity/Types.cpp index 15c367421..858828ba0 100644 --- a/libsolidity/Types.cpp +++ b/libsolidity/Types.cpp @@ -671,6 +671,23 @@ TypePointer ContractType::unaryOperatorResult(Token::Value _operator) const return _operator == Token::Delete ? make_shared() : TypePointer(); } +TypePointer ReferenceType::unaryOperatorResult(Token::Value _operator) const +{ + if (_operator != Token::Delete) + return TypePointer(); + // delete can be used on everything except calldata references or storage pointers + // (storage references are ok) + switch (location()) + { + case DataLocation::CallData: + return TypePointer(); + case DataLocation::Memory: + return make_shared(); + case DataLocation::Storage: + return m_isPointer ? TypePointer() : make_shared(); + } +} + TypePointer ReferenceType::copyForLocationIfReference(DataLocation _location, TypePointer const& _type) { if (auto type = dynamic_cast(_type.get())) @@ -738,13 +755,6 @@ bool ArrayType::isImplicitlyConvertibleTo(const Type& _convertTo) const } } -TypePointer ArrayType::unaryOperatorResult(Token::Value _operator) const -{ - if (_operator == Token::Delete) - return make_shared(); - return TypePointer(); -} - bool ArrayType::operator==(Type const& _other) const { if (_other.getCategory() != getCategory()) @@ -962,11 +972,6 @@ bool StructType::isImplicitlyConvertibleTo(const Type& _convertTo) const return this->m_struct == convertTo.m_struct; } -TypePointer StructType::unaryOperatorResult(Token::Value _operator) const -{ - return _operator == Token::Delete ? make_shared() : TypePointer(); -} - bool StructType::operator==(Type const& _other) const { if (_other.getCategory() != getCategory()) diff --git a/libsolidity/Types.h b/libsolidity/Types.h index 9d9aaaba7..d15e2f963 100644 --- a/libsolidity/Types.h +++ b/libsolidity/Types.h @@ -376,6 +376,7 @@ public: explicit ReferenceType(DataLocation _location): m_location(_location) {} DataLocation location() const { return m_location; } + virtual TypePointer unaryOperatorResult(Token::Value _operator) const override; virtual unsigned memoryHeadSize() const override { return 32; } /// @returns a copy of this type with location (recursively) changed to @a _location, @@ -444,7 +445,6 @@ public: {} virtual bool isImplicitlyConvertibleTo(Type const& _convertTo) const override; - virtual TypePointer unaryOperatorResult(Token::Value _operator) const override; virtual bool operator==(const Type& _other) const override; virtual unsigned getCalldataEncodedSize(bool _padded) const override; virtual bool isDynamicallySized() const override { return m_hasDynamicLength; } @@ -545,7 +545,6 @@ public: //@todo only storage until we have non-storage structs ReferenceType(DataLocation::Storage), m_struct(_struct) {} virtual bool isImplicitlyConvertibleTo(const Type& _convertTo) const override; - virtual TypePointer unaryOperatorResult(Token::Value _operator) const override; virtual bool operator==(Type const& _other) const override; virtual unsigned getCalldataEncodedSize(bool _padded) const override; virtual u256 getStorageSize() const override; From 1f5c86d01ab83a93671859a4e8e5ce82762f9edb Mon Sep 17 00:00:00 2001 From: chriseth Date: Fri, 26 Jun 2015 21:27:53 +0200 Subject: [PATCH 32/59] Disallow memory types containing mappings. --- libsolidity/AST.cpp | 11 ++++++++++- libsolidity/Types.h | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/libsolidity/AST.cpp b/libsolidity/AST.cpp index 09af49c67..2b73aab5e 100644 --- a/libsolidity/AST.cpp +++ b/libsolidity/AST.cpp @@ -535,7 +535,16 @@ void VariableDeclaration::checkTypeRequirements() BOOST_THROW_EXCEPTION(createTypeError("Variable cannot have void type.")); m_type = type->mobileType(); } - if (m_isStateVariable && getVisibility() >= Visibility::Public && !FunctionType(*this).externalType()) + solAssert(!!m_type, ""); + if (!m_isStateVariable) + { + if (m_type->dataStoredIn(DataLocation::Memory) || m_type->dataStoredIn(DataLocation::CallData)) + if (!m_type->canLiveOutsideStorage()) + BOOST_THROW_EXCEPTION(createTypeError( + "Type " + m_type->toString() + " is only valid in storage." + )); + } + else if (getVisibility() >= Visibility::Public && !FunctionType(*this).externalType()) BOOST_THROW_EXCEPTION(createTypeError("Internal type is not allowed for public state variables.")); } diff --git a/libsolidity/Types.h b/libsolidity/Types.h index d15e2f963..6b03b1ae2 100644 --- a/libsolidity/Types.h +++ b/libsolidity/Types.h @@ -449,6 +449,7 @@ public: virtual unsigned getCalldataEncodedSize(bool _padded) const override; virtual bool isDynamicallySized() const override { return m_hasDynamicLength; } virtual u256 getStorageSize() const override; + virtual bool canLiveOutsideStorage() const override { return m_baseType->canLiveOutsideStorage(); } virtual unsigned getSizeOnStack() const override; virtual std::string toString(bool _short) const override; virtual MemberList const& getMembers() const override From 2951cbd3cbba7ebd14eb24338361f6a0393fda77 Mon Sep 17 00:00:00 2001 From: chriseth Date: Sat, 27 Jun 2015 00:13:53 +0200 Subject: [PATCH 33/59] Fix gcc issue. --- libsolidity/Types.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libsolidity/Types.cpp b/libsolidity/Types.cpp index 858828ba0..a6d462a9a 100644 --- a/libsolidity/Types.cpp +++ b/libsolidity/Types.cpp @@ -685,6 +685,8 @@ TypePointer ReferenceType::unaryOperatorResult(Token::Value _operator) const return make_shared(); case DataLocation::Storage: return m_isPointer ? TypePointer() : make_shared(); + default: + solAssert(false, ""); } } From ed80e1322be01a74a96e41aadf7364fab0090c03 Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 26 Jun 2015 19:36:39 -0400 Subject: [PATCH 34/59] windows build fix --- eth/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/main.cpp b/eth/main.cpp index 235183ef6..64679762d 100644 --- a/eth/main.cpp +++ b/eth/main.cpp @@ -820,7 +820,7 @@ int main(int argc, char** argv) while (web3.ethereum()->blockQueue().items().first + web3.ethereum()->blockQueue().items().second > 0) { - sleep(1); + this_thread::sleep_for(chrono::seconds(1)); web3.ethereum()->syncQueue(100000); } double e = chrono::duration_cast(chrono::steady_clock::now() - t).count() / 1000.0; From 7beccf2a8dc8549c9ec413e8bce7861aed37ac3f Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 26 Jun 2015 23:49:52 -0400 Subject: [PATCH 35/59] Disable libsecp256k1-specific tests for windows compile. --- test/libdevcrypto/crypto.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/libdevcrypto/crypto.cpp b/test/libdevcrypto/crypto.cpp index 776c0a88e..135292506 100644 --- a/test/libdevcrypto/crypto.cpp +++ b/test/libdevcrypto/crypto.cpp @@ -22,7 +22,9 @@ */ #include +#if ETH_HAVE_SECP256K1 #include +#endif #include #include #include @@ -64,6 +66,7 @@ BOOST_AUTO_TEST_CASE(emptySHA3Types) BOOST_REQUIRE_EQUAL(emptyListSHA3, EmptyListSHA3); } +#if ETH_HAVE_SECP256K1 BOOST_AUTO_TEST_CASE(secp256k1lib) { secp256k1Init(); @@ -73,6 +76,7 @@ BOOST_AUTO_TEST_CASE(secp256k1lib) Public test = toPublic(k.sec()); BOOST_REQUIRE(k.pub() == test); } +#endif BOOST_AUTO_TEST_CASE(cryptopp_patch) { @@ -111,7 +115,9 @@ BOOST_AUTO_TEST_CASE(common_encrypt_decrypt) BOOST_AUTO_TEST_CASE(cryptopp_cryptopp_secp256k1libport) { +#if ETH_HAVE_SECP256K1 secp256k1_start(); +#endif // base secret Secret secret(sha3("privacy")); @@ -160,7 +166,9 @@ BOOST_AUTO_TEST_CASE(cryptopp_cryptopp_secp256k1libport) byte dersig[72]; size_t cssz = DSAConvertSignatureFormat(dersig, 72, DSA_DER, sig.data(), 64, DSA_P1363); BOOST_CHECK(cssz <= 72); +#if ETH_HAVE_SECP256K1 BOOST_REQUIRE(1 == secp256k1_ecdsa_verify(he.data(), sizeof(he), dersig, cssz, encpub, 65)); +#endif } } @@ -759,7 +767,6 @@ BOOST_AUTO_TEST_CASE(cryptopp_aes128_cbc) BOOST_AUTO_TEST_CASE(eth_keypairs) { cnote << "Testing Crypto..."; - secp256k1_start(); KeyPair p(Secret(fromHex("3ecb44df2159c26e0f995712d4f39b6f6e499b40749b1cf1246c37f9516cb6a4"))); BOOST_REQUIRE(p.pub() == Public(fromHex("97466f2b32bc3bb76d4741ae51cd1d8578b48d3f1e68da206d47321aec267ce78549b514e4453d74ef11b0cd5e4e4c364effddac8b51bcfc8de80682f952896f"))); @@ -783,7 +790,6 @@ BOOST_AUTO_TEST_CASE(eth_keypairs) int cryptoTest() { cnote << "Testing Crypto..."; - secp256k1_start(); KeyPair p(Secret(fromHex("3ecb44df2159c26e0f995712d4f39b6f6e499b40749b1cf1246c37f9516cb6a4"))); BOOST_REQUIRE(p.pub() == Public(fromHex("97466f2b32bc3bb76d4741ae51cd1d8578b48d3f1e68da206d47321aec267ce78549b514e4453d74ef11b0cd5e4e4c364effddac8b51bcfc8de80682f952896f"))); From 146949822c94fad535d67d900d26f26eee7b077a Mon Sep 17 00:00:00 2001 From: subtly Date: Sat, 27 Jun 2015 04:31:06 -0400 Subject: [PATCH 36/59] Check write status for errors in BlockChain and log error so IO problem doesn't put database or client into invalid state. --- libethereum/BlockChain.cpp | 39 +++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 9bf89665a..9114b1a58 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -96,6 +96,15 @@ ldb::Slice dev::eth::toSlice(h256 const& _h, unsigned _sub) #endif } +namespace dev +{ +class WriteBatchNoter: public ldb::WriteBatch::Handler +{ + virtual void Put(ldb::Slice const& _key, ldb::Slice const& _value) { cout << "Put" << toHex(bytesConstRef(_key)) << "=>" << toHex(bytesConstRef(_value)); } + virtual void Delete(ldb::Slice const& _key) { cout << "Delete" << toHex(bytesConstRef(_key)); } +}; +} + #if ETH_DEBUG&&0 static const chrono::system_clock::duration c_collectionDuration = chrono::seconds(15); static const unsigned c_collectionQueueSize = 2; @@ -638,8 +647,25 @@ ImportRoute BlockChain::import(VerifiedBlockRef const& _block, OverlayDB const& clog(BlockChainChat) << " Imported but not best (oTD:" << details(last).totalDifficulty << " > TD:" << td << ")"; } - m_blocksDB->Write(m_writeOptions, &blocksBatch); - m_extrasDB->Write(m_writeOptions, &extrasBatch); + ldb::Status o = m_blocksDB->Write(m_writeOptions, &blocksBatch); + if (!o.ok()) + { + cwarn << "Error writing to blockchain database: " << o.ToString(); + WriteBatchNoter n; + blocksBatch.Iterate(&n); + cwarn << "Fail writing to blockchain database. Bombing out."; + exit(-1); + } + + o = m_extrasDB->Write(m_writeOptions, &extrasBatch); + if (!o.ok()) + { + cwarn << "Error writing to extras database: " << o.ToString(); + WriteBatchNoter n; + extrasBatch.Iterate(&n); + cwarn << "Fail writing to extras database. Bombing out."; + exit(-1); + } #if ETH_PARANOIA || !ETH_TRUE if (isKnown(_block.info.hash()) && !details(_block.info.hash())) @@ -667,7 +693,14 @@ ImportRoute BlockChain::import(VerifiedBlockRef const& _block, OverlayDB const& { m_lastBlockHash = newLastBlockHash; m_lastBlockNumber = newLastBlockNumber; - m_extrasDB->Put(m_writeOptions, ldb::Slice("best"), ldb::Slice((char const*)&m_lastBlockHash, 32)); + o = m_extrasDB->Put(m_writeOptions, ldb::Slice("best"), ldb::Slice((char const*)&m_lastBlockHash, 32)); + if (!o.ok()) + { + cwarn << "Error writing to extras database: " << o.ToString(); + cout << "Put" << toHex(bytesConstRef(ldb::Slice("best"))) << "=>" << toHex(bytesConstRef(ldb::Slice((char const*)&m_lastBlockHash, 32))); + cwarn << "Fail writing to extras database. Bombing out."; + exit(-1); + } } #if ETH_PARANOIA || !ETH_TRUE From 3217e8dd16c5643cf29de8424b99f2bf7bccd606 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 27 Jun 2015 13:36:54 +0200 Subject: [PATCH 37/59] prevent syncing with busy peers & extra logging --- libethereum/BlockChainSync.cpp | 2 +- libethereum/EthereumPeer.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/libethereum/BlockChainSync.cpp b/libethereum/BlockChainSync.cpp index da341867a..8a402bb74 100644 --- a/libethereum/BlockChainSync.cpp +++ b/libethereum/BlockChainSync.cpp @@ -534,7 +534,7 @@ bool PV60Sync::shouldGrabBlocks(std::shared_ptr _peer) const void PV60Sync::attemptSync(std::shared_ptr _peer) { - if (m_state != SyncState::Idle) + if (m_state != SyncState::Idle || _peer->m_asking != Asking::Nothing) { clog(NetAllDetail) << "Can't sync with this peer - outstanding asks."; return; diff --git a/libethereum/EthereumPeer.cpp b/libethereum/EthereumPeer.cpp index 30c2bfc9a..88878334c 100644 --- a/libethereum/EthereumPeer.cpp +++ b/libethereum/EthereumPeer.cpp @@ -152,6 +152,8 @@ void EthereumPeer::requestHashes(u256 _number, unsigned _count) void EthereumPeer::requestHashes(h256 const& _lastHash) { + if (m_asking != Asking::Nothing) + clog(NetWarn) << "Asking hashes while requesting " << (m_asking == Asking::Nothing ? "nothing" : m_asking == Asking::State ? "state" : m_asking == Asking::Hashes ? "hashes" : m_asking == Asking::Blocks ? "blocks" : "?"); assert(m_asking == Asking::Nothing); setAsking(Asking::Hashes); RLPStream s; From f634b539fa73288390b3e74ad625693f3b12bd29 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 27 Jun 2015 19:18:22 +0200 Subject: [PATCH 38/59] removed superfluous transition --- libethereum/BlockChainSync.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libethereum/BlockChainSync.cpp b/libethereum/BlockChainSync.cpp index 8a402bb74..63ac620ce 100644 --- a/libethereum/BlockChainSync.cpp +++ b/libethereum/BlockChainSync.cpp @@ -649,9 +649,8 @@ void PV60Sync::noteDoneBlocks(std::shared_ptr _peer, bool _clemenc } resetSync(); downloadMan().reset(); - transition(_peer, SyncState::Idle); - } _peer->m_sub.doneFetch(); + } } void PV60Sync::onPeerHashes(std::shared_ptr _peer, h256s const& _hashes) From 19d3ad4ea5a4b26e8581a0fff4a4d9b589f3ca3a Mon Sep 17 00:00:00 2001 From: subtly Date: Sun, 28 Jun 2015 03:39:09 -0400 Subject: [PATCH 39/59] s/cout/cnote/ --- libethereum/BlockChain.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 9114b1a58..55c213a64 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -100,8 +100,8 @@ namespace dev { class WriteBatchNoter: public ldb::WriteBatch::Handler { - virtual void Put(ldb::Slice const& _key, ldb::Slice const& _value) { cout << "Put" << toHex(bytesConstRef(_key)) << "=>" << toHex(bytesConstRef(_value)); } - virtual void Delete(ldb::Slice const& _key) { cout << "Delete" << toHex(bytesConstRef(_key)); } + virtual void Put(ldb::Slice const& _key, ldb::Slice const& _value) { cnote << "Put" << toHex(bytesConstRef(_key)) << "=>" << toHex(bytesConstRef(_value)); } + virtual void Delete(ldb::Slice const& _key) { cnote << "Delete" << toHex(bytesConstRef(_key)); } }; } From 4cdea67cea1c03977442a6b1b6180d8a479adcb8 Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 29 Jun 2015 12:47:04 +0200 Subject: [PATCH 40/59] Another test for arrays in constructors. --- test/libsolidity/SolidityEndToEndTest.cpp | 41 ++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/test/libsolidity/SolidityEndToEndTest.cpp b/test/libsolidity/SolidityEndToEndTest.cpp index 9451c8cf2..a2dbc35e6 100644 --- a/test/libsolidity/SolidityEndToEndTest.cpp +++ b/test/libsolidity/SolidityEndToEndTest.cpp @@ -4493,7 +4493,7 @@ BOOST_AUTO_TEST_CASE(bytes_in_constructors_packer) } } contract Main is Base { - function Main(bytes s, uint x) Base(x, s){}//f(s)) {} + function Main(bytes s, uint x) Base(x, f(s)) {} function f(bytes s) returns (bytes) { return s; } @@ -4517,6 +4517,45 @@ BOOST_AUTO_TEST_CASE(bytes_in_constructors_packer) ); } +BOOST_AUTO_TEST_CASE(arrays_in_constructors) +{ + char const* sourceCode = R"( + contract Base { + uint public m_x; + address[] m_s; + function Base(uint x, address[] s) { + m_x = x; + m_s = s; + } + function part(uint i) returns (address) { + return m_s[i]; + } + } + contract Main is Base { + function Main(address[] s, uint x) Base(x, f(s)) {} + function f(address[] s) returns (address[]) { + return s; + } + } + contract Creator { + function f(uint x, address[] s) returns (uint r, address ch) { + var c = new Main(s, x); + r = c.m_x(); + ch = c.part(x); + } + } + )"; + compileAndRun(sourceCode, 0, "Creator"); + vector s1{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + bytes dyn1 = encodeArgs(u256(s1.size()), s1); + u256 x = 7; + bytes args1 = encodeArgs(x, u256(0x40)) + dyn1; + BOOST_REQUIRE( + callContractFunction("f(uint256,address[])", asString(args1)) == + encodeArgs(x, s1[unsigned(x)]) + ); +} + BOOST_AUTO_TEST_CASE(arrays_from_and_to_storage) { char const* sourceCode = R"( From 2e97a2da5480afae3ec5f8a1bf7aff6f5fa7c153 Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 29 Jun 2015 12:46:31 +0200 Subject: [PATCH 41/59] Updated wallet test to latest wallet source. --- test/libsolidity/SolidityWallet.cpp | 31 +++++++++++++++++++---------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/test/libsolidity/SolidityWallet.cpp b/test/libsolidity/SolidityWallet.cpp index 09820b87b..ba8f198f8 100644 --- a/test/libsolidity/SolidityWallet.cpp +++ b/test/libsolidity/SolidityWallet.cpp @@ -200,17 +200,17 @@ contract multiowned { } // the number of owners that must confirm the same operation before it is run. - uint m_required; + uint public m_required; // pointer used to find a free slot in m_owners - uint m_numOwners; + uint public m_numOwners; // list of owners - uint[256] m_owners; + uint[256] public m_owners; uint constant c_maxOwners = 250; // index on the list of owners to allow reverse lookup - mapping(uint => uint) m_ownerIndex; + mapping(uint => uint) public m_ownerIndex; // the ongoing operations. - mapping(bytes32 => PendingState) m_pending; - bytes32[] m_pendingIndex; + mapping(bytes32 => PendingState) public m_pending; + bytes32[] public m_pendingIndex; } // inheritable "property" contract that enables methods to be protected by placing a linear limit (specifiable) @@ -251,9 +251,9 @@ contract daylimit is multiowned { } // determines today's index. function today() private constant returns (uint) { return now / 1 days; } - uint m_spentToday; - uint m_dailyLimit; - uint m_lastDay; + uint public m_spentToday; + uint public m_dailyLimit; + uint public m_lastDay; } // interface contract for multisig proxy contracts; see below for docs. contract multisig { @@ -275,11 +275,14 @@ contract Wallet is multisig, multiowned, daylimit { uint value; bytes data; } + /* // logged events: // Funds has arrived into the wallet (record how much). event Deposit(address from, uint value); // Single transaction going out of the wallet (record who signed for it, how much, and to whom it's going). event SingleTransact(address owner, uint value, address to, bytes data); + // Multi-sig transaction going out of the wallet (record who signed for it last, the operation hash, how much, and to whom it's going). + event MultiTransact(address owner, bytes32 operation, uint value, address to, bytes data);*/ // constructor - just pass on the owner arra to the multiowned. event Created(); function Wallet() { @@ -299,7 +302,7 @@ contract Wallet is multisig, multiowned, daylimit { // If not, goes into multisig process. We provide a hash on return to allow the sender to provide // shortcuts for the other confirmations (allowing them to avoid replicating the _to, _value // and _data arguments). They still get the option of using them if they want, anyways. - function execute(address _to, uint _value, bytes _data) onlyowner external returns (bytes32 _r) { + function execute(address _to, uint _value, bytes _data) external onlyowner returns (bytes32 _r) { // first, take the opportunity to check that we're under the daily limit. if (underLimit(_value)) { SingleTransact(msg.sender, _value, _to, _data); @@ -332,8 +335,14 @@ contract Wallet is multisig, multiowned, daylimit { delete m_txs[m_pendingIndex[i]]; super.clearPending(); } + // // internally confirm transaction with all of the info. returns true iff confirmed good and executed. + // function confirmVerbose(bytes32 _h, address _to, uint _value, bytes _data) private onlymanyowners(_h) returns (bool) { + // _to.call.value(_value)(_data); + // MultiTransact("out", msg.sender, _h, _value, _to); + // return true; + // } // pending transactions we have at present. - mapping (bytes32 => Transaction) m_txs; + mapping (bytes32 => Transaction) public m_txs; } )DELIMITER"; From 6f649196987bf2732fd888bd9d652ed23f0483e7 Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 29 Jun 2015 13:23:02 +0200 Subject: [PATCH 42/59] Another compiler fix. --- libsolidity/Types.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libsolidity/Types.cpp b/libsolidity/Types.cpp index a6d462a9a..c1769d000 100644 --- a/libsolidity/Types.cpp +++ b/libsolidity/Types.cpp @@ -688,6 +688,7 @@ TypePointer ReferenceType::unaryOperatorResult(Token::Value _operator) const default: solAssert(false, ""); } + return TypePointer(); } TypePointer ReferenceType::copyForLocationIfReference(DataLocation _location, TypePointer const& _type) From baf4d450eb803cbb8fa40f4228a4d8107021e3f1 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Mon, 29 Jun 2015 15:29:58 +0200 Subject: [PATCH 43/59] a minor refactoring --- libwhisper/Interface.h | 1 - libwhisper/WhisperHost.h | 6 ++++-- test/libwhisper/bloomFilter.cpp | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/libwhisper/Interface.h b/libwhisper/Interface.h index 941cf3434..f53cb17a7 100644 --- a/libwhisper/Interface.h +++ b/libwhisper/Interface.h @@ -67,7 +67,6 @@ public: virtual Topics const& fullTopics(unsigned _id) const = 0; virtual unsigned installWatch(Topics const& _filter) = 0; -// virtual unsigned installWatchOnId(h256 _filterId) = 0; virtual void uninstallWatch(unsigned _watchId) = 0; virtual h256s peekWatch(unsigned _watchId) const = 0; virtual h256s checkWatch(unsigned _watchId) = 0; diff --git a/libwhisper/WhisperHost.h b/libwhisper/WhisperHost.h index 386153ac5..1a43eda3a 100644 --- a/libwhisper/WhisperHost.h +++ b/libwhisper/WhisperHost.h @@ -51,7 +51,8 @@ public: WhisperHost(); virtual ~WhisperHost(); unsigned protocolVersion() const { return WhisperProtocolVersion; } - void cleanup(); ///< remove old messages + /// remove old messages + void cleanup(); std::map all() const { dev::ReadGuard l(x_messages); return m_messages; } FixedHash bloom() const { dev::Guard l(m_filterLock); return m_bloom; } @@ -61,7 +62,8 @@ public: virtual void uninstallWatch(unsigned _watchId) override; virtual h256s peekWatch(unsigned _watchId) const override { dev::Guard l(m_filterLock); try { return m_watches.at(_watchId).changes; } catch (...) { return h256s(); } } virtual h256s checkWatch(unsigned _watchId) override { cleanup(); dev::Guard l(m_filterLock); h256s ret; try { ret = m_watches.at(_watchId).changes; m_watches.at(_watchId).changes.clear(); } catch (...) {} return ret; } - virtual h256s watchMessages(unsigned _watchId) override; ///< returns IDs of messages, which match specific watch criteria + /// returns IDs of messages, which match specific watch criteria + virtual h256s watchMessages(unsigned _watchId) override; virtual Envelope envelope(h256 _m) const override { try { dev::ReadGuard l(x_messages); return m_messages.at(_m); } catch (...) { return Envelope(); } } protected: diff --git a/test/libwhisper/bloomFilter.cpp b/test/libwhisper/bloomFilter.cpp index 7fe363033..3e71ca305 100644 --- a/test/libwhisper/bloomFilter.cpp +++ b/test/libwhisper/bloomFilter.cpp @@ -58,7 +58,7 @@ void testRemoveExistingBloom(TopicBloomFilterShort& _f, AbridgedTopic const& _h) BOOST_REQUIRE(!_f.containsBloom(_h)); } -double calculateExpected(TopicBloomFilterTest const& f, int const n) +double calculateExpected(TopicBloomFilterTest const& f, int n) { int const m = f.size * 8; // number of bits in the bloom int const k = f.BitsPerBloom; // number of hash functions (e.g. bits set to 1 in every bloom) @@ -79,7 +79,7 @@ double calculateExpected(TopicBloomFilterTest const& f, int const n) return kBitsSet; } -double testFalsePositiveRate(TopicBloomFilterTest const& f, int const inserted, Topic& x) +double testFalsePositiveRate(TopicBloomFilterTest const& f, int inserted, Topic& x) { int const c_sampleSize = 1000; int falsePositive = 0; @@ -114,7 +114,7 @@ BOOST_AUTO_TEST_CASE(falsePositiveRate) double expectedRate = 0; - for (int i = 1; i < 50 && expectedRate < 0.5; ++i) + for (int i = 1; i < 50 && isless(expectedRate, 0.5); ++i) { x = sha3(x); f.addBloom(AbridgedTopic(x)); From 45ccb4cc84b6e83ce54820c924d810424fa38430 Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Mon, 29 Jun 2015 16:21:19 +0200 Subject: [PATCH 44/59] Correctly set GPU instances in non-zero platforms Closes #2315 --- ethminer/MinerAux.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethminer/MinerAux.h b/ethminer/MinerAux.h index e123cdbb1..ec6ee57e7 100644 --- a/ethminer/MinerAux.h +++ b/ethminer/MinerAux.h @@ -265,7 +265,6 @@ public: ProofOfWork::CPUMiner::setNumInstances(m_miningThreads); else if (m_minerType == MinerType::GPU) { - ProofOfWork::GPUMiner::setNumInstances(m_miningThreads); if (!ProofOfWork::GPUMiner::configureGPU( m_openclPlatform, m_openclDevice, @@ -277,6 +276,7 @@ public: cout << "No GPU device with sufficient memory was found. Can't GPU mine. Remove the -G argument" << endl; exit(1); } + ProofOfWork::GPUMiner::setNumInstances(m_miningThreads); } if (mode == OperationMode::DAGInit) doInitDAG(m_initDAG); From 81ca67dd4c9536d7bf527e601c2855ed58905bfa Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Mon, 29 Jun 2015 16:55:57 +0200 Subject: [PATCH 45/59] configureGPU() should respect given platformID Related to #2314 --- libethash-cl/ethash_cl_miner.cpp | 3 ++- libethash-cl/ethash_cl_miner.h | 1 + libethcore/Ethash.cpp | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libethash-cl/ethash_cl_miner.cpp b/libethash-cl/ethash_cl_miner.cpp index b12e4d9f8..3a72810fa 100644 --- a/libethash-cl/ethash_cl_miner.cpp +++ b/libethash-cl/ethash_cl_miner.cpp @@ -139,6 +139,7 @@ unsigned ethash_cl_miner::getNumDevices(unsigned _platformId) } bool ethash_cl_miner::configureGPU( + unsigned _platformId, bool _allowCPU, unsigned _extraGPUMemory, boost::optional _currentBlock @@ -149,7 +150,7 @@ bool ethash_cl_miner::configureGPU( // by default let's only consider the DAG of the first epoch uint64_t dagSize = _currentBlock ? ethash_get_datasize(*_currentBlock) : 1073739904U; uint64_t requiredSize = dagSize + _extraGPUMemory; - return searchForAllDevices([&requiredSize](cl::Device const _device) -> bool + return searchForAllDevices(_platformId, [&requiredSize](cl::Device const _device) -> bool { cl_ulong result; _device.getInfo(CL_DEVICE_GLOBAL_MEM_SIZE, &result); diff --git a/libethash-cl/ethash_cl_miner.h b/libethash-cl/ethash_cl_miner.h index 996453c00..73bf7e94a 100644 --- a/libethash-cl/ethash_cl_miner.h +++ b/libethash-cl/ethash_cl_miner.h @@ -44,6 +44,7 @@ public: static std::string platform_info(unsigned _platformId = 0, unsigned _deviceId = 0); static void listDevices(); static bool configureGPU( + unsigned _platformId, bool _allowCPU, unsigned _extraGPUMemory, boost::optional _currentBlock diff --git a/libethcore/Ethash.cpp b/libethcore/Ethash.cpp index 70908ee44..46d19d164 100644 --- a/libethcore/Ethash.cpp +++ b/libethcore/Ethash.cpp @@ -418,7 +418,7 @@ bool Ethash::GPUMiner::configureGPU( { s_platformId = _platformId; s_deviceId = _deviceId; - return ethash_cl_miner::configureGPU(_allowCPU, _extraGPUMemory, _currentBlock); + return ethash_cl_miner::configureGPU(_platformId, _allowCPU, _extraGPUMemory, _currentBlock); } #endif From 0ce9b446d9d6b06b7e944f867786956ed7f128fb Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Mon, 29 Jun 2015 18:47:53 +0200 Subject: [PATCH 46/59] warning fixed --- libdevcore/Common.h | 1 + 1 file changed, 1 insertion(+) diff --git a/libdevcore/Common.h b/libdevcore/Common.h index c6ed25223..384be324e 100644 --- a/libdevcore/Common.h +++ b/libdevcore/Common.h @@ -45,6 +45,7 @@ #pragma warning(push) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-fpermissive" #include #if (BOOST_VERSION == 105800) #include "boost_multiprecision_number_compare_bug_workaround.hpp" From edd52f0553c2a955e75485ba6d0d9bf5d8d80030 Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 29 Jun 2015 20:05:41 +0200 Subject: [PATCH 47/59] Memory arrays cannot be resized. --- libsolidity/AST.cpp | 7 +++++-- test/libsolidity/SolidityNameAndTypeResolution.cpp | 13 +++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/libsolidity/AST.cpp b/libsolidity/AST.cpp index 2b73aab5e..cb935183f 100644 --- a/libsolidity/AST.cpp +++ b/libsolidity/AST.cpp @@ -936,8 +936,11 @@ void MemberAccess::checkTypeRequirements(TypePointers const* _argumentTypes) else if (type.getCategory() == Type::Category::Array) { auto const& arrayType(dynamic_cast(type)); - m_isLValue = (*m_memberName == "length" && - arrayType.location() != DataLocation::CallData && arrayType.isDynamicallySized()); + m_isLValue = ( + *m_memberName == "length" && + arrayType.location() == DataLocation::Storage && + arrayType.isDynamicallySized() + ); } else m_isLValue = false; diff --git a/test/libsolidity/SolidityNameAndTypeResolution.cpp b/test/libsolidity/SolidityNameAndTypeResolution.cpp index f24930ba5..df976eaea 100644 --- a/test/libsolidity/SolidityNameAndTypeResolution.cpp +++ b/test/libsolidity/SolidityNameAndTypeResolution.cpp @@ -2026,6 +2026,19 @@ BOOST_AUTO_TEST_CASE(dynamic_return_types_not_possible) BOOST_CHECK_THROW(parseTextAndResolveNames(sourceCode), TypeError); } +BOOST_AUTO_TEST_CASE(memory_arrays_not_resizeable) +{ + char const* sourceCode = R"( + contract C { + function f() { + uint[] memory x; + x.length = 2; + } + } + )"; + BOOST_CHECK_THROW(parseTextAndResolveNames(sourceCode), TypeError); +} + BOOST_AUTO_TEST_SUITE_END() } From d95a5afee002c88e551b95709e0957c2fc855ed7 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 29 Jun 2015 22:21:53 +0200 Subject: [PATCH 48/59] fixed race condition in transaction queue --- libethereum/TransactionQueue.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libethereum/TransactionQueue.cpp b/libethereum/TransactionQueue.cpp index 8931ee218..8fcf3cfb6 100644 --- a/libethereum/TransactionQueue.cpp +++ b/libethereum/TransactionQueue.cpp @@ -220,6 +220,7 @@ bool TransactionQueue::remove_WITH_LOCK(h256 const& _txHash) unsigned TransactionQueue::waiting(Address const& _a) const { + ReadGuard l(m_lock); auto it = m_senders.equal_range(_a); unsigned ret = 0; for (auto i = it.first; i != it.second; ++i, ++ret) {} From 813c365fc421e65301296a3054f0b07d1dc26674 Mon Sep 17 00:00:00 2001 From: subtly Date: Tue, 30 Jun 2015 03:58:41 -0400 Subject: [PATCH 49/59] Only modify logger to \r \r for interactive mode and do so with thread safely. --- eth/main.cpp | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/eth/main.cpp b/eth/main.cpp index 64679762d..ba55e5664 100644 --- a/eth/main.cpp +++ b/eth/main.cpp @@ -703,12 +703,24 @@ int main(int argc, char** argv) string logbuf; std::string additional; - g_logPost = [&](std::string const& a, char const*){ - if (g_silence) - logbuf += a + "\n"; - else - cout << "\r \r" << a << endl << additional << flush; - }; + if (interactive) + g_logPost = [&](std::string const& a, char const*){ + static SpinLock s_lock; + SpinGuard l(s_lock); + + if (g_silence) + logbuf += a + "\n"; + else + cout << "\r \r" << a << endl << additional << flush; + + // helpful to use OutputDebugString on windows + #ifdef _WIN32 + { + OutputDebugStringA(_s.data()); + OutputDebugStringA("\n"); + } + #endif + }; auto getPassword = [&](string const& prompt){ auto s = g_silence; From b39d687be274a74d42e16f316b3a9ed970dd7190 Mon Sep 17 00:00:00 2001 From: subtly Date: Tue, 30 Jun 2015 05:12:39 -0400 Subject: [PATCH 50/59] fix copy/paste fail --- eth/main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/main.cpp b/eth/main.cpp index ba55e5664..4361598d4 100644 --- a/eth/main.cpp +++ b/eth/main.cpp @@ -712,11 +712,11 @@ int main(int argc, char** argv) logbuf += a + "\n"; else cout << "\r \r" << a << endl << additional << flush; - + // helpful to use OutputDebugString on windows #ifdef _WIN32 { - OutputDebugStringA(_s.data()); + OutputDebugStringA(a.data()); OutputDebugStringA("\n"); } #endif From 43f0112456b7f5d56f746f00f1eb3c1cea0616ea Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Mon, 29 Jun 2015 12:38:06 +0200 Subject: [PATCH 51/59] Mutex & reference counter --- libwhisper/WhisperHost.cpp | 5 ++++- libwhisper/WhisperPeer.cpp | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index 1a47a56d1..2d037133c 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -94,8 +94,11 @@ unsigned WhisperHost::installWatch(shh::Topics const& _t) DEV_GUARDED(m_filterLock) { - if (!m_filters.count(h)) + auto it = m_filters.find(h); + if (m_filters.end() == it) m_filters.insert(make_pair(h, f)); + else + it->second.refCount++; ret = m_watches.size() ? m_watches.rbegin()->first + 1 : 0; m_watches[ret] = ClientWatch(h); diff --git a/libwhisper/WhisperPeer.cpp b/libwhisper/WhisperPeer.cpp index 665364f49..1f48468df 100644 --- a/libwhisper/WhisperPeer.cpp +++ b/libwhisper/WhisperPeer.cpp @@ -58,7 +58,10 @@ bool WhisperPeer::interpret(unsigned _id, RLP const& _r) disable("Invalid protocol version."); for (auto const& m: host()->all()) + { + Guard l(x_unseen); m_unseen.insert(make_pair(0, m.first)); + } if (session()->id() < host()->host()->id()) sendMessages(); From c50437a5f95be117e23ff5b6f5d341628e74d680 Mon Sep 17 00:00:00 2001 From: Vlad Gluhovsky Date: Mon, 29 Jun 2015 13:01:05 +0200 Subject: [PATCH 52/59] uninstallWatch() fixed + test --- libwhisper/WhisperHost.cpp | 7 +++---- test/libwhisper/whisperTopic.cpp | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index 2d037133c..150d5cd63 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -100,11 +100,10 @@ unsigned WhisperHost::installWatch(shh::Topics const& _t) else it->second.refCount++; + m_bloom.addRaw(f.filter.exportBloom()); ret = m_watches.size() ? m_watches.rbegin()->first + 1 : 0; m_watches[ret] = ClientWatch(h); cwatshh << "+++" << ret << h; - - m_bloom.addRaw(f.filter.exportBloom()); } noteAdvertiseTopicsOfInterest(); @@ -141,16 +140,16 @@ void WhisperHost::uninstallWatch(unsigned _i) auto it = m_watches.find(_i); if (it == m_watches.end()) return; + auto id = it->second.id; m_watches.erase(it); auto fit = m_filters.find(id); if (fit != m_filters.end()) { + m_bloom.removeRaw(fit->second.filter.exportBloom()); if (!--fit->second.refCount) m_filters.erase(fit); - - m_bloom.removeRaw(fit->second.filter.exportBloom()); } } diff --git a/test/libwhisper/whisperTopic.cpp b/test/libwhisper/whisperTopic.cpp index a152f756e..47d17503c 100644 --- a/test/libwhisper/whisperTopic.cpp +++ b/test/libwhisper/whisperTopic.cpp @@ -320,7 +320,7 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) Host host2("second", NetworkPreferences("127.0.0.1", 30305, false)); host2.setIdealPeerCount(1); auto whost2 = host2.registerCapability(new WhisperHost()); - whost2->installWatch(BuildTopicMask("test2")); + unsigned w2 = whost2->installWatch(BuildTopicMask("test2")); host2.start(); while (!host2.haveNetwork()) @@ -354,7 +354,7 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) BOOST_REQUIRE(bf1); BOOST_REQUIRE(!whost1->bloom()); - whost1->installWatch(BuildTopicMask("test1")); + unsigned w1 = whost1->installWatch(BuildTopicMask("test1")); for (int i = 0; i < 600; ++i) { @@ -372,6 +372,16 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) bf1 = whost1->bloom(); BOOST_REQUIRE_EQUAL(bf1, bf2); BOOST_REQUIRE(bf1); + + unsigned random = 0xC0FFEE; + whost1->uninstallWatch(w1); + whost1->uninstallWatch(random); + whost1->uninstallWatch(w1); + whost1->uninstallWatch(random); + whost2->uninstallWatch(random); + whost2->uninstallWatch(w2); + whost2->uninstallWatch(random); + whost2->uninstallWatch(w2); } BOOST_AUTO_TEST_SUITE_END() From 9c483859d1bf346a0af2ca026e3c767565d1594a Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 23 Jun 2015 16:56:59 +0200 Subject: [PATCH 53/59] Fixed checking of abstract functions. Fixes #2264 --- libsolidity/AST.cpp | 36 +++++++++++++------ .../SolidityNameAndTypeResolution.cpp | 17 +++++++++ 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/libsolidity/AST.cpp b/libsolidity/AST.cpp index cb935183f..fac4360f1 100644 --- a/libsolidity/AST.cpp +++ b/libsolidity/AST.cpp @@ -175,24 +175,40 @@ void ContractDefinition::checkDuplicateFunctions() const void ContractDefinition::checkAbstractFunctions() { - map functions; + // Mapping from name to function definition (exactly one per argument type equality class) and + // flag to indicate whether it is fully implemented. + using FunTypeAndFlag = std::pair; + map> functions; // Search from base to derived for (ContractDefinition const* contract: boost::adaptors::reverse(getLinearizedBaseContracts())) for (ASTPointer const& function: contract->getDefinedFunctions()) { - string const& name = function->getName(); - if (!function->isFullyImplemented() && functions.count(name) && functions[name]) - BOOST_THROW_EXCEPTION(function->createTypeError("Redeclaring an already implemented function as abstract")); - functions[name] = function->isFullyImplemented(); + auto& overloads = functions[function->getName()]; + FunctionTypePointer funType = make_shared(*function); + auto it = find_if(overloads.begin(), overloads.end(), [&](FunTypeAndFlag const& _funAndFlag) + { + return funType->hasEqualArgumentTypes(*_funAndFlag.first); + }); + if (it == overloads.end()) + overloads.push_back(make_pair(funType, function->isFullyImplemented())); + else if (it->second) + { + if (!function->isFullyImplemented()) + BOOST_THROW_EXCEPTION(function->createTypeError("Redeclaring an already implemented function as abstract")); + } + else if (function->isFullyImplemented()) + it->second = true; } + // Set to not fully implemented if at least one flag is false. for (auto const& it: functions) - if (!it.second) - { - setFullyImplemented(false); - break; - } + for (auto const& funAndFlag: it.second) + if (!funAndFlag.second) + { + setFullyImplemented(false); + return; + } } void ContractDefinition::checkAbstractConstructors() diff --git a/test/libsolidity/SolidityNameAndTypeResolution.cpp b/test/libsolidity/SolidityNameAndTypeResolution.cpp index df976eaea..4914ef975 100644 --- a/test/libsolidity/SolidityNameAndTypeResolution.cpp +++ b/test/libsolidity/SolidityNameAndTypeResolution.cpp @@ -403,6 +403,23 @@ BOOST_AUTO_TEST_CASE(abstract_contract) BOOST_CHECK(derived->getDefinedFunctions()[0]->isFullyImplemented()); } +BOOST_AUTO_TEST_CASE(abstract_contract_with_overload) +{ + ASTPointer sourceUnit; + char const* text = R"( + contract base { function foo(bool); } + contract derived is base { function foo(uint) {} } + )"; + ETH_TEST_REQUIRE_NO_THROW(sourceUnit = parseTextAndResolveNames(text), "Parsing and name Resolving failed"); + std::vector> nodes = sourceUnit->getNodes(); + ContractDefinition* base = dynamic_cast(nodes[0].get()); + ContractDefinition* derived = dynamic_cast(nodes[1].get()); + BOOST_REQUIRE(base); + BOOST_CHECK(!base->isFullyImplemented()); + BOOST_REQUIRE(derived); + BOOST_CHECK(!derived->isFullyImplemented()); +} + BOOST_AUTO_TEST_CASE(create_abstract_contract) { ASTPointer sourceUnit; From 49a861726163cd14837ecb0c44894c17265b6d27 Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 30 Jun 2015 14:08:26 +0200 Subject: [PATCH 54/59] fixed block memory management --- libethereum/BlockChain.cpp | 2 +- libethereum/BlockQueue.cpp | 6 +++--- libethereum/State.cpp | 2 +- libethereum/VerifiedBlock.h | 24 ++++++++++++++++++++++++ 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 55c213a64..d58a80ec2 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -541,7 +541,7 @@ ImportRoute BlockChain::import(VerifiedBlockRef const& _block, OverlayDB const& #endif } #if ETH_CATCH - catch (BadRoot& ex) + catch (BadRoot&) { cwarn << "BadRoot error. Retrying import later."; BOOST_THROW_EXCEPTION(FutureTime()); diff --git a/libethereum/BlockQueue.cpp b/libethereum/BlockQueue.cpp index 9e647c9c1..074278d9a 100644 --- a/libethereum/BlockQueue.cpp +++ b/libethereum/BlockQueue.cpp @@ -102,7 +102,7 @@ void BlockQueue::verifierBody() BlockInfo bi; bi.mixHash = work.hash; bi.parentHash = work.parentHash; - m_verifying.push_back(VerifiedBlock { VerifiedBlockRef { bytesConstRef(), move(bi), Transactions() }, bytes() }); + m_verifying.emplace_back(move(bi)); } VerifiedBlock res; @@ -148,7 +148,7 @@ void BlockQueue::verifierBody() m_knownBad.insert(res.verified.info.hash()); } else - m_verified.push_back(move(res)); + m_verified.emplace_back(move(res)); while (m_verifying.size() && !m_verifying.front().blockData.empty()) { if (m_knownBad.count(m_verifying.front().verified.info.parentHash)) @@ -157,7 +157,7 @@ void BlockQueue::verifierBody() m_knownBad.insert(res.verified.info.hash()); } else - m_verified.push_back(move(m_verifying.front())); + m_verified.emplace_back(move(m_verifying.front())); m_verifying.pop_front(); } ready = true; diff --git a/libethereum/State.cpp b/libethereum/State.cpp index ead72f496..badc05887 100644 --- a/libethereum/State.cpp +++ b/libethereum/State.cpp @@ -117,7 +117,7 @@ State::State(OverlayDB const& _db, BaseState _bs, Address _coinbaseAddress): PopulationStatistics State::populateFromChain(BlockChain const& _bc, h256 const& _h, ImportRequirements::value _ir) { - PopulationStatistics ret; + PopulationStatistics ret { 0.0, 0.0 }; if (!_bc.isKnown(_h)) { diff --git a/libethereum/VerifiedBlock.h b/libethereum/VerifiedBlock.h index ddd808901..6cafe4b2f 100644 --- a/libethereum/VerifiedBlock.h +++ b/libethereum/VerifiedBlock.h @@ -43,8 +43,32 @@ struct VerifiedBlockRef /// @brief Verified block info, combines block data and verified info/transactions struct VerifiedBlock { + VerifiedBlock() {}; + + VerifiedBlock(BlockInfo&& _bi) + { + verified.info = _bi; + } + + VerifiedBlock(VerifiedBlock&& _other): + verified(std::move(_other.verified)), + blockData(std::move(_other.blockData)) + { + } + + VerifiedBlock& operator=(VerifiedBlock&& _other) + { + verified = (std::move(_other.verified)); + blockData = (std::move(_other.blockData)); + return *this; + } + VerifiedBlockRef verified; ///< Verified block structures bytes blockData; ///< Block data + +private: + VerifiedBlock(VerifiedBlock const&) = delete; + VerifiedBlock operator=(VerifiedBlock const&) = delete; }; using VerifiedBlocks = std::vector; From e22e3d2d85ac10d2abf1609e304ba9b5996d63c5 Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 30 Jun 2015 15:45:28 +0200 Subject: [PATCH 55/59] fixing linux build --- libdevcore/Common.h | 1 - libwhisper/Common.h | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/libdevcore/Common.h b/libdevcore/Common.h index 4ed4e4365..f1d35bbc7 100644 --- a/libdevcore/Common.h +++ b/libdevcore/Common.h @@ -45,7 +45,6 @@ #pragma warning(push) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-fpermissive" #include #if (BOOST_VERSION == 105800) #include "boost_multiprecision_number_compare_bug_workaround.hpp" diff --git a/libwhisper/Common.h b/libwhisper/Common.h index deb13d111..c05ac34ca 100644 --- a/libwhisper/Common.h +++ b/libwhisper/Common.h @@ -58,8 +58,8 @@ enum WhisperPacket PacketCount }; -enum { TopicBloomFilterSize = 64 }; -enum { WhisperProtocolVersion = 3 }; +const int TopicBloomFilterSize = 64; +const int WhisperProtocolVersion = 3; using AbridgedTopic = FixedHash<4>; using Topic = h256; From 77bfce921b0698ce5e349eb4c9c0f3e3d335ee89 Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 30 Jun 2015 17:07:54 +0200 Subject: [PATCH 56/59] static --- libwhisper/Common.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libwhisper/Common.h b/libwhisper/Common.h index c05ac34ca..7a0707801 100644 --- a/libwhisper/Common.h +++ b/libwhisper/Common.h @@ -58,8 +58,8 @@ enum WhisperPacket PacketCount }; -const int TopicBloomFilterSize = 64; -const int WhisperProtocolVersion = 3; +static const int TopicBloomFilterSize = 64; +static const int WhisperProtocolVersion = 3; using AbridgedTopic = FixedHash<4>; using Topic = h256; From 4941aa82bfe34c643af49112ab834ed566eeb17e Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 30 Jun 2015 17:24:32 +0200 Subject: [PATCH 57/59] style --- libwhisper/BloomFilter.h | 2 +- libwhisper/Common.cpp | 6 +++--- libwhisper/Common.h | 8 +++++--- libwhisper/WhisperHost.h | 4 ++-- libwhisper/WhisperPeer.cpp | 4 ++-- libwhisper/WhisperPeer.h | 10 +++++----- test/libwhisper/bloomFilter.cpp | 4 ++-- test/libwhisper/whisperTopic.cpp | 4 ++-- 8 files changed, 22 insertions(+), 20 deletions(-) diff --git a/libwhisper/BloomFilter.h b/libwhisper/BloomFilter.h index 157f4b011..a624be157 100644 --- a/libwhisper/BloomFilter.h +++ b/libwhisper/BloomFilter.h @@ -91,7 +91,7 @@ bool TopicBloomFilterBase::isBitSet(FixedHash const& _h, unsigned _index) return (_h[iByte] & c_powerOfTwoBitMmask[iBit]) != 0; } -using TopicBloomFilter = TopicBloomFilterBase; +using TopicBloomFilter = TopicBloomFilterBase; } } diff --git a/libwhisper/Common.cpp b/libwhisper/Common.cpp index 79c87b96f..748180647 100644 --- a/libwhisper/Common.cpp +++ b/libwhisper/Common.cpp @@ -95,12 +95,12 @@ TopicFilter::TopicFilter(RLP const& _r) } } -FixedHash TopicFilter::exportBloom() const +TopicBloomFilterHash TopicFilter::exportBloom() const { - FixedHash ret; + TopicBloomFilterHash ret; for (TopicMask const& t: m_topicMasks) for (auto const& i: t) - ret |= i.first.template bloomPart(); + ret |= i.first.template bloomPart(); return ret; } diff --git a/libwhisper/Common.h b/libwhisper/Common.h index 7a0707801..d5d926291 100644 --- a/libwhisper/Common.h +++ b/libwhisper/Common.h @@ -58,8 +58,8 @@ enum WhisperPacket PacketCount }; -static const int TopicBloomFilterSize = 64; -static const int WhisperProtocolVersion = 3; +static const int c_topicBloomFilterSize = 64; +static const int c_whisperProtocolVersion = 3; using AbridgedTopic = FixedHash<4>; using Topic = h256; @@ -67,6 +67,8 @@ using Topic = h256; using AbridgedTopics = std::vector; using Topics = h256s; +using TopicBloomFilterHash = FixedHash; + AbridgedTopic abridge(Topic const& _topic); AbridgedTopics abridge(Topics const& _topics); @@ -107,7 +109,7 @@ public: void streamRLP(RLPStream& _s) const { _s << m_topicMasks; } h256 sha3() const; bool matches(Envelope const& _m) const; - FixedHash exportBloom() const; + TopicBloomFilterHash exportBloom() const; private: TopicMasks m_topicMasks; diff --git a/libwhisper/WhisperHost.h b/libwhisper/WhisperHost.h index 1a43eda3a..a6de09c38 100644 --- a/libwhisper/WhisperHost.h +++ b/libwhisper/WhisperHost.h @@ -50,11 +50,11 @@ class WhisperHost: public HostCapability, public Interface, public public: WhisperHost(); virtual ~WhisperHost(); - unsigned protocolVersion() const { return WhisperProtocolVersion; } + unsigned protocolVersion() const { return c_whisperProtocolVersion; } /// remove old messages void cleanup(); std::map all() const { dev::ReadGuard l(x_messages); return m_messages; } - FixedHash bloom() const { dev::Guard l(m_filterLock); return m_bloom; } + TopicBloomFilterHash bloom() const { dev::Guard l(m_filterLock); return m_bloom; } virtual void inject(Envelope const& _e, WhisperPeer* _from = nullptr) override; virtual Topics const& fullTopics(unsigned _id) const override { try { return m_filters.at(m_watches.at(_id).id).full; } catch (...) { return EmptyTopics; } } diff --git a/libwhisper/WhisperPeer.cpp b/libwhisper/WhisperPeer.cpp index 665364f49..4475769e3 100644 --- a/libwhisper/WhisperPeer.cpp +++ b/libwhisper/WhisperPeer.cpp @@ -74,7 +74,7 @@ bool WhisperPeer::interpret(unsigned _id, RLP const& _r) } case TopicFilterPacket: { - setBloom((FixedHash)_r[0]); + setBloom((TopicBloomFilterHash)_r[0]); break; } default: @@ -115,7 +115,7 @@ void WhisperPeer::noteNewMessage(h256 _h, Envelope const& _m) m_unseen.insert(make_pair(rating(_m), _h)); } -void WhisperPeer::sendTopicsOfInterest(FixedHash const& _bloom) +void WhisperPeer::sendTopicsOfInterest(TopicBloomFilterHash const& _bloom) { DEV_GUARDED(x_advertiseTopicsOfInterest) m_advertiseTopicsOfInterest = false; diff --git a/libwhisper/WhisperPeer.h b/libwhisper/WhisperPeer.h index 1be2df97e..48f984013 100644 --- a/libwhisper/WhisperPeer.h +++ b/libwhisper/WhisperPeer.h @@ -53,10 +53,10 @@ public: virtual ~WhisperPeer(); WhisperHost* host() const; static std::string name() { return "shh"; } - static u256 version() { return WhisperProtocolVersion; } + static u256 version() { return c_whisperProtocolVersion; } static unsigned messageCount() { return PacketCount; } - FixedHash bloom() const { dev::Guard g(x_bloom); return m_bloom; } - void sendTopicsOfInterest(FixedHash const& _bloom); ///< sends our bloom filter to remote peer + TopicBloomFilterHash bloom() const { dev::Guard g(x_bloom); return m_bloom; } + void sendTopicsOfInterest(TopicBloomFilterHash const& _bloom); ///< sends our bloom filter to remote peer void noteAdvertiseTopicsOfInterest() { dev::Guard g(x_advertiseTopicsOfInterest); m_advertiseTopicsOfInterest = true; } private: @@ -64,14 +64,14 @@ private: void sendMessages(); unsigned rating(Envelope const&) const { return 0; } // TODO void noteNewMessage(h256 _h, Envelope const& _m); - void setBloom(FixedHash const& _b) { dev::Guard g(x_bloom); m_bloom = _b; } + void setBloom(TopicBloomFilterHash const& _b) { dev::Guard g(x_bloom); m_bloom = _b; } mutable dev::Mutex x_unseen; std::multimap m_unseen; ///< Rated according to what they want. std::chrono::system_clock::time_point m_timer = std::chrono::system_clock::now(); mutable dev::Mutex x_bloom; - FixedHash m_bloom; ///< Peer's topics of interest + TopicBloomFilterHash m_bloom; ///< Peer's topics of interest mutable dev::Mutex x_advertiseTopicsOfInterest; bool m_advertiseTopicsOfInterest; diff --git a/test/libwhisper/bloomFilter.cpp b/test/libwhisper/bloomFilter.cpp index 3e71ca305..814990d52 100644 --- a/test/libwhisper/bloomFilter.cpp +++ b/test/libwhisper/bloomFilter.cpp @@ -28,7 +28,7 @@ using namespace dev; using namespace dev::shh; using TopicBloomFilterShort = TopicBloomFilterBase<4>; -using TopicBloomFilterTest = TopicBloomFilterBase; +using TopicBloomFilterTest = TopicBloomFilterBase; void testAddNonExisting(TopicBloomFilterShort& _f, AbridgedTopic const& _h) { @@ -244,4 +244,4 @@ BOOST_AUTO_TEST_CASE(bloomFilterRaw) BOOST_REQUIRE(!f.contains(b00110111)); } -BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file +BOOST_AUTO_TEST_SUITE_END() diff --git a/test/libwhisper/whisperTopic.cpp b/test/libwhisper/whisperTopic.cpp index a152f756e..e309239c9 100644 --- a/test/libwhisper/whisperTopic.cpp +++ b/test/libwhisper/whisperTopic.cpp @@ -348,8 +348,8 @@ BOOST_AUTO_TEST_CASE(topicAdvertising) } BOOST_REQUIRE(sessions.size()); - FixedHash bf1 = sessions.back().first->cap()->bloom(); - FixedHash bf2 = whost2->bloom(); + TopicBloomFilterHash bf1 = sessions.back().first->cap()->bloom(); + TopicBloomFilterHash bf2 = whost2->bloom(); BOOST_REQUIRE_EQUAL(bf1, bf2); BOOST_REQUIRE(bf1); BOOST_REQUIRE(!whost1->bloom()); From d1b22891b26e3abb0e02499f743fd90c169d172e Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Mon, 29 Jun 2015 14:14:35 +0200 Subject: [PATCH 58/59] Remove legacy code in blockchain.cpp --- libethereum/BlockChain.cpp | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index d58a80ec2..da491c774 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -24,6 +24,7 @@ #if ETH_PROFILING_GPERF #include #endif +#include #include #include #include @@ -291,15 +292,6 @@ void BlockChain::rebuild(std::string const& _path, std::function -bool contains(T const& _t, V const& _v) -{ - for (auto const& i: _t) - if (i == _v) - return true; - return false; -} - LastHashes BlockChain::lastHashes(unsigned _n) const { Guard l(x_lastLastHashes); @@ -950,7 +942,7 @@ void BlockChain::checkConsistency() if (p != h256() && p != m_genesisHash) // TODO: for some reason the genesis details with the children get squished. not sure why. { auto dp = details(p); - if (asserts(contains(dp.children, h))) + if (asserts(end(dp.children) != find(begin(dp.children), end(dp.children), h))) { cnote << "Apparently the database is corrupt. Not much we can do at this stage..."; } From b9b193647750d3a2c11f2dbad459e06c99074406 Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Tue, 30 Jun 2015 20:36:06 +0200 Subject: [PATCH 59/59] contains() gets moved to CommonData.h - It also now uses std::find() --- libdevcore/CommonData.h | 6 ++++++ libethereum/BlockChain.cpp | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/libdevcore/CommonData.h b/libdevcore/CommonData.h index 57360e95a..ed09e60ee 100644 --- a/libdevcore/CommonData.h +++ b/libdevcore/CommonData.h @@ -334,4 +334,10 @@ std::vector keysOf(std::unordered_map const& _m) return ret; } +template +bool contains(T const& _t, V const& _v) +{ + return std::end(_t) != std::find(std::begin(_t), std::end(_t), _v); +} + } diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index da491c774..4d05ea7db 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -24,7 +24,7 @@ #if ETH_PROFILING_GPERF #include #endif -#include + #include #include #include @@ -942,7 +942,7 @@ void BlockChain::checkConsistency() if (p != h256() && p != m_genesisHash) // TODO: for some reason the genesis details with the children get squished. not sure why. { auto dp = details(p); - if (asserts(end(dp.children) != find(begin(dp.children), end(dp.children), h))) + if (asserts(contains(dp.children, h))) { cnote << "Apparently the database is corrupt. Not much we can do at this stage..."; }