From 5436f90f04d1ec6a81f1fe90f57a8385e9e8635d Mon Sep 17 00:00:00 2001 From: subtly Date: Mon, 5 Jan 2015 00:49:16 +0100 Subject: [PATCH 01/71] Pass 1 integrating node table. TBD: whether to store/relay cap info. --- libp2p/Host.cpp | 202 +++++++++++++++++++++++-------------------- libp2p/Host.h | 66 +++++++++----- libp2p/NodeTable.cpp | 19 ++-- libp2p/NodeTable.h | 92 +++++++++++--------- libp2p/Session.cpp | 2 +- libp2p/Session.h | 6 +- libp2p/UDP.h | 2 - 7 files changed, 214 insertions(+), 175 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 4a99ac90f..73fabe7f2 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -44,7 +44,8 @@ Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n, bool m_ifAddresses(Network::getInterfaceAddresses()), m_ioService(2), m_tcp4Acceptor(m_ioService), - m_key(KeyPair::create()) + m_key(KeyPair::create()), + m_nodeTable(new NodeTable(m_ioService, m_key)) { for (auto address: m_ifAddresses) if (address.is_v4()) @@ -143,11 +144,12 @@ void Host::doneWorking() unsigned Host::protocolVersion() const { - return 2; + return 3; } void Host::registerPeer(std::shared_ptr _s, CapDescs const& _caps) { +#warning integration: todo rework so this is an exception if (!_s->m_node || !_s->m_node->id) { cwarn << "Attempting to register a peer without node information!"; @@ -180,7 +182,8 @@ void Host::seal(bytes& _b) _b[7] = len & 0xff; } -shared_ptr Host::noteNode(NodeId _id, bi::tcp::endpoint _a, Origin _o, bool _ready, NodeId _oldId) +#warning integration: todo remove origin, ready, oldid. port to NodeTable. see Session.cpp#244,363 +shared_ptr Host::noteNode(NodeId _id, bi::tcp::endpoint _a, Origin _o, bool _ready, NodeId _oldId) { RecursiveGuard l(x_peers); if (_a.port() < 30300 || _a.port() > 30305) @@ -192,11 +195,11 @@ shared_ptr Host::noteNode(NodeId _id, bi::tcp::endpoint _a, Origin _o, boo _a = bi::tcp::endpoint(_a.address(), 0); } -// cnote << "Node:" << _id.abridged() << _a << (_ready ? "ready" : "used") << _oldId.abridged() << (m_nodes.count(_id) ? "[have]" : "[NEW]"); +// cnote << "NodeInfo:" << _id.abridged() << _a << (_ready ? "ready" : "used") << _oldId.abridged() << (m_nodes.count(_id) ? "[have]" : "[NEW]"); // First check for another node with the same connection credentials, and put it in oldId if found. if (!_oldId) - for (pair> const& n: m_nodes) + for (pair> const& n: m_nodes) if (n.second->address == _a && n.second->id != _id) { _oldId = n.second->id; @@ -217,7 +220,7 @@ shared_ptr Host::noteNode(NodeId _id, bi::tcp::endpoint _a, Origin _o, boo i = m_nodesList.size(); m_nodesList.push_back(_id); } - m_nodes[_id] = make_shared(); + m_nodes[_id] = make_shared(); m_nodes[_id]->id = _id; m_nodes[_id]->index = i; m_nodes[_id]->idOrigin = _o; @@ -246,6 +249,7 @@ shared_ptr Host::noteNode(NodeId _id, bi::tcp::endpoint _a, Origin _o, boo return m_nodes[_id]; } +#warning integration: TBD caps in NodeTable/NodeEntry Nodes Host::potentialPeers(RangeMask const& _known) { RecursiveGuard l(x_peers); @@ -376,6 +380,7 @@ string Host::pocHost() return "poc-" + strs[1] + ".ethdev.com"; } +#warning integration: todo remove all connect() w/ addNode makeRequired (this requires pubkey) void Host::connect(std::string const& _addr, unsigned short _port) noexcept { if (!m_run) @@ -428,13 +433,13 @@ void Host::connect(bi::tcp::endpoint const& _ep) }); } -void Host::connect(std::shared_ptr const& _n) +void Host::connect(std::shared_ptr const& _n) { if (!m_run) return; // prevent concurrently connecting to a node; todo: better abstraction - Node *nptr = _n.get(); + NodeInfo *nptr = _n.get(); { Guard l(x_pendingNodeConns); if (m_pendingNodeConns.count(nptr)) @@ -488,7 +493,7 @@ bool Host::havePeer(NodeId _id) const return !!m_peers.count(_id); } -unsigned Node::fallbackSeconds() const +unsigned NodeInfo::fallbackSeconds() const { switch (lastDisconnect) { @@ -510,85 +515,83 @@ unsigned Node::fallbackSeconds() const } } -bool Node::shouldReconnect() const -{ - return chrono::system_clock::now() > lastAttempted + chrono::seconds(fallbackSeconds()); -} - -void Host::growPeers() -{ - RecursiveGuard l(x_peers); - int morePeers = (int)m_idealPeerCount - m_peers.size(); - if (morePeers > 0) - { - auto toTry = m_ready; - if (!m_netPrefs.localNetworking) - toTry -= m_private; - set ns; - for (auto i: toTry) - if (m_nodes[m_nodesList[i]]->shouldReconnect()) - ns.insert(*m_nodes[m_nodesList[i]]); - - if (ns.size()) - for (Node const& i: ns) - { - connect(m_nodes[i.id]); - if (!--morePeers) - return; - } - else - for (auto const& i: m_peers) - if (auto p = i.second.lock()) - p->ensureNodesRequested(); - } -} - -void Host::prunePeers() -{ - RecursiveGuard l(x_peers); - // We'll keep at most twice as many as is ideal, halfing what counts as "too young to kill" until we get there. - set dc; - for (unsigned old = 15000; m_peers.size() - dc.size() > m_idealPeerCount * 2 && old > 100; old /= 2) - if (m_peers.size() - dc.size() > m_idealPeerCount) - { - // look for worst peer to kick off - // first work out how many are old enough to kick off. - shared_ptr worst; - unsigned agedPeers = 0; - for (auto i: m_peers) - if (!dc.count(i.first)) - if (auto p = i.second.lock()) - if (/*(m_mode != NodeMode::Host || p->m_caps != 0x01) &&*/ chrono::steady_clock::now() > p->m_connect + chrono::milliseconds(old)) // don't throw off new peers; peer-servers should never kick off other peer-servers. - { - ++agedPeers; - if ((!worst || p->rating() < worst->rating() || (p->rating() == worst->rating() && p->m_connect > worst->m_connect))) // kill older ones - worst = p; - } - if (!worst || agedPeers <= m_idealPeerCount) - break; - dc.insert(worst->id()); - worst->disconnect(TooManyPeers); - } - - // Remove dead peers from list. - for (auto i = m_peers.begin(); i != m_peers.end();) - if (i->second.lock().get()) - ++i; - else - i = m_peers.erase(i); -} - -PeerInfos Host::peers(bool _updatePing) const +#warning integration: ---- grow/prunePeers +#warning integration: todo grow/prune into 'maintainPeers' & evaluate reputation instead of availability. schedule via deadline timer. +//void Host::growPeers() +//{ +// RecursiveGuard l(x_peers); +// int morePeers = (int)m_idealPeerCount - m_peers.size(); +// if (morePeers > 0) +// { +// auto toTry = m_ready; +// if (!m_netPrefs.localNetworking) +// toTry -= m_private; +// set ns; +// for (auto i: toTry) +// if (m_nodes[m_nodesList[i]]->shouldReconnect()) +// ns.insert(*m_nodes[m_nodesList[i]]); +// +// if (ns.size()) +// for (NodeInfo const& i: ns) +// { +// connect(m_nodes[i.id]); +// if (!--morePeers) +// return; +// } +// else +// for (auto const& i: m_peers) +// if (auto p = i.second.lock()) +// p->ensureNodesRequested(); +// } +//} +// +//void Host::prunePeers() +//{ +// RecursiveGuard l(x_peers); +// // We'll keep at most twice as many as is ideal, halfing what counts as "too young to kill" until we get there. +// set dc; +// for (unsigned old = 15000; m_peers.size() - dc.size() > m_idealPeerCount * 2 && old > 100; old /= 2) +// if (m_peers.size() - dc.size() > m_idealPeerCount) +// { +// // look for worst peer to kick off +// // first work out how many are old enough to kick off. +// shared_ptr worst; +// unsigned agedPeers = 0; +// for (auto i: m_peers) +// if (!dc.count(i.first)) +// if (auto p = i.second.lock()) +// if (/*(m_mode != NodeMode::Host || p->m_caps != 0x01) &&*/ chrono::steady_clock::now() > p->m_connect + chrono::milliseconds(old)) // don't throw off new peers; peer-servers should never kick off other peer-servers. +// { +// ++agedPeers; +// if ((!worst || p->rating() < worst->rating() || (p->rating() == worst->rating() && p->m_connect > worst->m_connect))) // kill older ones +// worst = p; +// } +// if (!worst || agedPeers <= m_idealPeerCount) +// break; +// dc.insert(worst->id()); +// worst->disconnect(TooManyPeers); +// } +// +// // Remove dead peers from list. +// for (auto i = m_peers.begin(); i != m_peers.end();) +// if (i->second.lock().get()) +// ++i; +// else +// i = m_peers.erase(i); +//} + +PeerInfos Host::peers() const { if (!m_run) return PeerInfos(); +#warning integration: ---- pingAll. It is called every 30secs via deadline timer. RecursiveGuard l(x_peers); - if (_updatePing) - { - const_cast(this)->pingAll(); - this_thread::sleep_for(chrono::milliseconds(200)); - } +// if (_updatePing) +// { +// const_cast(this)->pingAll(); +// this_thread::sleep_for(chrono::milliseconds(200)); +// } std::vector ret; for (auto& i: m_peers) if (auto j = i.second.lock()) @@ -610,13 +613,14 @@ void Host::run(boost::system::error_code const&) return; } - m_lastTick += c_timerInterval; - if (m_lastTick >= c_timerInterval * 10) - { - growPeers(); - prunePeers(); - m_lastTick = 0; - } +#warning integration: ---- +// m_lastTick += c_timerInterval; +// if (m_lastTick >= c_timerInterval * 10) +// { +// growPeers(); +// prunePeers(); +// m_lastTick = 0; +// } if (m_hadNewNodes) { @@ -670,12 +674,19 @@ void Host::startedWorking() if (m_listenPort > 0) runAcceptor(); + +#warning integration: ++++ + if (!m_tcpPublic.address().is_unspecified()) + m_nodeTable.reset(new NodeTable(m_ioService, m_key, m_listenPort, m_tcpPublic)); + else + m_nodeTable.reset(new NodeTable(m_ioService, m_key, m_listenPort > 0 ? m_listenPort : 30303)); } - // if m_public address is valid then add us to node list - // todo: abstract empty() and emplace logic - if (!m_tcpPublic.address().is_unspecified() && (m_nodes.empty() || m_nodes[m_nodesList[0]]->id != id())) - noteNode(id(), m_tcpPublic, Origin::Perfect, false); +#warning integration: ---- +// // if m_public address is valid then add us to node list +// // todo: abstract empty() and emplace logic +// if (!m_tcpPublic.address().is_unspecified() && (m_nodes.empty() || m_nodes[m_nodesList[0]]->id != id())) +// noteNode(id(), m_tcpPublic, Origin::Perfect, false); clog(NetNote) << "Id:" << id().abridged(); @@ -697,6 +708,7 @@ void Host::pingAll() m_lastPing = chrono::steady_clock::now(); } +#warning integration: todo save/restoreNodes bytes Host::saveNodes() const { RLPStream nodes; @@ -705,7 +717,7 @@ bytes Host::saveNodes() const RecursiveGuard l(x_peers); for (auto const& i: m_nodes) { - Node const& n = *(i.second); + NodeInfo const& n = *(i.second); // TODO: PoC-7: Figure out why it ever shares these ports.//n.address.port() >= 30300 && n.address.port() <= 30305 && if (!n.dead && chrono::system_clock::now() - n.lastConnected < chrono::seconds(3600 * 48) && n.address.port() > 0 && n.address.port() < /*49152*/32768 && n.id != id() && !isPrivateAddress(n.address.address())) { diff --git a/libp2p/Host.h b/libp2p/Host.h index 8ed25f2ae..9600c723b 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -34,6 +34,7 @@ #include #include #include +#include "NodeTable.h" #include "HostCapability.h" #include "Network.h" #include "Common.h" @@ -59,29 +60,47 @@ enum class Origin Perfect, }; -struct Node +struct NodeInfo { NodeId id; ///< Their id/public key. unsigned index; ///< Index into m_nodesList + + // p2p: move to NodeEndpoint bi::tcp::endpoint address; ///< As reported from the node itself. - int score = 0; ///< All time cumulative. - int rating = 0; ///< Trending. - bool dead = false; ///< If true, we believe this node is permanently dead - forget all about it. + + // p2p: This information is relevant to the network-stack, ex: firewall, rather than node itself std::chrono::system_clock::time_point lastConnected; std::chrono::system_clock::time_point lastAttempted; unsigned failedAttempts = 0; DisconnectReason lastDisconnect = NoDisconnect; ///< Reason for disconnect that happened last. + // p2p: just remove node + // p2p: revisit logic in see Session.cpp#210 + bool dead = false; ///< If true, we believe this node is permanently dead - forget all about it. + + // p2p: move to protocol-specific map + int score = 0; ///< All time cumulative. + int rating = 0; ///< Trending. + + // p2p: remove Origin idOrigin = Origin::Unknown; ///< How did we get to know this node's id? + // p2p: move to NodeEndpoint int secondsSinceLastConnected() const { return lastConnected == std::chrono::system_clock::time_point() ? -1 : (int)std::chrono::duration_cast(std::chrono::system_clock::now() - lastConnected).count(); } + // p2p: move to NodeEndpoint int secondsSinceLastAttempted() const { return lastAttempted == std::chrono::system_clock::time_point() ? -1 : (int)std::chrono::duration_cast(std::chrono::system_clock::now() - lastAttempted).count(); } + // p2p: move to NodeEndpoint unsigned fallbackSeconds() const; - bool shouldReconnect() const; + // p2p: move to NodeEndpoint + bool shouldReconnect() const { return std::chrono::system_clock::now() > lastAttempted + std::chrono::seconds(fallbackSeconds()); } + // p2p: This has two meanings now. It's possible UDP works but TPC is down (unable to punch hole). + // p2p: Rename to isConnect() and move to endpoint. revisit Session.cpp#245, MainWin.cpp#877 bool isOffline() const { return lastAttempted > lastConnected; } - bool operator<(Node const& _n) const + + // p2p: Remove (in favor of lru eviction and sub-protocol ratings). + bool operator<(NodeInfo const& _n) const { if (isOffline() != _n.isOffline()) return isOffline(); @@ -101,7 +120,7 @@ struct Node } }; -using Nodes = std::vector; +using Nodes = std::vector; /** * @brief The Host class @@ -111,7 +130,7 @@ class Host: public Worker { friend class Session; friend class HostCapabilityFace; - friend struct Node; + friend struct NodeInfo; public: /// Start server, listening for connections on the given port. @@ -134,16 +153,19 @@ public: static std::string pocHost(); void connect(std::string const& _addr, unsigned short _port = 30303) noexcept; void connect(bi::tcp::endpoint const& _ep); - void connect(std::shared_ptr const& _n); + void connect(std::shared_ptr const& _n); /// @returns true iff we have a peer of the given id. bool havePeer(NodeId _id) const; /// Set ideal number of peers. void setIdealPeerCount(unsigned _n) { m_idealPeerCount = _n; } + + /// p2p: template? + void setIdealPeerCount(HostCapabilityFace* _cap, unsigned _n) { m_capIdealPeerCount[_cap->capDesc()] = _n; } /// Get peer information. - PeerInfos peers(bool _updatePing = false) const; + PeerInfos peers() const; /// Get number of peers connected; equivalent to, but faster than, peers().size(). size_t peerCount() const { RecursiveGuard l(x_peers); return m_peers.size(); } @@ -178,7 +200,7 @@ public: void registerPeer(std::shared_ptr _s, CapDescs const& _caps); - std::shared_ptr node(NodeId _id) const { if (m_nodes.count(_id)) return m_nodes.at(_id); return std::shared_ptr(); } + std::shared_ptr node(NodeId _id) const { if (m_nodes.count(_id)) return m_nodes.at(_id); return std::shared_ptr(); } private: /// Populate m_peerAddresses with available public addresses. @@ -189,9 +211,6 @@ private: void seal(bytes& _b); - void growPeers(); - void prunePeers(); - /// Called by Worker. Not thread-safe; to be called only by worker. virtual void startedWorking(); /// Called by startedWorking. Not thread-safe; to be called only be Worker. @@ -203,34 +222,35 @@ private: /// Shutdown network. Not thread-safe; to be called only by worker. virtual void doneWorking(); - std::shared_ptr noteNode(NodeId _id, bi::tcp::endpoint _a, Origin _o, bool _ready, NodeId _oldId = NodeId()); + std::shared_ptr noteNode(NodeId _id, bi::tcp::endpoint _a, Origin _o, bool _ready, NodeId _oldId = NodeId()); Nodes potentialPeers(RangeMask const& _known); bool m_run = false; ///< Whether network is running. - std::mutex x_runTimer; ///< Start/stop mutex. + std::mutex x_runTimer; ///< Start/stop mutex. std::string m_clientVersion; ///< Our version string. NetworkPreferences m_netPrefs; ///< Network settings. /// Interface addresses (private, public) - std::vector m_ifAddresses; ///< Interface addresses. + std::vector m_ifAddresses; ///< Interface addresses. int m_listenPort = -1; ///< What port are we listening on. -1 means binding failed or acceptor hasn't been initialized. - ba::io_service m_ioService; ///< IOService for network stuff. - bi::tcp::acceptor m_tcp4Acceptor; ///< Listening acceptor. + ba::io_service m_ioService; ///< IOService for network stuff. + bi::tcp::acceptor m_tcp4Acceptor; ///< Listening acceptor. std::unique_ptr m_socket; ///< Listening socket. std::unique_ptr m_timer; ///< Timer which, when network is running, calls scheduler() every c_timerInterval ms. static const unsigned c_timerInterval = 100; ///< Interval which m_timer is run when network is connected. - unsigned m_lastTick = 0; ///< Used by run() for scheduling; must not be mutated outside of run(). - std::set m_pendingNodeConns; /// Used only by connect(Node&) to limit concurrently connecting to same node. See connect(shared_ptrconst&). + std::set m_pendingNodeConns; /// Used only by connect(NodeInfo&) to limit concurrently connecting to same node. See connect(shared_ptrconst&). Mutex x_pendingNodeConns; bi::tcp::endpoint m_tcpPublic; ///< Our public listening endpoint. - KeyPair m_key; ///< Our unique ID. + KeyPair m_key; ///< Our unique ID. + std::shared_ptr m_nodeTable; ///< Node table (uses kademlia-like discovery). + std::map m_capIdealPeerCount; ///< Ideal peer count for capability. bool m_hadNewNodes = false; @@ -242,7 +262,7 @@ private: /// Nodes to which we may connect (or to which we have connected). /// TODO: does this need a lock? - std::map > m_nodes; + std::map > m_nodes; /// A list of node IDs. This contains every index from m_nodes; the order is guaranteed to remain the same. std::vector m_nodesList; diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 20f9f5fdf..54c8c9f35 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -24,12 +24,15 @@ using namespace std; using namespace dev; using namespace dev::p2p; -NodeTable::NodeTable(ba::io_service& _io, KeyPair _alias, uint16_t _listenPort): +NodeEntry::NodeEntry(Node _src, Public _pubk, NodeDefaultEndpoint _gw): Node(_pubk, _gw), distance(NodeTable::dist(_src.id,_pubk)) {} +NodeEntry::NodeEntry(Node _src, Public _pubk, bi::udp::endpoint _udp): Node(_pubk, NodeDefaultEndpoint(_udp)), distance(NodeTable::dist(_src.id,_pubk)) {} + +NodeTable::NodeTable(ba::io_service& _io, KeyPair _alias, uint16_t _udp, bi::tcp::endpoint _ep): m_node(Node(_alias.pub(), bi::udp::endpoint())), m_secret(_alias.sec()), - m_socket(new NodeSocket(_io, *this, _listenPort)), - m_socketPtr(m_socket.get()), m_io(_io), + m_socket(new NodeSocket(m_io, *this, _udp)), + m_socketPtr(m_socket.get()), m_bucketRefreshTimer(m_io), m_evictionCheckTimer(m_io) { @@ -64,7 +67,7 @@ std::list NodeTable::nodes() const return std::move(nodes); } -list NodeTable::state() const +list NodeTable::state() const { list ret; Guard l(x_state); @@ -74,7 +77,7 @@ list NodeTable::state() const return move(ret); } -NodeTable::NodeEntry NodeTable::operator[](NodeId _id) +NodeEntry NodeTable::operator[](NodeId _id) { Guard l(x_nodes); return *m_nodes[_id]; @@ -135,7 +138,7 @@ void NodeTable::doFindNode(NodeId _node, unsigned _round, std::shared_ptr> NodeTable::findNearest(NodeId _target) +std::vector> NodeTable::findNearest(NodeId _target) { // send s_alpha FindNode packets to nodes we know, closest to target static unsigned lastBin = s_bins - 1; @@ -259,7 +262,7 @@ void NodeTable::noteNode(Public const& _pubk, bi::udp::endpoint const& _endpoint } } - // todo: why is this necessary? + // todo: sometimes node is nullptr here if (!!node) noteNode(node); } @@ -365,7 +368,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes // clog(NodeTableMessageSummary) << "Received FindNode from " << _from.address().to_string() << ":" << _from.port(); FindNode in = FindNode::fromBytesConstRef(_from, rlpBytes); - std::vector> nearest = findNearest(in.target); + std::vector> nearest = findNearest(in.target); static unsigned const nlimit = (m_socketPtr->maxDatagramSize - 11) / 86; for (unsigned offset = 0; offset < nearest.size(); offset += nlimit) { diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 5466a3ae3..e3503b0b9 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -31,6 +31,41 @@ namespace dev namespace p2p { +struct NodeDefaultEndpoint +{ + NodeDefaultEndpoint(bi::udp::endpoint _udp): udp(_udp) {} + NodeDefaultEndpoint(bi::tcp::endpoint _tcp): tcp(_tcp) {} + NodeDefaultEndpoint(bi::udp::endpoint _udp, bi::tcp::endpoint _tcp): udp(_udp), tcp(_tcp) {} + + bi::udp::endpoint udp; + bi::tcp::endpoint tcp; +}; + +struct Node +{ + Node(Public _pubk, NodeDefaultEndpoint _udp): id(_pubk), endpoint(_udp) {} + Node(Public _pubk, bi::udp::endpoint _udp): Node(_pubk, NodeDefaultEndpoint(_udp)) {} + + virtual NodeId const& address() const { return id; } + virtual Public const& publicKey() const { return id; } + + NodeId id; + NodeDefaultEndpoint endpoint; +}; + + +/** + * NodeEntry + * @brief Entry in Node Table + */ +struct NodeEntry: public Node +{ + NodeEntry(Node _src, Public _pubk, NodeDefaultEndpoint _gw); //: Node(_pubk, _gw), distance(dist(_src.id,_pubk)) {} + NodeEntry(Node _src, Public _pubk, bi::udp::endpoint _udp); //: Node(_pubk, NodeDefaultEndpoint(_udp)), distance(dist(_src.id,_pubk)) {} + + const unsigned distance; ///< Node's distance from _src (see constructor). +}; + /** * NodeTable using S/Kademlia system for node discovery and preference. * untouched buckets are refreshed if they have not been touched within an hour @@ -50,7 +85,7 @@ namespace p2p * @todo expiration and sha3(id) 'to' for messages which are replies (prevents replay) * @todo std::shared_ptr m_cachedPingPacket; * @todo std::shared_ptr m_cachedFindSelfPacket; - * @todo store root node in table? + * @todo store self (root) in table? (potential bug. alt is to add to list when self is closest to target) * * [Networking] * @todo TCP endpoints @@ -70,46 +105,9 @@ class NodeTable: UDPSocketEvents, public std::enable_shared_from_this using NodeSocket = UDPSocket; using TimePoint = std::chrono::steady_clock::time_point; using EvictionTimeout = std::pair,NodeId>; ///< First NodeId may be evicted and replaced with second NodeId. - - struct NodeDefaultEndpoint - { - NodeDefaultEndpoint(bi::udp::endpoint _udp): udp(_udp) {} - bi::udp::endpoint udp; - }; - - struct Node - { - Node(Public _pubk, NodeDefaultEndpoint _udp): id(_pubk), endpoint(_udp) {} - Node(Public _pubk, bi::udp::endpoint _udp): Node(_pubk, NodeDefaultEndpoint(_udp)) {} - - virtual NodeId const& address() const { return id; } - virtual Public const& publicKey() const { return id; } - - NodeId id; - NodeDefaultEndpoint endpoint; - }; - - /** - * NodeEntry - * @todo Type of id will become template parameter. - */ - struct NodeEntry: public Node - { - NodeEntry(Node _src, Public _pubk, NodeDefaultEndpoint _gw): Node(_pubk, _gw), distance(dist(_src.id,_pubk)) {} - NodeEntry(Node _src, Public _pubk, bi::udp::endpoint _udp): Node(_pubk, NodeDefaultEndpoint(_udp)), distance(dist(_src.id,_pubk)) {} - - const unsigned distance; ///< Node's distance from _src (see constructor). - }; - - struct NodeBucket - { - unsigned distance; - TimePoint modified; - std::list> nodes; - }; public: - NodeTable(ba::io_service& _io, KeyPair _alias, uint16_t _port = 30300); + NodeTable(ba::io_service& _io, KeyPair _alias, uint16_t _udpPort = 30303, bi::tcp::endpoint _ep = bi::tcp::endpoint()); ~NodeTable(); /// Constants for Kademlia, mostly derived from address space. @@ -121,7 +119,7 @@ public: /// Chosen constants - static unsigned const s_bucketSize = 16; ///< Denoted by k in [Kademlia]. Number of nodes stored in each bucket. + static unsigned const s_bucketSize = 16; ///< Denoted by k in [Kademlia]. Number of nodes stored in each bucket. static unsigned const s_alpha = 3; ///< Denoted by \alpha in [Kademlia]. Number of concurrent FindNode requests. /// Intervals @@ -142,6 +140,13 @@ public: protected: + struct NodeBucket + { + unsigned distance; + TimePoint modified; + std::list> nodes; + }; + /// Repeatedly sends s_alpha concurrent requests to nodes nearest to target, for nodes nearest to target, up to s_maxSteps rounds. void doFindNode(NodeId _node, unsigned _round = 0, std::shared_ptr>> _tried = std::shared_ptr>>()); @@ -193,10 +198,11 @@ protected: Mutex x_evictions; std::deque m_evictions; ///< Eviction timeouts. - + + ba::io_service& m_io; ///< Used by bucket refresh timer. std::shared_ptr m_socket; ///< Shared pointer for our UDPSocket; ASIO requires shared_ptr. NodeSocket* m_socketPtr; ///< Set to m_socket.get(). - ba::io_service& m_io; ///< Used by bucket refresh timer. + boost::asio::deadline_timer m_bucketRefreshTimer; ///< Timer which schedules and enacts bucket refresh. boost::asio::deadline_timer m_evictionCheckTimer; ///< Timer for handling node evictions. }; @@ -317,7 +323,7 @@ struct Neighbors: RLPXDatagram }; using RLPXDatagram::RLPXDatagram; - Neighbors(bi::udp::endpoint _to, std::vector> const& _nearest, unsigned _offset = 0, unsigned _limit = 0): RLPXDatagram(_to) + Neighbors(bi::udp::endpoint _to, std::vector> const& _nearest, unsigned _offset = 0, unsigned _limit = 0): RLPXDatagram(_to) { auto limit = _limit ? std::min(_nearest.size(), (size_t)(_offset + _limit)) : _nearest.size(); for (auto i = _offset; i < limit; i++) diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index 3a0dcf1cf..9a0dd2d2e 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -47,7 +47,7 @@ Session::Session(Host* _s, bi::tcp::socket _socket, bi::tcp::endpoint const& _ma m_info = PeerInfo({NodeId(), "?", m_manualEndpoint.address().to_string(), 0, std::chrono::steady_clock::duration(0), CapDescSet(), 0, map()}); } -Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr const& _n, bool _force): +Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr const& _n, bool _force): m_server(_s), m_socket(std::move(_socket)), m_node(_n), diff --git a/libp2p/Session.h b/libp2p/Session.h index cabef2cbf..869eca96e 100644 --- a/libp2p/Session.h +++ b/libp2p/Session.h @@ -39,7 +39,7 @@ namespace dev namespace p2p { -struct Node; +struct NodeInfo; /** * @brief The Session class @@ -51,7 +51,7 @@ class Session: public std::enable_shared_from_this friend class HostCapabilityFace; public: - Session(Host* _server, bi::tcp::socket _socket, std::shared_ptr const& _n, bool _force = false); + Session(Host* _server, bi::tcp::socket _socket, std::shared_ptr const& _n, bool _force = false); Session(Host* _server, bi::tcp::socket _socket, bi::tcp::endpoint const& _manual); virtual ~Session(); @@ -113,7 +113,7 @@ private: PeerInfo m_info; ///< Dynamic information about this peer. unsigned m_protocolVersion = 0; ///< The protocol version of the peer. - std::shared_ptr m_node; ///< The Node object. Might be null if we constructed using a bare address/port. + std::shared_ptr m_node; ///< The NodeInfo object. Might be null if we constructed using a bare address/port. bi::tcp::endpoint m_manualEndpoint; ///< The endpoint as specified by the constructor. bool m_force = false; ///< If true, ignore IDs being different. This could open you up to MitM attacks. bool m_dropped = false; ///< If true, we've already divested ourselves of this peer. We're just waiting for the reads & writes to fail before the shared_ptr goes OOS and the destructor kicks in. diff --git a/libp2p/UDP.h b/libp2p/UDP.h index ac4afb0b1..8e732d3ff 100644 --- a/libp2p/UDP.h +++ b/libp2p/UDP.h @@ -57,8 +57,6 @@ protected: /** * @brief RLPX Datagram which can be signed. - * @todo compact templates - * @todo make data private/functional (see UDPDatagram) */ struct RLPXDatagramFace: public UDPDatagram { From da9668c0f5c6fa5c6f9013c903e7abf2ad760a8c Mon Sep 17 00:00:00 2001 From: subtly Date: Wed, 7 Jan 2015 02:26:14 +0100 Subject: [PATCH 02/71] Pass 2 integrating node table. Pruning and merging old node lifecycle logic with new. Begin moving node identification and authentication into Host so session can be directly-constructed with NodeInfo and is not created until after authentication. Require session to be passed a valid node. --- libp2p/Host.cpp | 191 +++++++++++++++------------------------ libp2p/Host.h | 27 ++---- libp2p/NodeTable.cpp | 34 ++++--- libp2p/NodeTable.h | 5 +- libp2p/Session.cpp | 58 ++++-------- libp2p/Session.h | 6 +- libwebthree/WebThree.cpp | 2 +- test/peer.cpp | 2 +- test/whisperTopic.cpp | 2 +- 9 files changed, 122 insertions(+), 205 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index bdb72985c..ffc63cf8e 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -149,12 +149,8 @@ unsigned Host::protocolVersion() const void Host::registerPeer(std::shared_ptr _s, CapDescs const& _caps) { -#warning integration: todo rework so this is an exception - if (!_s->m_node || !_s->m_node->id) - { - cwarn << "Attempting to register a peer without node information!"; - return; - } + assert(!!_s->m_node); + assert(!!_s->m_node->id); { RecursiveGuard l(x_peers); @@ -182,8 +178,8 @@ void Host::seal(bytes& _b) _b[7] = len & 0xff; } -#warning integration: todo remove origin, ready, oldid. port to NodeTable. see Session.cpp#244,363 -shared_ptr Host::noteNode(NodeId _id, bi::tcp::endpoint _a, Origin _o, bool _ready, NodeId _oldId) +// TODO P2P: remove oldid. port to NodeTable. (see noteNode calls, Session.cpp#218,337) +shared_ptr Host::noteNode(NodeId _id, bi::tcp::endpoint _a, NodeId _oldId) { RecursiveGuard l(x_peers); if (_a.port() < 30300 || _a.port() > 30305) @@ -195,8 +191,6 @@ shared_ptr Host::noteNode(NodeId _id, bi::tcp::endpoint _a, Origin _o, _a = bi::tcp::endpoint(_a.address(), 0); } -// cnote << "NodeInfo:" << _id.abridged() << _a << (_ready ? "ready" : "used") << _oldId.abridged() << (m_nodes.count(_id) ? "[have]" : "[NEW]"); - // First check for another node with the same connection credentials, and put it in oldId if found. if (!_oldId) for (pair> const& n: m_nodes) @@ -223,33 +217,21 @@ shared_ptr Host::noteNode(NodeId _id, bi::tcp::endpoint _a, Origin _o, m_nodes[_id] = make_shared(); m_nodes[_id]->id = _id; m_nodes[_id]->index = i; - m_nodes[_id]->idOrigin = _o; } else - { i = m_nodes[_id]->index; - m_nodes[_id]->idOrigin = max(m_nodes[_id]->idOrigin, _o); - } m_nodes[_id]->address = _a; - m_ready.extendAll(i); m_private.extendAll(i); - if (_ready) - m_ready += i; - else - m_ready -= i; if (!_a.port() || (isPrivateAddress(_a.address()) && !m_netPrefs.localNetworking)) m_private += i; else m_private -= i; -// cnote << m_nodes[_id]->index << ":" << m_ready; - - m_hadNewNodes = true; - return m_nodes[_id]; } -#warning integration: TBD caps in NodeTable/NodeEntry +// TODO P2P: should be based on target +// TODO P2P: store caps in NodeTable/NodeEntry Nodes Host::potentialPeers(RangeMask const& _known) { RecursiveGuard l(x_peers); @@ -340,13 +322,7 @@ void Host::runAcceptor() { try { - try { - clog(NetConnect) << "Accepted connection from " << m_socket->remote_endpoint(); - } catch (...){} - bi::address remoteAddress = m_socket->remote_endpoint().address(); - // Port defaults to 0 - we let the hello tell us which port the peer listens to - auto p = std::make_shared(this, std::move(*m_socket.release()), bi::tcp::endpoint(remoteAddress, 0)); - p->start(); + doHandshake(m_socket.release()); success = true; } catch (Exception const& _e) @@ -373,6 +349,16 @@ void Host::runAcceptor() } } +void Host::doHandshake(bi::tcp::socket* _socket, NodeId _egressNodeId) +{ + try { + clog(NetConnect) << "Accepting connection for " << _socket->remote_endpoint(); + } catch (...){} + + auto p = std::make_shared(this, std::move(*_socket), m_nodes[_egressNodeId]); + p->start(); +} + string Host::pocHost() { vector strs; @@ -380,56 +366,51 @@ string Host::pocHost() return "poc-" + strs[1] + ".ethdev.com"; } -#warning integration: todo remove all connect() w/ addNode makeRequired (this requires pubkey) -void Host::connect(std::string const& _addr, unsigned short _port) noexcept +// TODO P2P: support for TCP+UDP when manually connecting +// TODO P2P: remove in favor of addNode(NodeId, string, uint16_t, bool _required = false) +void Host::connect(NodeId const& _node, std::string const& _addr, unsigned short _port) noexcept { if (!m_run) return; + + assert(_node); - for (auto first: {true, false}) + auto n = m_nodes[_node]; + + // TODO: refactor into async_resolve + m_ioService.post([=]() { - try + for (auto first: {true, false}) { - if (first) + try { - bi::tcp::resolver r(m_ioService); - connect(r.resolve({_addr, toString(_port)})->endpoint()); + bi::tcp::endpoint ep; + if (first) + { + bi::tcp::resolver r(m_ioService); + ep = r.resolve({_addr, toString(_port)})->endpoint(); + } + else + ep = bi::tcp::endpoint(bi::address::from_string(_addr), _port); + + if (!n) + m_nodes[_node] = make_shared(); + m_nodes[_node]->id = _node; + m_nodes[_node]->address = ep; + connect(m_nodes[_node]); + break; + } + catch (Exception const& _e) + { + // Couldn't connect + clog(NetConnect) << "Bad host " << _addr << "\n" << diagnostic_information(_e); + } + catch (exception const& e) + { + // Couldn't connect + clog(NetConnect) << "Bad host " << _addr << " (" << e.what() << ")"; } - else - connect(bi::tcp::endpoint(bi::address::from_string(_addr), _port)); - break; - } - catch (Exception const& _e) - { - // Couldn't connect - clog(NetConnect) << "Bad host " << _addr << "\n" << diagnostic_information(_e); - } - catch (exception const& e) - { - // Couldn't connect - clog(NetConnect) << "Bad host " << _addr << " (" << e.what() << ")"; - } - } -} - -void Host::connect(bi::tcp::endpoint const& _ep) -{ - if (!m_run) - return; - - clog(NetConnect) << "Attempting single-shot connection to " << _ep; - bi::tcp::socket* s = new bi::tcp::socket(m_ioService); - s->async_connect(_ep, [=](boost::system::error_code const& ec) - { - if (ec) - clog(NetConnect) << "Connection refused to " << _ep << " (" << ec.message() << ")"; - else - { - auto p = make_shared(this, std::move(*s), _ep); - clog(NetConnect) << "Connected to " << _ep; - p->start(); } - delete s; }); } @@ -438,7 +419,7 @@ void Host::connect(std::shared_ptr const& _n) if (!m_run) return; - // prevent concurrently connecting to a node; todo: better abstraction + // prevent concurrently connecting to a node NodeInfo *nptr = _n.get(); { Guard l(x_pendingNodeConns); @@ -450,7 +431,6 @@ void Host::connect(std::shared_ptr const& _n) clog(NetConnect) << "Attempting connection to node" << _n->id.abridged() << "@" << _n->address << "from" << id().abridged(); _n->lastAttempted = std::chrono::system_clock::now(); _n->failedAttempts++; - m_ready -= _n->index; bi::tcp::socket* s = new bi::tcp::socket(m_ioService); auto n = node(_n->id); @@ -462,13 +442,12 @@ void Host::connect(std::shared_ptr const& _n) clog(NetConnect) << "Connection refused to node" << _n->id.abridged() << "@" << _n->address << "(" << ec.message() << ")"; _n->lastDisconnect = TCPError; _n->lastAttempted = std::chrono::system_clock::now(); - m_ready += _n->index; } else { clog(NetConnect) << "Connected to" << _n->id.abridged() << "@" << _n->address; _n->lastConnected = std::chrono::system_clock::now(); - auto p = make_shared(this, std::move(*s), n, true); // true because we don't care about ids matched for now. Once we have permenant IDs this will matter a lot more and we can institute a safer mechanism. + auto p = make_shared(this, std::move(*s), n); p->start(); } delete s; @@ -476,7 +455,7 @@ void Host::connect(std::shared_ptr const& _n) m_pendingNodeConns.erase(nptr); }); else - clog(NetWarn) << "Trying to connect to node not in node table."; + clog(NetWarn) << "Aborted connect. Node not in node table."; } bool Host::havePeer(NodeId _id) const @@ -515,8 +494,8 @@ unsigned NodeInfo::fallbackSeconds() const } } -#warning integration: ---- grow/prunePeers -#warning integration: todo grow/prune into 'maintainPeers' & evaluate reputation instead of availability. schedule via deadline timer. +// TODO P2P: rebuild noetable when localNetworking is enabled/disabled +// TODO P2P: migrate grow/prunePeers into 'maintainPeers' & evaluate reputation instead of availability. schedule via deadline timer. //void Host::growPeers() //{ // RecursiveGuard l(x_peers); @@ -585,14 +564,8 @@ PeerInfos Host::peers() const if (!m_run) return PeerInfos(); -#warning integration: ---- pingAll. It is called every 30secs via deadline timer. - RecursiveGuard l(x_peers); -// if (_updatePing) -// { -// const_cast(this)->pingAll(); -// this_thread::sleep_for(chrono::milliseconds(200)); -// } std::vector ret; + RecursiveGuard l(x_peers); for (auto& i: m_peers) if (auto j = i.second.lock()) if (j->m_socket.is_open()) @@ -604,6 +577,9 @@ void Host::run(boost::system::error_code const&) { if (!m_run) { + // reset NodeTable + m_nodeTable.reset(); + // stopping io service allows running manual network operations for shutdown // and also stops blocking worker thread, allowing worker thread to exit m_ioService.stop(); @@ -612,24 +588,10 @@ void Host::run(boost::system::error_code const&) m_timer.reset(); return; } - -#warning integration: ---- -// m_lastTick += c_timerInterval; -// if (m_lastTick >= c_timerInterval * 10) -// { -// growPeers(); -// prunePeers(); -// m_lastTick = 0; -// } - if (m_hadNewNodes) - { - for (auto p: m_peers) - if (auto pp = p.second.lock()) - pp->serviceNodesRequest(); - - m_hadNewNodes = false; - } + for (auto p: m_peers) + if (auto pp = p.second.lock()) + pp->serviceNodesRequest(); if (chrono::steady_clock::now() - m_lastPing > chrono::seconds(30)) // ping every 30s. { @@ -675,20 +637,13 @@ void Host::startedWorking() if (m_listenPort > 0) runAcceptor(); -#warning integration: ++++ if (!m_tcpPublic.address().is_unspecified()) m_nodeTable.reset(new NodeTable(m_ioService, m_key, m_listenPort, m_tcpPublic)); else m_nodeTable.reset(new NodeTable(m_ioService, m_key, m_listenPort > 0 ? m_listenPort : 30303)); } - -#warning integration: ---- -// // if m_public address is valid then add us to node list -// // todo: abstract empty() and emplace logic -// if (!m_tcpPublic.address().is_unspecified() && (m_nodes.empty() || m_nodes[m_nodesList[0]]->id != id())) -// noteNode(id(), m_tcpPublic, Origin::Perfect, false); - - clog(NetNote) << "Id:" << id().abridged(); + + clog(NetNote) << "p2p.started id:" << id().abridged(); run(boost::system::error_code()); } @@ -708,7 +663,7 @@ void Host::pingAll() m_lastPing = chrono::steady_clock::now(); } -#warning integration: todo save/restoreNodes +// TODO P2P: integration: todo save/restoreNodes bytes Host::saveNodes() const { RLPStream nodes; @@ -726,7 +681,7 @@ bytes Host::saveNodes() const nodes << n.address.address().to_v4().to_bytes(); else nodes << n.address.address().to_v6().to_bytes(); - nodes << n.address.port() << n.id << (int)n.idOrigin + nodes << n.address.port() << n.id /* << (int)n.idOrigin */ << 0 << chrono::duration_cast(n.lastConnected.time_since_epoch()).count() << chrono::duration_cast(n.lastAttempted.time_since_epoch()).count() << n.failedAttempts << (unsigned)n.lastDisconnect << n.score << n.rating; @@ -751,7 +706,7 @@ void Host::restoreNodes(bytesConstRef _b) { auto oldId = id(); m_key = KeyPair(r[1].toHash()); - noteNode(id(), m_tcpPublic, Origin::Perfect, false, oldId); + noteNode(id(), m_tcpPublic, oldId); for (auto i: r[2]) { @@ -763,8 +718,8 @@ void Host::restoreNodes(bytesConstRef _b) auto id = (NodeId)i[2]; if (!m_nodes.count(id)) { - auto o = (Origin)i[3].toInt(); - auto n = noteNode(id, ep, o, true); +// auto o = (Origin)i[3].toInt(); + auto n = noteNode(id, ep); n->lastConnected = chrono::system_clock::time_point(chrono::seconds(i[4].toInt())); n->lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[5].toInt())); n->failedAttempts = i[6].toInt(); @@ -787,7 +742,7 @@ void Host::restoreNodes(bytesConstRef _b) ep = bi::tcp::endpoint(bi::address_v4(i[0].toArray()), i[1].toInt()); else ep = bi::tcp::endpoint(bi::address_v6(i[0].toArray()), i[1].toInt()); - auto n = noteNode(id, ep, Origin::Self, true); + auto n = noteNode(id, ep); } } } diff --git a/libp2p/Host.h b/libp2p/Host.h index 9600c723b..817a844cd 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -51,15 +51,6 @@ namespace p2p class Host; -enum class Origin -{ - Unknown, - Self, - SelfThird, - PerfectThird, - Perfect, -}; - struct NodeInfo { NodeId id; ///< Their id/public key. @@ -81,9 +72,6 @@ struct NodeInfo // p2p: move to protocol-specific map int score = 0; ///< All time cumulative. int rating = 0; ///< Trending. - - // p2p: remove - Origin idOrigin = Origin::Unknown; ///< How did we get to know this node's id? // p2p: move to NodeEndpoint int secondsSinceLastConnected() const { return lastConnected == std::chrono::system_clock::time_point() ? -1 : (int)std::chrono::duration_cast(std::chrono::system_clock::now() - lastConnected).count(); } @@ -125,6 +113,7 @@ using Nodes = std::vector; /** * @brief The Host class * Capabilities should be registered prior to startNetwork, since m_capabilities is not thread-safe. + * @todo determinePublic: ipv6, udp */ class Host: public Worker { @@ -151,8 +140,8 @@ public: /// Connect to a peer explicitly. static std::string pocHost(); - void connect(std::string const& _addr, unsigned short _port = 30303) noexcept; - void connect(bi::tcp::endpoint const& _ep); + void connect(NodeId const& _node, std::string const& _addr, unsigned short _port = 30303) noexcept; + void connect(NodeId const& _node, bi::tcp::endpoint const& _ep); void connect(std::shared_ptr const& _n); /// @returns true iff we have a peer of the given id. @@ -209,6 +198,9 @@ private: /// Called only from startedWorking(). void runAcceptor(); + /// Handler for verifying handshake siganture before creating session. _egressNodeId is passed for outbound connections. + void doHandshake(bi::tcp::socket* _socket, NodeId _egressNodeId = NodeId()); + void seal(bytes& _b); /// Called by Worker. Not thread-safe; to be called only by worker. @@ -222,7 +214,7 @@ private: /// Shutdown network. Not thread-safe; to be called only by worker. virtual void doneWorking(); - std::shared_ptr noteNode(NodeId _id, bi::tcp::endpoint _a, Origin _o, bool _ready, NodeId _oldId = NodeId()); + std::shared_ptr noteNode(NodeId _id, bi::tcp::endpoint _a, NodeId _oldId = NodeId()); Nodes potentialPeers(RangeMask const& _known); bool m_run = false; ///< Whether network is running. @@ -252,8 +244,6 @@ private: std::shared_ptr m_nodeTable; ///< Node table (uses kademlia-like discovery). std::map m_capIdealPeerCount; ///< Ideal peer count for capability. - bool m_hadNewNodes = false; - mutable RecursiveMutex x_peers; /// The nodes to which we are currently connected. @@ -261,13 +251,12 @@ private: mutable std::map> m_peers; /// Nodes to which we may connect (or to which we have connected). - /// TODO: does this need a lock? + /// TODO: mutex; replace with nodeTable std::map > m_nodes; /// A list of node IDs. This contains every index from m_nodes; the order is guaranteed to remain the same. std::vector m_nodesList; - RangeMask m_ready; ///< Indices into m_nodesList over to which nodes we are not currently connected, connecting or otherwise ignoring. RangeMask m_private; ///< Indices into m_nodesList over to which nodes are private. unsigned m_idealPeerCount = 5; ///< Ideal number of peers to be connected to. diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index abf762fff..699f0d0c9 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -239,29 +239,27 @@ void NodeTable::evict(shared_ptr _leastSeen, shared_ptr _n ping(_leastSeen.get()); } +shared_ptr NodeTable::addNode(Public const& _pubk, bi::udp::endpoint const& _udp, bi::tcp::endpoint const& _tcp) +{ + shared_ptr ret; + Guard l(x_nodes); + if (auto n = m_nodes[_pubk]) + ret = n; + else + { + ret.reset(new NodeEntry(m_node, _pubk, _udp)); + m_nodes[_pubk] = ret; + } + return move(ret); +} + void NodeTable::noteNode(Public const& _pubk, bi::udp::endpoint const& _endpoint) { - // Don't add ourself if (_pubk == m_node.address()) return; - shared_ptr node; - { - Guard l(x_nodes); - auto n = m_nodes.find(_pubk); - if (n == m_nodes.end()) - { - node.reset(new NodeEntry(m_node, _pubk, _endpoint)); - m_nodes[_pubk] = node; -// clog(NodeTableMessageSummary) << "Adding node to cache: " << _pubk; - } - else - { - node = n->second; -// clog(NodeTableMessageSummary) << "Found node in cache: " << _pubk; - } - } - + shared_ptr node(addNode(_pubk, _endpoint)); + // todo: sometimes node is nullptr here if (!!node) noteNode(node); diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index d99119f02..d209018a2 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -131,6 +131,9 @@ public: static unsigned dist(NodeId const& _a, NodeId const& _b) { u512 d = _a ^ _b; unsigned ret; for (ret = 0; d >>= 1; ++ret) {}; return ret; } + /// Add node details and attempt adding to node table if node responds to ping. NodeEntry will immediately be returned and may be used for required connectivity. + std::shared_ptr addNode(Public const& _pubk, bi::udp::endpoint const& _udp, bi::tcp::endpoint const& _tcp = bi::tcp::endpoint()); + void join(); NodeEntry root() const { return NodeEntry(m_node, m_node.publicKey(), m_node.endpoint.udp); } @@ -192,7 +195,7 @@ protected: Secret m_secret; ///< This nodes secret key. mutable Mutex x_nodes; ///< Mutable for thread-safe copy in nodes() const. - std::map> m_nodes; ///< NodeId -> Node table (most common lookup path) + std::map> m_nodes; ///< NodeId -> Node table mutable Mutex x_state; std::array m_state; ///< State table of binned nodes. diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index d53a89f09..c31335357 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -36,23 +36,11 @@ using namespace dev::p2p; #endif #define clogS(X) dev::LogOutputStream(false) << "| " << std::setw(2) << m_socket.native_handle() << "] " -Session::Session(Host* _s, bi::tcp::socket _socket, bi::tcp::endpoint const& _manual): - m_server(_s), - m_socket(std::move(_socket)), - m_node(nullptr), - m_manualEndpoint(_manual) // NOTE: the port on this shouldn't be used if it's zero. -{ - m_lastReceived = m_connect = std::chrono::steady_clock::now(); - - m_info = PeerInfo({NodeId(), "?", m_manualEndpoint.address().to_string(), 0, std::chrono::steady_clock::duration(0), CapDescSet(), 0, map()}); -} - -Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr const& _n, bool _force): +Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr const& _n): m_server(_s), m_socket(std::move(_socket)), m_node(_n), - m_manualEndpoint(_n->address), - m_force(_force) + m_manualEndpoint(_n->address) { m_lastReceived = m_connect = std::chrono::steady_clock::now(); m_info = PeerInfo({m_node->id, "?", _n->address.address().to_string(), _n->address.port(), std::chrono::steady_clock::duration(0), CapDescSet(), 0, map()}); @@ -60,13 +48,9 @@ Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr co Session::~Session() { - if (m_node) - { - if (id() && !isPermanentProblem(m_node->lastDisconnect) && !m_node->dead) - m_server->m_ready += m_node->index; - else - m_node->lastConnected = m_node->lastAttempted - chrono::seconds(1); - } + // TODO P2P: revisit (refactored from previous logic) + if (m_node && !(id() && !isPermanentProblem(m_node->lastDisconnect) && !m_node->dead)) + m_node->lastConnected = m_node->lastAttempted - chrono::seconds(1); // Read-chain finished for one reason or another. for (auto& i: m_capabilities) @@ -103,6 +87,7 @@ int Session::rating() const return m_node->rating; } +// TODO P2P: integration: session->? should be unavailable when socket isn't open bi::tcp::endpoint Session::endpoint() const { if (m_socket.is_open() && m_node) @@ -132,6 +117,7 @@ template vector randomSelection(vector const& _t, unsigned _n) return ret; } +// TODO P2P: integration: replace w/asio post -> serviceNodesRequest() void Session::ensureNodesRequested() { if (isOpen() && !m_weRequestedNodes) @@ -207,25 +193,8 @@ bool Session::interpret(RLP const& _r) return true; } - if (m_node && m_node->id != id) - { - if (m_force || m_node->idOrigin <= Origin::SelfThird) - // SECURITY: We're forcing through the new ID, despite having been told - clogS(NetWarn) << "Connected to node, but their ID has changed since last time. This could indicate a MitM attack. Allowing anyway..."; - else - { - clogS(NetWarn) << "Connected to node, but their ID has changed since last time. This could indicate a MitM attack. Disconnecting."; - disconnect(UnexpectedIdentity); - return true; - } - - if (m_server->havePeer(id)) - { - m_node->dead = true; - disconnect(DuplicatePeer); - return true; - } - } + assert(!!m_node); + assert(!!m_node->id); if (m_server->havePeer(id)) { @@ -241,9 +210,14 @@ bool Session::interpret(RLP const& _r) return true; } - m_node = m_server->noteNode(id, bi::tcp::endpoint(m_socket.remote_endpoint().address(), listenPort), Origin::Self, false, !m_node || m_node->id == id ? NodeId() : m_node->id); + // TODO P2P: first pass, implement signatures. if signature fails, drop connection. if egress, flag node's endpoint as stale. + // TODO P2P: remove oldid + // TODO P2P: with encrypted transport the handshake will fail and we won't get here + m_node = m_server->noteNode(id, bi::tcp::endpoint(m_socket.remote_endpoint().address(), listenPort), m_node->id); if (m_node->isOffline()) m_node->lastConnected = chrono::system_clock::now(); + + // TODO P2P: introduce map of nodes we've given to this node (if GetPeers/Peers stays in TCP) m_knownNodes.extendAll(m_node->index); m_knownNodes.unionWith(m_node->index); @@ -360,7 +334,7 @@ bool Session::interpret(RLP const& _r) // OK passed all our checks. Assume it's good. addRating(1000); - m_server->noteNode(id, ep, m_node->idOrigin == Origin::Perfect ? Origin::PerfectThird : Origin::SelfThird, true); + m_server->noteNode(id, ep); clogS(NetTriviaDetail) << "New peer: " << ep << "(" << id .abridged()<< ")"; CONTINUE:; } diff --git a/libp2p/Session.h b/libp2p/Session.h index 869eca96e..19dc60a28 100644 --- a/libp2p/Session.h +++ b/libp2p/Session.h @@ -51,8 +51,7 @@ class Session: public std::enable_shared_from_this friend class HostCapabilityFace; public: - Session(Host* _server, bi::tcp::socket _socket, std::shared_ptr const& _n, bool _force = false); - Session(Host* _server, bi::tcp::socket _socket, bi::tcp::endpoint const& _manual); + Session(Host* _server, bi::tcp::socket _socket, std::shared_ptr const& _n); virtual ~Session(); void start(); @@ -113,9 +112,8 @@ private: PeerInfo m_info; ///< Dynamic information about this peer. unsigned m_protocolVersion = 0; ///< The protocol version of the peer. - std::shared_ptr m_node; ///< The NodeInfo object. Might be null if we constructed using a bare address/port. + std::shared_ptr m_node; ///< The NodeInfo object. bi::tcp::endpoint m_manualEndpoint; ///< The endpoint as specified by the constructor. - bool m_force = false; ///< If true, ignore IDs being different. This could open you up to MitM attacks. bool m_dropped = false; ///< If true, we've already divested ourselves of this peer. We're just waiting for the reads & writes to fail before the shared_ptr goes OOS and the destructor kicks in. bool m_theyRequestedNodes = false; ///< Has the peer requested nodes from us without receiveing an answer from us? diff --git a/libwebthree/WebThree.cpp b/libwebthree/WebThree.cpp index c9f9d56e3..fd51b5947 100644 --- a/libwebthree/WebThree.cpp +++ b/libwebthree/WebThree.cpp @@ -102,5 +102,5 @@ void WebThreeDirect::restoreNodes(bytesConstRef _saved) void WebThreeDirect::connect(std::string const& _seedHost, unsigned short _port) { - m_net.connect(_seedHost, _port); + m_net.connect(NodeId(), _seedHost, _port); } diff --git a/test/peer.cpp b/test/peer.cpp index a99ce7201..70d03e9bf 100644 --- a/test/peer.cpp +++ b/test/peer.cpp @@ -49,7 +49,7 @@ int peerTest(int argc, char** argv) Host ph("Test", NetworkPreferences(listenPort)); if (!remoteHost.empty()) - ph.connect(remoteHost, remotePort); + ph.connect(NodeId(), remoteHost, remotePort); for (int i = 0; ; ++i) { diff --git a/test/whisperTopic.cpp b/test/whisperTopic.cpp index 79adf3d6a..8ecabde9b 100644 --- a/test/whisperTopic.cpp +++ b/test/whisperTopic.cpp @@ -72,7 +72,7 @@ BOOST_AUTO_TEST_CASE(topic) this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); this_thread::sleep_for(chrono::milliseconds(500)); - ph.connect("127.0.0.1", 50303); + ph.connect(NodeId(), "127.0.0.1", 50303); KeyPair us = KeyPair::create(); for (int i = 0; i < 10; ++i) From fa11fc01490b7b32f397531b859ee47c36b927d4 Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 8 Jan 2015 18:36:58 +0100 Subject: [PATCH 03/71] Persist host identifier to disk. Replace noteNode with addNode. Add udp node port to connect. Add addNode to node table which pings node, potentially adding node to table if node respons. Rename NodeEndpoint to NodeIPEndpoint. --- libp2p/Host.cpp | 184 ++++++++++++++++++++++++------------------- libp2p/Host.h | 34 +++++--- libp2p/NodeTable.cpp | 19 +++-- libp2p/NodeTable.h | 40 ++++++---- libp2p/Session.cpp | 28 ++++--- 5 files changed, 182 insertions(+), 123 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index ffc63cf8e..392d02dc4 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include "Session.h" #include "Common.h" #include "Capability.h" @@ -44,14 +45,14 @@ Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n, bool m_ifAddresses(Network::getInterfaceAddresses()), m_ioService(2), m_tcp4Acceptor(m_ioService), - m_key(KeyPair::create()), + m_key(move(getHostIdentifier())), m_nodeTable(new NodeTable(m_ioService, m_key)) { for (auto address: m_ifAddresses) if (address.is_v4()) clog(NetNote) << "IP Address: " << address << " = " << (isPrivateAddress(address) ? "[LOCAL]" : "[PEER]"); - clog(NetNote) << "Id:" << id().abridged(); + clog(NetNote) << "Id:" << id(); if (_start) start(); } @@ -178,69 +179,78 @@ void Host::seal(bytes& _b) _b[7] = len & 0xff; } -// TODO P2P: remove oldid. port to NodeTable. (see noteNode calls, Session.cpp#218,337) -shared_ptr Host::noteNode(NodeId _id, bi::tcp::endpoint _a, NodeId _oldId) -{ - RecursiveGuard l(x_peers); - if (_a.port() < 30300 || _a.port() > 30305) - cwarn << "Weird port being recorded: " << _a.port(); - - if (_a.port() >= /*49152*/32768) - { - cwarn << "Private port being recorded - setting to 0"; - _a = bi::tcp::endpoint(_a.address(), 0); - } +// TODO: P2P port to NodeTable. (see noteNode calls, Session.cpp) +//shared_ptr Host::noteNode(NodeId _id, bi::tcp::endpoint _a) +//{ +// RecursiveGuard l(x_peers); +// if (_a.port() < 30300 || _a.port() > 30305) +// cwarn << "Weird port being recorded: " << _a.port(); +// +// if (_a.port() >= /*49152*/32768) +// { +// cwarn << "Private port being recorded - setting to 0"; +// _a = bi::tcp::endpoint(_a.address(), 0); +// } +// +// unsigned i; +// if (!m_nodes.count(_id)) +// { +// i = m_nodesList.size(); +// m_nodesList.push_back(_id); +// m_nodes[_id] = make_shared(); +// m_nodes[_id]->id = _id; +// m_nodes[_id]->index = i; +// } +// else +// i = m_nodes[_id]->index; +// m_nodes[_id]->address = _a; +// m_private.extendAll(i); +// if (!_a.port() || (isPrivateAddress(_a.address()) && !m_netPrefs.localNetworking)) +// m_private += i; +// else +// m_private -= i; +// +// return m_nodes[_id]; +//} - // First check for another node with the same connection credentials, and put it in oldId if found. - if (!_oldId) - for (pair> const& n: m_nodes) - if (n.second->address == _a && n.second->id != _id) - { - _oldId = n.second->id; - break; - } +// TODO: P2P base on target +// TODO: P2P store caps in NodeTable/NodeEntry +//Nodes Host::potentialPeers(RangeMask const& _known) +//{ +// RecursiveGuard l(x_peers); +// Nodes ret; +// +// // todo: if localnetworking is enabled it should only share peers if remote +// // is within the same network as our interfaces. +// // this requires flagging nodes when we receive them as to if they're on private network +// auto ns = (m_netPrefs.localNetworking ? _known : (m_private + _known)).inverted(); +// for (auto i: ns) +// ret.push_back(*m_nodes[m_nodesList[i]]); +// return ret; +//} - unsigned i; - if (!m_nodes.count(_id)) +KeyPair Host::getHostIdentifier() +{ + static string s_file(getDataDir() + "/host"); + static mutex s_x; + lock_guard l(s_x); + + h256 secret; + bytes b = contents(s_file); + if (b.size() == 32) + memcpy(secret.data(), b.data(), 32); + else { - if (m_nodes.count(_oldId)) - { - i = m_nodes[_oldId]->index; - m_nodes.erase(_oldId); - m_nodesList[i] = _id; - } - else - { - i = m_nodesList.size(); - m_nodesList.push_back(_id); - } - m_nodes[_id] = make_shared(); - m_nodes[_id]->id = _id; - m_nodes[_id]->index = i; + // todo: replace w/user entropy; abstract to devcrypto + std::mt19937_64 s_eng(time(0) + chrono::high_resolution_clock::now().time_since_epoch().count()); + std::uniform_int_distribution d(0, 255); + for (unsigned i = 0; i < 32; ++i) + secret[i] = (byte)d(s_eng); } - else - i = m_nodes[_id]->index; - m_nodes[_id]->address = _a; - m_private.extendAll(i); - if (!_a.port() || (isPrivateAddress(_a.address()) && !m_netPrefs.localNetworking)) - m_private += i; - else - m_private -= i; - - return m_nodes[_id]; -} - -// TODO P2P: should be based on target -// TODO P2P: store caps in NodeTable/NodeEntry -Nodes Host::potentialPeers(RangeMask const& _known) -{ - RecursiveGuard l(x_peers); - Nodes ret; - - auto ns = (m_netPrefs.localNetworking ? _known : (m_private + _known)).inverted(); - for (auto i: ns) - ret.push_back(*m_nodes[m_nodesList[i]]); - return ret; + + if (!secret) + BOOST_THROW_EXCEPTION(crypto::InvalidState()); + return move(KeyPair(move(secret))); } void Host::determinePublic(string const& _publicAddress, bool _upnp) @@ -366,9 +376,25 @@ string Host::pocHost() return "poc-" + strs[1] + ".ethdev.com"; } -// TODO P2P: support for TCP+UDP when manually connecting -// TODO P2P: remove in favor of addNode(NodeId, string, uint16_t, bool _required = false) -void Host::connect(NodeId const& _node, std::string const& _addr, unsigned short _port) noexcept +void Host::addNode(NodeId const& _node, std::string const& _addr, unsigned short _tcpPeerPort, unsigned short _udpNodePort) +{ + boost::system::error_code ec; + bi::address addr = bi::address::from_string(_addr, ec); + if (ec) + { + bi::tcp::resolver r(m_ioService); + r.async_resolve({_addr, toString(_tcpPeerPort)}, [=](boost::system::error_code const& _ec, bi::tcp::resolver::iterator _epIt) { + if (_ec) + return; + bi::tcp::endpoint tcp = *_epIt; + addNode(Node(_node, NodeIPEndpoint(bi::udp::endpoint(tcp.address(), _udpNodePort), tcp))); + }); + } + else + addNode(Node(_node, NodeIPEndpoint(bi::udp::endpoint(addr, _udpNodePort), bi::tcp::endpoint(addr, _tcpPeerPort)))); +} + +void Host::connect(NodeId const& _node, std::string const& _addr, unsigned short _peerPort, unsigned short _nodePort) noexcept { if (!m_run) return; @@ -388,10 +414,10 @@ void Host::connect(NodeId const& _node, std::string const& _addr, unsigned short if (first) { bi::tcp::resolver r(m_ioService); - ep = r.resolve({_addr, toString(_port)})->endpoint(); + ep = r.resolve({_addr, toString(_peerPort)})->endpoint(); } else - ep = bi::tcp::endpoint(bi::address::from_string(_addr), _port); + ep = bi::tcp::endpoint(bi::address::from_string(_addr), _peerPort); if (!n) m_nodes[_node] = make_shared(); @@ -494,8 +520,8 @@ unsigned NodeInfo::fallbackSeconds() const } } -// TODO P2P: rebuild noetable when localNetworking is enabled/disabled -// TODO P2P: migrate grow/prunePeers into 'maintainPeers' & evaluate reputation instead of availability. schedule via deadline timer. +// TODO: P2P rebuild noetable when localNetworking is enabled/disabled +// TODO: P2P migrate grow/prunePeers into 'maintainPeers' & evaluate reputation instead of availability. schedule via deadline timer. //void Host::growPeers() //{ // RecursiveGuard l(x_peers); @@ -663,7 +689,6 @@ void Host::pingAll() m_lastPing = chrono::steady_clock::now(); } -// TODO P2P: integration: todo save/restoreNodes bytes Host::saveNodes() const { RLPStream nodes; @@ -704,9 +729,8 @@ void Host::restoreNodes(bytesConstRef _b) { case 0: { - auto oldId = id(); m_key = KeyPair(r[1].toHash()); - noteNode(id(), m_tcpPublic, oldId); +// noteNode(id(), m_tcpPublic); for (auto i: r[2]) { @@ -718,14 +742,14 @@ void Host::restoreNodes(bytesConstRef _b) auto id = (NodeId)i[2]; if (!m_nodes.count(id)) { -// auto o = (Origin)i[3].toInt(); - auto n = noteNode(id, ep); - n->lastConnected = chrono::system_clock::time_point(chrono::seconds(i[4].toInt())); - n->lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[5].toInt())); - n->failedAttempts = i[6].toInt(); - n->lastDisconnect = (DisconnectReason)i[7].toInt(); - n->score = (int)i[8].toInt(); - n->rating = (int)i[9].toInt(); +//// auto o = (Origin)i[3].toInt(); +// auto n = noteNode(id, ep); +// n->lastConnected = chrono::system_clock::time_point(chrono::seconds(i[4].toInt())); +// n->lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[5].toInt())); +// n->failedAttempts = i[6].toInt(); +// n->lastDisconnect = (DisconnectReason)i[7].toInt(); +// n->score = (int)i[8].toInt(); +// n->rating = (int)i[9].toInt(); } } } @@ -742,7 +766,7 @@ void Host::restoreNodes(bytesConstRef _b) ep = bi::tcp::endpoint(bi::address_v4(i[0].toArray()), i[1].toInt()); else ep = bi::tcp::endpoint(bi::address_v6(i[0].toArray()), i[1].toInt()); - auto n = noteNode(id, ep); +// auto n = noteNode(id, ep); } } } diff --git a/libp2p/Host.h b/libp2p/Host.h index 817a844cd..edfa79df3 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -56,7 +56,7 @@ struct NodeInfo NodeId id; ///< Their id/public key. unsigned index; ///< Index into m_nodesList - // p2p: move to NodeEndpoint + // p2p: move to NodeIPEndpoint bi::tcp::endpoint address; ///< As reported from the node itself. // p2p: This information is relevant to the network-stack, ex: firewall, rather than node itself @@ -73,14 +73,14 @@ struct NodeInfo int score = 0; ///< All time cumulative. int rating = 0; ///< Trending. - // p2p: move to NodeEndpoint + // p2p: move to NodeIPEndpoint int secondsSinceLastConnected() const { return lastConnected == std::chrono::system_clock::time_point() ? -1 : (int)std::chrono::duration_cast(std::chrono::system_clock::now() - lastConnected).count(); } - // p2p: move to NodeEndpoint + // p2p: move to NodeIPEndpoint int secondsSinceLastAttempted() const { return lastAttempted == std::chrono::system_clock::time_point() ? -1 : (int)std::chrono::duration_cast(std::chrono::system_clock::now() - lastAttempted).count(); } - // p2p: move to NodeEndpoint + // p2p: move to NodeIPEndpoint unsigned fallbackSeconds() const; - // p2p: move to NodeEndpoint + // p2p: move to NodeIPEndpoint bool shouldReconnect() const { return std::chrono::system_clock::now() > lastAttempted + std::chrono::seconds(fallbackSeconds()); } // p2p: This has two meanings now. It's possible UDP works but TPC is down (unable to punch hole). @@ -114,6 +114,7 @@ using Nodes = std::vector; * @brief The Host class * Capabilities should be registered prior to startNetwork, since m_capabilities is not thread-safe. * @todo determinePublic: ipv6, udp + * @todo handle conflict if addNode/requireNode called and Node already exists w/conflicting tcp or udp port */ class Host: public Worker { @@ -128,6 +129,9 @@ public: /// Will block on network process events. virtual ~Host(); + /// Default host for current version of client. + static std::string pocHost(); + /// Basic peer network protocol version. unsigned protocolVersion() const; @@ -137,10 +141,12 @@ public: bool haveCapability(CapDesc const& _name) const { return m_capabilities.count(_name) != 0; } CapDescs caps() const { CapDescs ret; for (auto const& i: m_capabilities) ret.push_back(i.first); return ret; } template std::shared_ptr cap() const { try { return std::static_pointer_cast(m_capabilities.at(std::make_pair(T::staticName(), T::staticVersion()))); } catch (...) { return nullptr; } } + + /// Manually add node. + void addNode(NodeId const& _node, std::string const& _addr, unsigned short _tcpPort = 30303, unsigned short _udpPort = 30303); /// Connect to a peer explicitly. - static std::string pocHost(); - void connect(NodeId const& _node, std::string const& _addr, unsigned short _port = 30303) noexcept; + void connect(NodeId const& _node, std::string const& _addr, unsigned short _tcpPort = 30303, unsigned short _udpPort = 30303) noexcept; void connect(NodeId const& _node, bi::tcp::endpoint const& _ep); void connect(std::shared_ptr const& _n); @@ -192,6 +198,8 @@ public: std::shared_ptr node(NodeId _id) const { if (m_nodes.count(_id)) return m_nodes.at(_id); return std::shared_ptr(); } private: + KeyPair getHostIdentifier(); + /// Populate m_peerAddresses with available public addresses. void determinePublic(std::string const& _publicAddress, bool _upnp); @@ -213,9 +221,11 @@ private: /// Shutdown network. Not thread-safe; to be called only by worker. virtual void doneWorking(); + + /// Add node + void addNode(Node const& _nodeInfo) { m_nodeTable->addNode(_nodeInfo); } - std::shared_ptr noteNode(NodeId _id, bi::tcp::endpoint _a, NodeId _oldId = NodeId()); - Nodes potentialPeers(RangeMask const& _known); +// Nodes potentialPeers(RangeMask const& _known); bool m_run = false; ///< Whether network is running. std::mutex x_runTimer; ///< Start/stop mutex. @@ -254,10 +264,10 @@ private: /// TODO: mutex; replace with nodeTable std::map > m_nodes; - /// A list of node IDs. This contains every index from m_nodes; the order is guaranteed to remain the same. - std::vector m_nodesList; +// /// A list of node IDs. This contains every index from m_nodes; the order is guaranteed to remain the same. +// std::vector m_nodesList; - RangeMask m_private; ///< Indices into m_nodesList over to which nodes are private. +// RangeMask m_private; ///< Indices into m_nodesList over to which nodes are private. unsigned m_idealPeerCount = 5; ///< Ideal number of peers to be connected to. diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 699f0d0c9..53acc9dbe 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -24,8 +24,8 @@ using namespace std; using namespace dev; using namespace dev::p2p; -NodeEntry::NodeEntry(Node _src, Public _pubk, NodeDefaultEndpoint _gw): Node(_pubk, _gw), distance(NodeTable::dist(_src.id,_pubk)) {} -NodeEntry::NodeEntry(Node _src, Public _pubk, bi::udp::endpoint _udp): Node(_pubk, NodeDefaultEndpoint(_udp)), distance(NodeTable::dist(_src.id,_pubk)) {} +NodeEntry::NodeEntry(Node _src, Public _pubk, NodeIPEndpoint _gw): Node(_pubk, _gw), distance(NodeTable::dist(_src.id,_pubk)) {} +NodeEntry::NodeEntry(Node _src, Public _pubk, bi::udp::endpoint _udp): Node(_pubk, NodeIPEndpoint(_udp)), distance(NodeTable::dist(_src.id,_pubk)) {} NodeTable::NodeTable(ba::io_service& _io, KeyPair _alias, uint16_t _udp, bi::tcp::endpoint _ep): m_node(Node(_alias.pub(), bi::udp::endpoint())), @@ -240,15 +240,24 @@ void NodeTable::evict(shared_ptr _leastSeen, shared_ptr _n } shared_ptr NodeTable::addNode(Public const& _pubk, bi::udp::endpoint const& _udp, bi::tcp::endpoint const& _tcp) +{ + auto node = Node(_pubk, NodeIPEndpoint(_udp, _tcp)); + return move(addNode(node)); +} + +shared_ptr NodeTable::addNode(Node const& _node) { shared_ptr ret; Guard l(x_nodes); - if (auto n = m_nodes[_pubk]) + if (auto n = m_nodes[_node.id]) ret = n; else { - ret.reset(new NodeEntry(m_node, _pubk, _udp)); - m_nodes[_pubk] = ret; + ret.reset(new NodeEntry(m_node, _node.id, NodeIPEndpoint(_node.endpoint.udp, _node.endpoint.tcp))); + m_nodes[_node.id] = ret; + PingNode p(_node.endpoint.udp, m_node.endpoint.udp.address().to_string(), m_node.endpoint.udp.port()); + p.sign(m_secret); + m_socketPtr->send(p); } return move(ret); } diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index d209018a2..89bd68be1 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -31,37 +31,45 @@ namespace dev namespace p2p { -struct NodeDefaultEndpoint +/** + * @brief IPv4,UDP/TCP endpoints. + */ +struct NodeIPEndpoint { - NodeDefaultEndpoint(bi::udp::endpoint _udp): udp(_udp) {} - NodeDefaultEndpoint(bi::tcp::endpoint _tcp): tcp(_tcp) {} - NodeDefaultEndpoint(bi::udp::endpoint _udp, bi::tcp::endpoint _tcp): udp(_udp), tcp(_tcp) {} - + NodeIPEndpoint(bi::udp::endpoint _udp): udp(_udp) {} + NodeIPEndpoint(bi::tcp::endpoint _tcp): tcp(_tcp) {} + NodeIPEndpoint(bi::udp::endpoint _udp, bi::tcp::endpoint _tcp): udp(_udp), tcp(_tcp) {} + bi::udp::endpoint udp; bi::tcp::endpoint tcp; }; struct Node { - Node(Public _pubk, NodeDefaultEndpoint _udp): id(_pubk), endpoint(_udp) {} - Node(Public _pubk, bi::udp::endpoint _udp): Node(_pubk, NodeDefaultEndpoint(_udp)) {} + Node(Public _pubk, NodeIPEndpoint _ip, bool _required = false): id(_pubk), endpoint(_ip), required(_required) {} + Node(Public _pubk, bi::udp::endpoint _udp, bool _required = false): Node(_pubk, NodeIPEndpoint(_udp), _required) {} virtual NodeId const& address() const { return id; } virtual Public const& publicKey() const { return id; } NodeId id; - NodeDefaultEndpoint endpoint; + + /// Endpoints by which we expect to reach node. + NodeIPEndpoint endpoint; + + /// If true, node will not be removed from Node list. + bool required = false; }; - + /** * NodeEntry * @brief Entry in Node Table */ struct NodeEntry: public Node { - NodeEntry(Node _src, Public _pubk, NodeDefaultEndpoint _gw); //: Node(_pubk, _gw), distance(dist(_src.id,_pubk)) {} - NodeEntry(Node _src, Public _pubk, bi::udp::endpoint _udp); //: Node(_pubk, NodeDefaultEndpoint(_udp)), distance(dist(_src.id,_pubk)) {} + NodeEntry(Node _src, Public _pubk, NodeIPEndpoint _gw); //: Node(_pubk, _gw), distance(dist(_src.id,_pubk)) {} + NodeEntry(Node _src, Public _pubk, bi::udp::endpoint _udp); //: Node(_pubk, NodeIPEndpoint(_udp)), distance(dist(_src.id,_pubk)) {} const unsigned distance; ///< Node's distance from _src (see constructor). }; @@ -131,9 +139,12 @@ public: static unsigned dist(NodeId const& _a, NodeId const& _b) { u512 d = _a ^ _b; unsigned ret; for (ret = 0; d >>= 1; ++ret) {}; return ret; } - /// Add node details and attempt adding to node table if node responds to ping. NodeEntry will immediately be returned and may be used for required connectivity. + /// Add node. Node will be pinged if it's not already known. std::shared_ptr addNode(Public const& _pubk, bi::udp::endpoint const& _udp, bi::tcp::endpoint const& _tcp = bi::tcp::endpoint()); + /// Add node. Node will be pinged if it's not already known. + std::shared_ptr addNode(Node const& _node); + void join(); NodeEntry root() const { return NodeEntry(m_node, m_node.publicKey(), m_node.endpoint.udp); } @@ -142,7 +153,6 @@ public: NodeEntry operator[](NodeId _id); - protected: struct NodeBucket { @@ -195,10 +205,10 @@ protected: Secret m_secret; ///< This nodes secret key. mutable Mutex x_nodes; ///< Mutable for thread-safe copy in nodes() const. - std::map> m_nodes; ///< NodeId -> Node table + std::map> m_nodes; ///< Nodes mutable Mutex x_state; - std::array m_state; ///< State table of binned nodes. + std::array m_state; ///< State of p2p node network. Mutex x_evictions; std::deque m_evictions; ///< Eviction timeouts. diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index c31335357..c6797b69c 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -48,7 +48,7 @@ Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr co Session::~Session() { - // TODO P2P: revisit (refactored from previous logic) + // TODO: P2P revisit (refactored from previous logic) if (m_node && !(id() && !isPermanentProblem(m_node->lastDisconnect) && !m_node->dead)) m_node->lastConnected = m_node->lastAttempted - chrono::seconds(1); @@ -87,7 +87,7 @@ int Session::rating() const return m_node->rating; } -// TODO P2P: integration: session->? should be unavailable when socket isn't open +// TODO: P2P integration: session->? should be unavailable when socket isn't open bi::tcp::endpoint Session::endpoint() const { if (m_socket.is_open() && m_node) @@ -117,7 +117,7 @@ template vector randomSelection(vector const& _t, unsigned _n) return ret; } -// TODO P2P: integration: replace w/asio post -> serviceNodesRequest() +// TODO: P2P integration: replace w/asio post -> serviceNodesRequest() void Session::ensureNodesRequested() { if (isOpen() && !m_weRequestedNodes) @@ -133,7 +133,9 @@ void Session::serviceNodesRequest() if (!m_theyRequestedNodes) return; - auto peers = m_server->potentialPeers(m_knownNodes); +// TODO: P2P +// auto peers = m_server->potentialPeers(m_knownNodes); + Nodes peers; if (peers.empty()) { addNote("peers", "requested"); @@ -210,14 +212,16 @@ bool Session::interpret(RLP const& _r) return true; } - // TODO P2P: first pass, implement signatures. if signature fails, drop connection. if egress, flag node's endpoint as stale. - // TODO P2P: remove oldid - // TODO P2P: with encrypted transport the handshake will fail and we won't get here - m_node = m_server->noteNode(id, bi::tcp::endpoint(m_socket.remote_endpoint().address(), listenPort), m_node->id); + // TODO: P2P first pass, implement signatures. if signature fails, drop connection. if egress, flag node's endpoint as stale. + // Discussion: Most this to Host so we consolidate authentication logic and eschew peer deduplication logic. + // TODO: P2P Move all node-lifecycle information into Host. Determine best way to handle peer-lifecycle properties vs node lifecycle. + // TODO: P2P remove oldid + // TODO: P2P with encrypted transport the handshake will fail and we won't get here +// m_node = m_server->noteNode(m_node->id, bi::tcp::endpoint(m_socket.remote_endpoint().address(), listenPort)); if (m_node->isOffline()) m_node->lastConnected = chrono::system_clock::now(); - - // TODO P2P: introduce map of nodes we've given to this node (if GetPeers/Peers stays in TCP) +// +// // TODO: P2P introduce map of nodes we've given to this node (if GetPeers/Peers stays in TCP) m_knownNodes.extendAll(m_node->index); m_knownNodes.unionWith(m_node->index); @@ -334,7 +338,9 @@ bool Session::interpret(RLP const& _r) // OK passed all our checks. Assume it's good. addRating(1000); - m_server->noteNode(id, ep); + + // TODO: P2P change to addNode() +// m_server->noteNode(id, ep); clogS(NetTriviaDetail) << "New peer: " << ep << "(" << id .abridged()<< ")"; CONTINUE:; } From 22efa458330bb9297d82c388d3bb6456e1cb23d5 Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 8 Jan 2015 22:20:11 +0100 Subject: [PATCH 04/71] remove private ports from upnp port range --- libp2p/UPnP.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libp2p/UPnP.cpp b/libp2p/UPnP.cpp index 42868d67a..36795c540 100644 --- a/libp2p/UPnP.cpp +++ b/libp2p/UPnP.cpp @@ -132,7 +132,7 @@ int UPnP::addRedirect(char const* _addr, int _port) srand(time(NULL)); for (unsigned i = 0; i < 10; ++i) { - _port = rand() % (65535 - 1024) + 1024; + _port = rand() % (32768 - 1024) + 1024; sprintf(ext_port_str, "%d", _port); if (!UPNP_AddPortMapping(m_urls->controlURL, m_data->first.servicetype, ext_port_str, port_str, _addr, "ethereum", "TCP", NULL, NULL)) return _port; From 40e07b312a973992317bf7fc19857bad0596a482 Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 8 Jan 2015 22:26:21 +0100 Subject: [PATCH 05/71] Consolidate use of pingAll into keepAlivePeers. Add bool operators for Node and NodeIPEndpoint population. NodeTable returns Node instead of NodeEntry (subject to change). Begin transition from NodeInfo to NodeTable Node. --- libp2p/Host.cpp | 26 +++++++++++++------------- libp2p/Host.h | 4 ++-- libp2p/NodeTable.cpp | 5 +++-- libp2p/NodeTable.h | 8 +++++++- test/peer.cpp | 2 +- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 392d02dc4..5f756f2a5 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -401,7 +401,7 @@ void Host::connect(NodeId const& _node, std::string const& _addr, unsigned short assert(_node); - auto n = m_nodes[_node]; + auto n = (*m_nodeTable)[_node]; // TODO: refactor into async_resolve m_ioService.post([=]() @@ -619,14 +619,8 @@ void Host::run(boost::system::error_code const&) if (auto pp = p.second.lock()) pp->serviceNodesRequest(); - if (chrono::steady_clock::now() - m_lastPing > chrono::seconds(30)) // ping every 30s. - { - for (auto p: m_peers) - if (auto pp = p.second.lock()) - if (chrono::steady_clock::now() - pp->m_lastReceived > chrono::seconds(60)) - pp->disconnect(PingTimeout); - pingAll(); - } + if (chrono::steady_clock::now() - m_lastPing >= chrono::seconds(30)) // ping every 30s. + keepAlivePeers(); auto runcb = [this](boost::system::error_code const& error) -> void { run(error); }; m_timer->expires_from_now(boost::posix_time::milliseconds(c_timerInterval)); @@ -680,12 +674,18 @@ void Host::doWork() m_ioService.run(); } -void Host::pingAll() +void Host::keepAlivePeers() { RecursiveGuard l(x_peers); - for (auto& i: m_peers) - if (auto j = i.second.lock()) - j->ping(); + for (auto p: m_peers) + if (auto pp = p.second.lock()) + { + if (chrono::steady_clock::now() - pp->m_lastReceived >= chrono::seconds(60)) + pp->disconnect(PingTimeout); + else + pp->ping(); + } + m_lastPing = chrono::steady_clock::now(); } diff --git a/libp2p/Host.h b/libp2p/Host.h index edfa79df3..5892b5b0d 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -165,8 +165,8 @@ public: /// Get number of peers connected; equivalent to, but faster than, peers().size(). size_t peerCount() const { RecursiveGuard l(x_peers); return m_peers.size(); } - /// Ping the peers, to update the latency information. - void pingAll(); + /// Ping the peers to update the latency information and disconnect peers which have timed out. + void keepAlivePeers(); /// Get the port we're listening on currently. unsigned short listenPort() const { return m_tcpPublic.port(); } diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 53acc9dbe..1b74abbda 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -77,10 +77,11 @@ list NodeTable::state() const return move(ret); } -NodeEntry NodeTable::operator[](NodeId _id) +Node NodeTable::operator[](NodeId _id) { Guard l(x_nodes); - return *m_nodes[_id]; + auto n = m_nodes[_id]; + return !!n ? *n : Node(); } void NodeTable::requestNeighbours(NodeEntry const& _node, NodeId _target) const diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 89bd68be1..79ce6a9e3 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -36,16 +36,20 @@ namespace p2p */ struct NodeIPEndpoint { + NodeIPEndpoint(): udp(bi::udp::endpoint()), tcp(bi::tcp::endpoint()) {} NodeIPEndpoint(bi::udp::endpoint _udp): udp(_udp) {} NodeIPEndpoint(bi::tcp::endpoint _tcp): tcp(_tcp) {} NodeIPEndpoint(bi::udp::endpoint _udp, bi::tcp::endpoint _tcp): udp(_udp), tcp(_tcp) {} bi::udp::endpoint udp; bi::tcp::endpoint tcp; + + operator bool() const { return udp.address().is_unspecified() && tcp.address().is_unspecified(); } }; struct Node { + Node(): endpoint(NodeIPEndpoint()) {}; Node(Public _pubk, NodeIPEndpoint _ip, bool _required = false): id(_pubk), endpoint(_ip), required(_required) {} Node(Public _pubk, bi::udp::endpoint _udp, bool _required = false): Node(_pubk, NodeIPEndpoint(_udp), _required) {} @@ -59,6 +63,8 @@ struct Node /// If true, node will not be removed from Node list. bool required = false; + + operator bool() const { return (bool)id; } }; @@ -151,7 +157,7 @@ public: std::list nodes() const; std::list state() const; - NodeEntry operator[](NodeId _id); + Node operator[](NodeId _id); protected: struct NodeBucket diff --git a/test/peer.cpp b/test/peer.cpp index 70d03e9bf..d13ea97f3 100644 --- a/test/peer.cpp +++ b/test/peer.cpp @@ -55,7 +55,7 @@ int peerTest(int argc, char** argv) { this_thread::sleep_for(chrono::milliseconds(100)); if (!(i % 10)) - ph.pingAll(); + ph.keepAlivePeers(); } return 0; From 94c09508fd9d7495e91f57c333a56220a9060961 Mon Sep 17 00:00:00 2001 From: subtly Date: Sat, 10 Jan 2015 19:45:20 +0100 Subject: [PATCH 06/71] Merging in new data structure for nodes from node-table. End result will be consolidation into NodeId, Node (id and endpoints), NodeEntry (as in table), and Peer (connected node as in host). Rename PeerInfo to PeerSessionInfo. Rename NodeInfo to PeerInfo. PeerSessionInfo which is information about the Peer connection and will be split/merged into Node and PeerInfo. Add node-table callbacks for Host to perform connect node if there are not enough nodes. --- alethzero/MainWin.cpp | 2 +- libp2p/Common.h | 4 +- libp2p/Host.cpp | 342 ++++++++++++++++---------------------- libp2p/Host.h | 88 +++++----- libp2p/HostCapability.cpp | 4 +- libp2p/NodeTable.cpp | 55 +++--- libp2p/NodeTable.h | 44 ++++- libp2p/Session.cpp | 138 +++++++-------- libp2p/Session.h | 10 +- libwebthree/WebThree.cpp | 4 +- libwebthree/WebThree.h | 4 +- neth/main.cpp | 2 +- test/peer.cpp | 16 +- test/whisperTopic.cpp | 2 +- 14 files changed, 354 insertions(+), 361 deletions(-) diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index b73a2cd0d..40663db8d 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -906,7 +906,7 @@ void Main::refreshNetwork() if (web3()->haveNetwork()) { map clients; - for (PeerInfo const& i: ps) + for (PeerSessionInfo const& i: ps) ui->peers->addItem(QString("[%8 %7] %3 ms - %1:%2 - %4 %5 %6") .arg(QString::fromStdString(i.host)) .arg(i.port) diff --git a/libp2p/Common.h b/libp2p/Common.h index d46c5eed1..059e1a64e 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -116,7 +116,7 @@ typedef std::pair CapDesc; typedef std::set CapDescSet; typedef std::vector CapDescs; -struct PeerInfo +struct PeerSessionInfo { NodeId id; std::string clientVersion; @@ -128,7 +128,7 @@ struct PeerInfo std::map notes; }; -using PeerInfos = std::vector; +using PeerSessionInfos = std::vector; } } diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 5f756f2a5..243738918 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -38,6 +38,13 @@ using namespace std; using namespace dev; using namespace dev::p2p; +HostNodeTableHandler::HostNodeTableHandler(Host& _host): m_host(_host) {} + +void HostNodeTableHandler::processEvent(NodeId _n, NodeTableEventType _e) +{ + m_host.onNodeTableEvent(_n, _e); +} + Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n, bool _start): Worker("p2p", 0), m_clientVersion(_clientVersion), @@ -116,8 +123,8 @@ void Host::doneWorking() for (unsigned n = 0;; n = 0) { { - RecursiveGuard l(x_peers); - for (auto i: m_peers) + RecursiveGuard l(x_sessions); + for (auto i: m_sessions) if (auto p = i.second.lock()) if (p->isOpen()) { @@ -139,8 +146,8 @@ void Host::doneWorking() m_ioService.reset(); // finally, clear out peers (in case they're lingering) - RecursiveGuard l(x_peers); - m_peers.clear(); + RecursiveGuard l(x_sessions); + m_sessions.clear(); } unsigned Host::protocolVersion() const @@ -150,12 +157,12 @@ unsigned Host::protocolVersion() const void Host::registerPeer(std::shared_ptr _s, CapDescs const& _caps) { - assert(!!_s->m_node); - assert(!!_s->m_node->id); + assert(!!_s->m_peer); + assert(!!_s->m_peer->id); { - RecursiveGuard l(x_peers); - m_peers[_s->m_node->id] = _s; + RecursiveGuard l(x_sessions); + m_sessions[_s->m_peer->id] = _s; } unsigned o = (unsigned)UserPacket; for (auto const& i: _caps) @@ -166,6 +173,34 @@ void Host::registerPeer(std::shared_ptr _s, CapDescs const& _caps) } } +void Host::onNodeTableEvent(NodeId _n, NodeTableEventType _e) +{ + if (_e == NodeEntryAdded) + { + auto n = (*m_nodeTable)[_n]; + if (n) + { + RecursiveGuard l(x_sessions); + auto p = m_peers[_n]; + if (!p) + { + m_peers[_n] = make_shared(); + p = m_peers[_n]; + p->id = _n; + } + p->address = n.endpoint.tcp; + + if (peerCount() < m_idealPeerCount) + connect(p); + } + } + else if (_e == NodeEntryRemoved) + { + RecursiveGuard l(x_sessions); + m_peers.erase(_n); + } +} + void Host::seal(bytes& _b) { _b[0] = 0x22; @@ -179,80 +214,6 @@ void Host::seal(bytes& _b) _b[7] = len & 0xff; } -// TODO: P2P port to NodeTable. (see noteNode calls, Session.cpp) -//shared_ptr Host::noteNode(NodeId _id, bi::tcp::endpoint _a) -//{ -// RecursiveGuard l(x_peers); -// if (_a.port() < 30300 || _a.port() > 30305) -// cwarn << "Weird port being recorded: " << _a.port(); -// -// if (_a.port() >= /*49152*/32768) -// { -// cwarn << "Private port being recorded - setting to 0"; -// _a = bi::tcp::endpoint(_a.address(), 0); -// } -// -// unsigned i; -// if (!m_nodes.count(_id)) -// { -// i = m_nodesList.size(); -// m_nodesList.push_back(_id); -// m_nodes[_id] = make_shared(); -// m_nodes[_id]->id = _id; -// m_nodes[_id]->index = i; -// } -// else -// i = m_nodes[_id]->index; -// m_nodes[_id]->address = _a; -// m_private.extendAll(i); -// if (!_a.port() || (isPrivateAddress(_a.address()) && !m_netPrefs.localNetworking)) -// m_private += i; -// else -// m_private -= i; -// -// return m_nodes[_id]; -//} - -// TODO: P2P base on target -// TODO: P2P store caps in NodeTable/NodeEntry -//Nodes Host::potentialPeers(RangeMask const& _known) -//{ -// RecursiveGuard l(x_peers); -// Nodes ret; -// -// // todo: if localnetworking is enabled it should only share peers if remote -// // is within the same network as our interfaces. -// // this requires flagging nodes when we receive them as to if they're on private network -// auto ns = (m_netPrefs.localNetworking ? _known : (m_private + _known)).inverted(); -// for (auto i: ns) -// ret.push_back(*m_nodes[m_nodesList[i]]); -// return ret; -//} - -KeyPair Host::getHostIdentifier() -{ - static string s_file(getDataDir() + "/host"); - static mutex s_x; - lock_guard l(s_x); - - h256 secret; - bytes b = contents(s_file); - if (b.size() == 32) - memcpy(secret.data(), b.data(), 32); - else - { - // todo: replace w/user entropy; abstract to devcrypto - std::mt19937_64 s_eng(time(0) + chrono::high_resolution_clock::now().time_since_epoch().count()); - std::uniform_int_distribution d(0, 255); - for (unsigned i = 0; i < 32; ++i) - secret[i] = (byte)d(s_eng); - } - - if (!secret) - BOOST_THROW_EXCEPTION(crypto::InvalidState()); - return move(KeyPair(move(secret))); -} - void Host::determinePublic(string const& _publicAddress, bool _upnp) { m_peerAddresses.clear(); @@ -324,15 +285,16 @@ void Host::runAcceptor() { clog(NetConnect) << "Listening on local port " << m_listenPort << " (public: " << m_tcpPublic << ")"; m_accepting = true; - m_socket.reset(new bi::tcp::socket(m_ioService)); - m_tcp4Acceptor.async_accept(*m_socket, [=](boost::system::error_code ec) + + bi::tcp::socket* s = new bi::tcp::socket(m_ioService); + m_tcp4Acceptor.async_accept(*s, [=](boost::system::error_code ec) { bool success = false; if (!ec) { try { - doHandshake(m_socket.release()); + doHandshake(s); success = true; } catch (Exception const& _e) @@ -345,27 +307,29 @@ void Host::runAcceptor() } } - if (!success && m_socket->is_open()) + if (!success && s->is_open()) { boost::system::error_code ec; - m_socket->shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec); - m_socket->close(); + s->shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec); + s->close(); } m_accepting = false; + delete s; + if (ec.value() < 1) runAcceptor(); }); } } -void Host::doHandshake(bi::tcp::socket* _socket, NodeId _egressNodeId) +void Host::doHandshake(bi::tcp::socket* _socket, NodeId _nodeId) { try { clog(NetConnect) << "Accepting connection for " << _socket->remote_endpoint(); } catch (...){} - auto p = std::make_shared(this, std::move(*_socket), m_nodes[_egressNodeId]); + auto p = std::make_shared(this, std::move(*_socket), m_peers[_nodeId]); p->start(); } @@ -378,6 +342,15 @@ string Host::pocHost() void Host::addNode(NodeId const& _node, std::string const& _addr, unsigned short _tcpPeerPort, unsigned short _udpNodePort) { + if (_tcpPeerPort < 30300 || _tcpPeerPort > 30305) + cwarn << "Weird port being recorded: " << _tcpPeerPort; + + if (_tcpPeerPort >= /*49152*/32768) + { + cwarn << "Private port being recorded - setting to 0"; + _tcpPeerPort = 0; + } + boost::system::error_code ec; bi::address addr = bi::address::from_string(_addr, ec); if (ec) @@ -394,59 +367,25 @@ void Host::addNode(NodeId const& _node, std::string const& _addr, unsigned short addNode(Node(_node, NodeIPEndpoint(bi::udp::endpoint(addr, _udpNodePort), bi::tcp::endpoint(addr, _tcpPeerPort)))); } -void Host::connect(NodeId const& _node, std::string const& _addr, unsigned short _peerPort, unsigned short _nodePort) noexcept +void Host::connect(std::shared_ptr const& _n) { if (!m_run) return; - assert(_node); - - auto n = (*m_nodeTable)[_node]; + if (havePeerSession(_n->id)) + { + clog(NetWarn) << "Aborted connect. Node already connected."; + return; + } - // TODO: refactor into async_resolve - m_ioService.post([=]() + if (!m_nodeTable->haveNode(_n->id)) { - for (auto first: {true, false}) - { - try - { - bi::tcp::endpoint ep; - if (first) - { - bi::tcp::resolver r(m_ioService); - ep = r.resolve({_addr, toString(_peerPort)})->endpoint(); - } - else - ep = bi::tcp::endpoint(bi::address::from_string(_addr), _peerPort); - - if (!n) - m_nodes[_node] = make_shared(); - m_nodes[_node]->id = _node; - m_nodes[_node]->address = ep; - connect(m_nodes[_node]); - break; - } - catch (Exception const& _e) - { - // Couldn't connect - clog(NetConnect) << "Bad host " << _addr << "\n" << diagnostic_information(_e); - } - catch (exception const& e) - { - // Couldn't connect - clog(NetConnect) << "Bad host " << _addr << " (" << e.what() << ")"; - } - } - }); -} - -void Host::connect(std::shared_ptr const& _n) -{ - if (!m_run) + clog(NetWarn) << "Aborted connect. Node not in node table."; return; + } // prevent concurrently connecting to a node - NodeInfo *nptr = _n.get(); + PeerInfo *nptr = _n.get(); { Guard l(x_pendingNodeConns); if (m_pendingNodeConns.count(nptr)) @@ -457,48 +396,32 @@ void Host::connect(std::shared_ptr const& _n) clog(NetConnect) << "Attempting connection to node" << _n->id.abridged() << "@" << _n->address << "from" << id().abridged(); _n->lastAttempted = std::chrono::system_clock::now(); _n->failedAttempts++; + bi::tcp::socket* s = new bi::tcp::socket(m_ioService); - - auto n = node(_n->id); - if (n) - s->async_connect(_n->address, [=](boost::system::error_code const& ec) + s->async_connect(_n->address, [=](boost::system::error_code const& ec) + { + if (ec) { - if (ec) - { - clog(NetConnect) << "Connection refused to node" << _n->id.abridged() << "@" << _n->address << "(" << ec.message() << ")"; - _n->lastDisconnect = TCPError; - _n->lastAttempted = std::chrono::system_clock::now(); - } - else - { - clog(NetConnect) << "Connected to" << _n->id.abridged() << "@" << _n->address; - _n->lastConnected = std::chrono::system_clock::now(); - auto p = make_shared(this, std::move(*s), n); - p->start(); - } - delete s; - Guard l(x_pendingNodeConns); - m_pendingNodeConns.erase(nptr); - }); - else - clog(NetWarn) << "Aborted connect. Node not in node table."; -} - -bool Host::havePeer(NodeId _id) const -{ - RecursiveGuard l(x_peers); - - // Remove dead peers from list. - for (auto i = m_peers.begin(); i != m_peers.end();) - if (i->second.lock().get()) - ++i; + clog(NetConnect) << "Connection refused to node" << _n->id.abridged() << "@" << _n->address << "(" << ec.message() << ")"; + _n->lastDisconnect = TCPError; + _n->lastAttempted = std::chrono::system_clock::now(); + } else - i = m_peers.erase(i); - - return !!m_peers.count(_id); + { + clog(NetConnect) << "Connected to" << _n->id.abridged() << "@" << _n->address; + + _n->lastConnected = std::chrono::system_clock::now(); + auto p = make_shared(this, std::move(*s), _n); + p->start(); + + } + delete s; + Guard l(x_pendingNodeConns); + m_pendingNodeConns.erase(nptr); + }); } -unsigned NodeInfo::fallbackSeconds() const +unsigned PeerInfo::fallbackSeconds() const { switch (lastDisconnect) { @@ -524,27 +447,27 @@ unsigned NodeInfo::fallbackSeconds() const // TODO: P2P migrate grow/prunePeers into 'maintainPeers' & evaluate reputation instead of availability. schedule via deadline timer. //void Host::growPeers() //{ -// RecursiveGuard l(x_peers); -// int morePeers = (int)m_idealPeerCount - m_peers.size(); +// RecursiveGuard l(x_sessions); +// int morePeers = (int)m_idealPeerCount - m_sessions.size(); // if (morePeers > 0) // { // auto toTry = m_ready; // if (!m_netPrefs.localNetworking) // toTry -= m_private; -// set ns; +// set ns; // for (auto i: toTry) // if (m_nodes[m_nodesList[i]]->shouldReconnect()) // ns.insert(*m_nodes[m_nodesList[i]]); // // if (ns.size()) -// for (NodeInfo const& i: ns) +// for (PeerInfo const& i: ns) // { // connect(m_nodes[i.id]); // if (!--morePeers) // return; // } // else -// for (auto const& i: m_peers) +// for (auto const& i: m_sessions) // if (auto p = i.second.lock()) // p->ensureNodesRequested(); // } @@ -552,17 +475,17 @@ unsigned NodeInfo::fallbackSeconds() const // //void Host::prunePeers() //{ -// RecursiveGuard l(x_peers); +// RecursiveGuard l(x_sessions); // // We'll keep at most twice as many as is ideal, halfing what counts as "too young to kill" until we get there. // set dc; -// for (unsigned old = 15000; m_peers.size() - dc.size() > m_idealPeerCount * 2 && old > 100; old /= 2) -// if (m_peers.size() - dc.size() > m_idealPeerCount) +// for (unsigned old = 15000; m_sessions.size() - dc.size() > m_idealPeerCount * 2 && old > 100; old /= 2) +// if (m_sessions.size() - dc.size() > m_idealPeerCount) // { // // look for worst peer to kick off // // first work out how many are old enough to kick off. // shared_ptr worst; // unsigned agedPeers = 0; -// for (auto i: m_peers) +// for (auto i: m_sessions) // if (!dc.count(i.first)) // if (auto p = i.second.lock()) // if (/*(m_mode != NodeMode::Host || p->m_caps != 0x01) &&*/ chrono::steady_clock::now() > p->m_connect + chrono::milliseconds(old)) // don't throw off new peers; peer-servers should never kick off other peer-servers. @@ -578,21 +501,21 @@ unsigned NodeInfo::fallbackSeconds() const // } // // // Remove dead peers from list. -// for (auto i = m_peers.begin(); i != m_peers.end();) +// for (auto i = m_sessions.begin(); i != m_sessions.end();) // if (i->second.lock().get()) // ++i; // else -// i = m_peers.erase(i); +// i = m_sessions.erase(i); //} -PeerInfos Host::peers() const +PeerSessionInfos Host::peers() const { if (!m_run) - return PeerInfos(); + return PeerSessionInfos(); - std::vector ret; - RecursiveGuard l(x_peers); - for (auto& i: m_peers) + std::vector ret; + RecursiveGuard l(x_sessions); + for (auto& i: m_sessions) if (auto j = i.second.lock()) if (j->m_socket.is_open()) ret.push_back(j->m_info); @@ -615,7 +538,7 @@ void Host::run(boost::system::error_code const&) return; } - for (auto p: m_peers) + for (auto p: m_sessions) if (auto pp = p.second.lock()) pp->serviceNodesRequest(); @@ -658,7 +581,8 @@ void Host::startedWorking() runAcceptor(); if (!m_tcpPublic.address().is_unspecified()) - m_nodeTable.reset(new NodeTable(m_ioService, m_key, m_listenPort, m_tcpPublic)); + // TODO: add m_tcpPublic endpoint; sort out endpoint stuff for nodetable + m_nodeTable.reset(new NodeTable(m_ioService, m_key, m_listenPort)); else m_nodeTable.reset(new NodeTable(m_ioService, m_key, m_listenPort > 0 ? m_listenPort : 30303)); } @@ -676,8 +600,8 @@ void Host::doWork() void Host::keepAlivePeers() { - RecursiveGuard l(x_peers); - for (auto p: m_peers) + RecursiveGuard l(x_sessions); + for (auto p: m_sessions) if (auto pp = p.second.lock()) { if (chrono::steady_clock::now() - pp->m_lastReceived >= chrono::seconds(60)) @@ -694,10 +618,10 @@ bytes Host::saveNodes() const RLPStream nodes; int count = 0; { - RecursiveGuard l(x_peers); - for (auto const& i: m_nodes) + RecursiveGuard l(x_sessions); + for (auto const& i: m_peers) { - NodeInfo const& n = *(i.second); + PeerInfo const& n = *(i.second); // TODO: PoC-7: Figure out why it ever shares these ports.//n.address.port() >= 30300 && n.address.port() <= 30305 && if (!n.dead && chrono::system_clock::now() - n.lastConnected < chrono::seconds(3600 * 48) && n.address.port() > 0 && n.address.port() < /*49152*/32768 && n.id != id() && !isPrivateAddress(n.address.address())) { @@ -722,7 +646,7 @@ bytes Host::saveNodes() const void Host::restoreNodes(bytesConstRef _b) { - RecursiveGuard l(x_peers); + RecursiveGuard l(x_sessions); RLP r(_b); if (r.itemCount() > 0 && r[0].isInt()) switch (r[0].toInt()) @@ -740,7 +664,7 @@ void Host::restoreNodes(bytesConstRef _b) else ep = bi::tcp::endpoint(bi::address_v6(i[0].toArray()), i[1].toInt()); auto id = (NodeId)i[2]; - if (!m_nodes.count(id)) + if (!m_peers.count(id)) { //// auto o = (Origin)i[3].toInt(); // auto n = noteNode(id, ep); @@ -759,7 +683,7 @@ void Host::restoreNodes(bytesConstRef _b) for (auto i: r) { auto id = (NodeId)i[2]; - if (!m_nodes.count(id)) + if (!m_peers.count(id)) { bi::tcp::endpoint ep; if (i[0].itemCount() == 4) @@ -770,3 +694,27 @@ void Host::restoreNodes(bytesConstRef _b) } } } + +KeyPair Host::getHostIdentifier() +{ + static string s_file(getDataDir() + "/host"); + static mutex s_x; + lock_guard l(s_x); + + h256 secret; + bytes b = contents(s_file); + if (b.size() == 32) + memcpy(secret.data(), b.data(), 32); + else + { + // todo: replace w/user entropy; abstract to devcrypto + std::mt19937_64 s_eng(time(0) + chrono::high_resolution_clock::now().time_since_epoch().count()); + std::uniform_int_distribution d(0, 255); + for (unsigned i = 0; i < 32; ++i) + secret[i] = (byte)d(s_eng); + } + + if (!secret) + BOOST_THROW_EXCEPTION(crypto::InvalidState()); + return move(KeyPair(move(secret))); +} diff --git a/libp2p/Host.h b/libp2p/Host.h index 5892b5b0d..097b07433 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -51,7 +51,7 @@ namespace p2p class Host; -struct NodeInfo +struct PeerInfo { NodeId id; ///< Their id/public key. unsigned index; ///< Index into m_nodesList @@ -88,7 +88,7 @@ struct NodeInfo bool isOffline() const { return lastAttempted > lastConnected; } // p2p: Remove (in favor of lru eviction and sub-protocol ratings). - bool operator<(NodeInfo const& _n) const + bool operator<(PeerInfo const& _n) const { if (isOffline() != _n.isOffline()) return isOffline(); @@ -108,19 +108,30 @@ struct NodeInfo } }; -using Nodes = std::vector; +using Nodes = std::vector; + +class Host; +class HostNodeTableHandler: public NodeTableEventHandler +{ + HostNodeTableHandler(Host& _host); + virtual void processEvent(NodeId _n, NodeTableEventType _e); + Host& m_host; +}; + /** * @brief The Host class * Capabilities should be registered prior to startNetwork, since m_capabilities is not thread-safe. + * @todo gracefully disconnect peer if peer already connected * @todo determinePublic: ipv6, udp * @todo handle conflict if addNode/requireNode called and Node already exists w/conflicting tcp or udp port */ class Host: public Worker { + friend class HostNodeTableHandler; friend class Session; friend class HostCapabilityFace; - friend struct NodeInfo; + friend struct PeerInfo; public: /// Start server, listening for connections on the given port. @@ -142,31 +153,19 @@ public: CapDescs caps() const { CapDescs ret; for (auto const& i: m_capabilities) ret.push_back(i.first); return ret; } template std::shared_ptr cap() const { try { return std::static_pointer_cast(m_capabilities.at(std::make_pair(T::staticName(), T::staticVersion()))); } catch (...) { return nullptr; } } - /// Manually add node. - void addNode(NodeId const& _node, std::string const& _addr, unsigned short _tcpPort = 30303, unsigned short _udpPort = 30303); - - /// Connect to a peer explicitly. - void connect(NodeId const& _node, std::string const& _addr, unsigned short _tcpPort = 30303, unsigned short _udpPort = 30303) noexcept; - void connect(NodeId const& _node, bi::tcp::endpoint const& _ep); - void connect(std::shared_ptr const& _n); - - /// @returns true iff we have a peer of the given id. - bool havePeer(NodeId _id) const; - + bool havePeerSession(NodeId _id) { RecursiveGuard l(x_sessions); if (m_sessions.count(_id)) return !!m_sessions[_id].lock(); else return false; } + + /// Add node. + void addNode(NodeId const& _node, std::string const& _addr, unsigned short _tcpPort, unsigned short _udpPort); + /// Set ideal number of peers. void setIdealPeerCount(unsigned _n) { m_idealPeerCount = _n; } - - /// p2p: template? - void setIdealPeerCount(HostCapabilityFace* _cap, unsigned _n) { m_capIdealPeerCount[_cap->capDesc()] = _n; } /// Get peer information. - PeerInfos peers() const; + PeerSessionInfos peers() const; /// Get number of peers connected; equivalent to, but faster than, peers().size(). - size_t peerCount() const { RecursiveGuard l(x_peers); return m_peers.size(); } - - /// Ping the peers to update the latency information and disconnect peers which have timed out. - void keepAlivePeers(); + size_t peerCount() const { RecursiveGuard l(x_sessions); return m_peers.size(); } /// Get the port we're listening on currently. unsigned short listenPort() const { return m_tcpPublic.port(); } @@ -177,7 +176,8 @@ public: /// Deserialise the data and populate the set of known peers. void restoreNodes(bytesConstRef _b); - Nodes nodes() const { RecursiveGuard l(x_peers); Nodes ret; for (auto const& i: m_nodes) ret.push_back(*i.second); return ret; } + // TODO: P2P this should be combined with peers into a HostStat object of some kind; coalesce data, as it's only used for status information. + Nodes nodes() const { RecursiveGuard l(x_sessions); Nodes ret; for (auto const& i: m_peers) ret.push_back(*i.second); return ret; } void setNetworkPreferences(NetworkPreferences const& _p) { auto had = isStarted(); if (had) stop(); m_netPrefs = _p; if (had) start(); } @@ -195,14 +195,20 @@ public: void registerPeer(std::shared_ptr _s, CapDescs const& _caps); - std::shared_ptr node(NodeId _id) const { if (m_nodes.count(_id)) return m_nodes.at(_id); return std::shared_ptr(); } +// std::shared_ptr node(NodeId _id) const { if (m_nodes.count(_id)) return m_nodes.at(_id); return std::shared_ptr(); } + +protected: + void onNodeTableEvent(NodeId _n, NodeTableEventType _e); private: - KeyPair getHostIdentifier(); - /// Populate m_peerAddresses with available public addresses. void determinePublic(std::string const& _publicAddress, bool _upnp); + void connect(std::shared_ptr const& _n); + + /// Ping the peers to update the latency information and disconnect peers which have timed out. + void keepAlivePeers(); + /// Called only from startedWorking(). void runAcceptor(); @@ -223,9 +229,10 @@ private: virtual void doneWorking(); /// Add node - void addNode(Node const& _nodeInfo) { m_nodeTable->addNode(_nodeInfo); } + void addNode(Node const& _node) { m_nodeTable->addNode(_node); } -// Nodes potentialPeers(RangeMask const& _known); + /// Get or create host identifier (KeyPair). + KeyPair getHostIdentifier(); bool m_run = false; ///< Whether network is running. std::mutex x_runTimer; ///< Start/stop mutex. @@ -241,33 +248,24 @@ private: ba::io_service m_ioService; ///< IOService for network stuff. bi::tcp::acceptor m_tcp4Acceptor; ///< Listening acceptor. - std::unique_ptr m_socket; ///< Listening socket. std::unique_ptr m_timer; ///< Timer which, when network is running, calls scheduler() every c_timerInterval ms. static const unsigned c_timerInterval = 100; ///< Interval which m_timer is run when network is connected. - std::set m_pendingNodeConns; /// Used only by connect(NodeInfo&) to limit concurrently connecting to same node. See connect(shared_ptrconst&). + std::set m_pendingNodeConns; /// Used only by connect(PeerInfo&) to limit concurrently connecting to same node. See connect(shared_ptrconst&). Mutex x_pendingNodeConns; bi::tcp::endpoint m_tcpPublic; ///< Our public listening endpoint. KeyPair m_key; ///< Our unique ID. std::shared_ptr m_nodeTable; ///< Node table (uses kademlia-like discovery). - std::map m_capIdealPeerCount; ///< Ideal peer count for capability. - - mutable RecursiveMutex x_peers; + std::map> m_peers; + + mutable RecursiveMutex x_sessions; + /// The nodes to which we are currently connected. /// Mutable because we flush zombie entries (null-weakptrs) as regular maintenance from a const method. - mutable std::map> m_peers; - - /// Nodes to which we may connect (or to which we have connected). - /// TODO: mutex; replace with nodeTable - std::map > m_nodes; - -// /// A list of node IDs. This contains every index from m_nodes; the order is guaranteed to remain the same. -// std::vector m_nodesList; - -// RangeMask m_private; ///< Indices into m_nodesList over to which nodes are private. + mutable std::map> m_sessions; unsigned m_idealPeerCount = 5; ///< Ideal number of peers to be connected to. @@ -280,6 +278,6 @@ private: bool m_accepting = false; }; - + } } diff --git a/libp2p/HostCapability.cpp b/libp2p/HostCapability.cpp index 0728bef2c..8ff74d3b6 100644 --- a/libp2p/HostCapability.cpp +++ b/libp2p/HostCapability.cpp @@ -34,9 +34,9 @@ void HostCapabilityFace::seal(bytes& _b) std::vector > HostCapabilityFace::peers() const { - RecursiveGuard l(m_host->x_peers); + RecursiveGuard l(m_host->x_sessions); std::vector > ret; - for (auto const& i: m_host->m_peers) + for (auto const& i: m_host->m_sessions) if (std::shared_ptr p = i.second.lock()) if (p->m_capabilities.count(capDesc())) ret.push_back(p); diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 1b74abbda..b384c42d2 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -27,7 +27,7 @@ using namespace dev::p2p; NodeEntry::NodeEntry(Node _src, Public _pubk, NodeIPEndpoint _gw): Node(_pubk, _gw), distance(NodeTable::dist(_src.id,_pubk)) {} NodeEntry::NodeEntry(Node _src, Public _pubk, bi::udp::endpoint _udp): Node(_pubk, NodeIPEndpoint(_udp)), distance(NodeTable::dist(_src.id,_pubk)) {} -NodeTable::NodeTable(ba::io_service& _io, KeyPair _alias, uint16_t _udp, bi::tcp::endpoint _ep): +NodeTable::NodeTable(ba::io_service& _io, KeyPair _alias, uint16_t _udp): m_node(Node(_alias.pub(), bi::udp::endpoint())), m_secret(_alias.sec()), m_io(_io), @@ -53,11 +53,32 @@ NodeTable::~NodeTable() m_socketPtr->disconnect(); } +shared_ptr NodeTable::addNode(Public const& _pubk, bi::udp::endpoint const& _udp, bi::tcp::endpoint const& _tcp) +{ + auto node = Node(_pubk, NodeIPEndpoint(_udp, _tcp)); + return move(addNode(node)); +} + +shared_ptr NodeTable::addNode(Node const& _node) +{ + Guard l(x_nodes); + shared_ptr ret = m_nodes[_node.id]; + if (!ret) + { + ret.reset(new NodeEntry(m_node, _node.id, NodeIPEndpoint(_node.endpoint.udp, _node.endpoint.tcp))); + m_nodes[_node.id] = ret; + PingNode p(_node.endpoint.udp, m_node.endpoint.udp.address().to_string(), m_node.endpoint.udp.port()); + p.sign(m_secret); + m_socketPtr->send(p); + } + return move(ret); +} + void NodeTable::join() { doFindNode(m_node.id); } - + list NodeTable::nodes() const { list nodes; @@ -84,6 +105,13 @@ Node NodeTable::operator[](NodeId _id) return !!n ? *n : Node(); } +shared_ptr NodeTable::getNodeEntry(NodeId _id) +{ + Guard l(x_nodes); + auto n = m_nodes[_id]; + return !!n ? move(n) : move(shared_ptr()); +} + void NodeTable::requestNeighbours(NodeEntry const& _node, NodeId _target) const { FindNode p(_node.endpoint.udp, _target); @@ -240,29 +268,6 @@ void NodeTable::evict(shared_ptr _leastSeen, shared_ptr _n ping(_leastSeen.get()); } -shared_ptr NodeTable::addNode(Public const& _pubk, bi::udp::endpoint const& _udp, bi::tcp::endpoint const& _tcp) -{ - auto node = Node(_pubk, NodeIPEndpoint(_udp, _tcp)); - return move(addNode(node)); -} - -shared_ptr NodeTable::addNode(Node const& _node) -{ - shared_ptr ret; - Guard l(x_nodes); - if (auto n = m_nodes[_node.id]) - ret = n; - else - { - ret.reset(new NodeEntry(m_node, _node.id, NodeIPEndpoint(_node.endpoint.udp, _node.endpoint.tcp))); - m_nodes[_node.id] = ret; - PingNode p(_node.endpoint.udp, m_node.endpoint.udp.address().to_string(), m_node.endpoint.udp.port()); - p.sign(m_secret); - m_socketPtr->send(p); - } - return move(ret); -} - void NodeTable::noteNode(Public const& _pubk, bi::udp::endpoint const& _endpoint) { if (_pubk == m_node.address()) diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 79ce6a9e3..7f75bf27e 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -22,6 +22,7 @@ #pragma once #include +#include #include #include #include @@ -74,14 +75,37 @@ struct Node */ struct NodeEntry: public Node { - NodeEntry(Node _src, Public _pubk, NodeIPEndpoint _gw); //: Node(_pubk, _gw), distance(dist(_src.id,_pubk)) {} - NodeEntry(Node _src, Public _pubk, bi::udp::endpoint _udp); //: Node(_pubk, NodeIPEndpoint(_udp)), distance(dist(_src.id,_pubk)) {} + NodeEntry(Node _src, Public _pubk, NodeIPEndpoint _gw); + NodeEntry(Node _src, Public _pubk, bi::udp::endpoint _udp); - const unsigned distance; ///< Node's distance from _src (see constructor). + const unsigned distance; ///< Node's distance (xor of _src as integer). +}; + +enum NodeTableEventType { + NodeEntryAdded, + NodeEntryRemoved +}; +class NodeTable; +class NodeTableEventHandler +{ + friend class NodeTable; +public: + virtual void processEvent(NodeId _n, NodeTableEventType _e) =0; + +protected: + /// Called by NodeTable on behalf of an implementation (Host) to process new events without blocking nodetable. + void processEvents() { std::list> events; { Guard l(x_events); if (!m_nodeEvents.size()) return; m_nodeEvents.unique(); for (auto const& n: m_nodeEvents) events.push_back(std::make_pair(n,m_events[n])); m_nodeEvents.empty(); m_events.empty(); } for (auto const& e: events) processEvent(e.first, e.second); } + + /// Called by NodeTable to append event. + virtual void appendEvent(NodeId _n, NodeTableEventType _e) { Guard l(x_events); m_nodeEvents.push_back(_n); m_events[_n] = _e; } + + Mutex x_events; + std::list m_nodeEvents; + std::map m_events; }; /** - * NodeTable using S/Kademlia system for node discovery and preference. + * NodeTable using modified kademlia for node discovery and preference. * untouched buckets are refreshed if they have not been touched within an hour * * Thread-safety is ensured by modifying NodeEntry details via @@ -122,7 +146,7 @@ class NodeTable: UDPSocketEvents, public std::enable_shared_from_this using EvictionTimeout = std::pair,NodeId>; ///< First NodeId may be evicted and replaced with second NodeId. public: - NodeTable(ba::io_service& _io, KeyPair _alias, uint16_t _udpPort = 30303, bi::tcp::endpoint _ep = bi::tcp::endpoint()); + NodeTable(ba::io_service& _io, KeyPair _alias, uint16_t _udpPort = 30303); ~NodeTable(); /// Constants for Kademlia, mostly derived from address space. @@ -145,6 +169,12 @@ public: static unsigned dist(NodeId const& _a, NodeId const& _b) { u512 d = _a ^ _b; unsigned ret; for (ret = 0; d >>= 1; ++ret) {}; return ret; } + /// Set event handler for NodeEntryAdded and NodeEntryRemoved events. + void setEventHandler(NodeTableEventHandler* _handler) { m_nodeEvents.reset(_handler); } + + /// Called by implementation which provided handler to process NodeEntryAdded/NodeEntryRemoved events. Events are coalesced by type whereby old events are ignored. + void processEvents() { if (m_nodeEvents) m_nodeEvents->processEvents(); } + /// Add node. Node will be pinged if it's not already known. std::shared_ptr addNode(Public const& _pubk, bi::udp::endpoint const& _udp, bi::tcp::endpoint const& _tcp = bi::tcp::endpoint()); @@ -157,7 +187,9 @@ public: std::list nodes() const; std::list state() const; + bool haveNode(NodeId _id) { Guard l(x_nodes); return !!m_nodes[_id]; } Node operator[](NodeId _id); + std::shared_ptr getNodeEntry(NodeId _id); protected: struct NodeBucket @@ -207,6 +239,8 @@ protected: /// Sends FindNeighbor packet. See doFindNode. void requestNeighbours(NodeEntry const& _node, NodeId _target) const; + std::unique_ptr m_nodeEvents; ///< Event handler for node events. + Node m_node; ///< This node. Secret m_secret; ///< This nodes secret key. diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index c6797b69c..0ef06ab50 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -36,21 +36,21 @@ using namespace dev::p2p; #endif #define clogS(X) dev::LogOutputStream(false) << "| " << std::setw(2) << m_socket.native_handle() << "] " -Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr const& _n): +Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr const& _n): m_server(_s), m_socket(std::move(_socket)), - m_node(_n), + m_info({m_peer->id, "?", _n->address.address().to_string(), _n->address.port(), std::chrono::steady_clock::duration(0), CapDescSet(), 0, map()}), + m_peer(_n), m_manualEndpoint(_n->address) { m_lastReceived = m_connect = std::chrono::steady_clock::now(); - m_info = PeerInfo({m_node->id, "?", _n->address.address().to_string(), _n->address.port(), std::chrono::steady_clock::duration(0), CapDescSet(), 0, map()}); } Session::~Session() { // TODO: P2P revisit (refactored from previous logic) - if (m_node && !(id() && !isPermanentProblem(m_node->lastDisconnect) && !m_node->dead)) - m_node->lastConnected = m_node->lastAttempted - chrono::seconds(1); + if (m_peer && !(id() && !isPermanentProblem(m_peer->lastDisconnect) && !m_peer->dead)) + m_peer->lastConnected = m_peer->lastAttempted - chrono::seconds(1); // Read-chain finished for one reason or another. for (auto& i: m_capabilities) @@ -70,35 +70,35 @@ Session::~Session() NodeId Session::id() const { - return m_node ? m_node->id : NodeId(); + return m_peer ? m_peer->id : NodeId(); } void Session::addRating(unsigned _r) { - if (m_node) + if (m_peer) { - m_node->rating += _r; - m_node->score += _r; + m_peer->rating += _r; + m_peer->score += _r; } } int Session::rating() const { - return m_node->rating; + return m_peer->rating; } // TODO: P2P integration: session->? should be unavailable when socket isn't open bi::tcp::endpoint Session::endpoint() const { - if (m_socket.is_open() && m_node) + if (m_socket.is_open() && m_peer) try { - return bi::tcp::endpoint(m_socket.remote_endpoint().address(), m_node->address.port()); + return bi::tcp::endpoint(m_socket.remote_endpoint().address(), m_peer->address.port()); } catch (...) {} - if (m_node) - return m_node->address; + if (m_peer) + return m_peer->address; return m_manualEndpoint; } @@ -195,10 +195,11 @@ bool Session::interpret(RLP const& _r) return true; } - assert(!!m_node); - assert(!!m_node->id); + assert(!!m_peer); + assert(!!m_peer->id); - if (m_server->havePeer(id)) + // TODO: P2P ensure disabled logic is covered + if (false /* m_server->havePeer(id) */) { // Already connected. clogS(NetWarn) << "Already connected to a peer with id" << id.abridged(); @@ -217,20 +218,28 @@ bool Session::interpret(RLP const& _r) // TODO: P2P Move all node-lifecycle information into Host. Determine best way to handle peer-lifecycle properties vs node lifecycle. // TODO: P2P remove oldid // TODO: P2P with encrypted transport the handshake will fail and we won't get here -// m_node = m_server->noteNode(m_node->id, bi::tcp::endpoint(m_socket.remote_endpoint().address(), listenPort)); - if (m_node->isOffline()) - m_node->lastConnected = chrono::system_clock::now(); +// m_peer = m_server->noteNode(m_peer->id, bi::tcp::endpoint(m_socket.remote_endpoint().address(), listenPort)); + if (m_peer->isOffline()) + m_peer->lastConnected = chrono::system_clock::now(); // // // TODO: P2P introduce map of nodes we've given to this node (if GetPeers/Peers stays in TCP) - m_knownNodes.extendAll(m_node->index); - m_knownNodes.unionWith(m_node->index); + m_knownNodes.extendAll(m_peer->index); + m_knownNodes.unionWith(m_peer->index); if (m_protocolVersion != m_server->protocolVersion()) { disconnect(IncompatibleProtocol); return true; } - m_info = PeerInfo({id, clientVersion, m_socket.remote_endpoint().address().to_string(), listenPort, std::chrono::steady_clock::duration(), _r[3].toSet(), (unsigned)m_socket.native_handle(), map() }); + + // TODO: P2P migrate auth to Host and Handshake to constructor + m_info.clientVersion = clientVersion; + m_info.host = m_socket.remote_endpoint().address().to_string(); + m_info.port = listenPort; + m_info.lastPing = std::chrono::steady_clock::duration(); + m_info.caps = _r[3].toSet(); + m_info.socket = (unsigned)m_socket.native_handle(); + m_info.notes = map(); m_server->registerPeer(shared_from_this(), caps); break; @@ -285,64 +294,63 @@ bool Session::interpret(RLP const& _r) } auto ep = bi::tcp::endpoint(peerAddress, _r[i][1].toInt()); NodeId id = _r[i][2].toHash(); - clogS(NetAllDetail) << "Checking: " << ep << "(" << id.abridged() << ")" << isPrivateAddress(peerAddress) << this->id().abridged() << isPrivateAddress(endpoint().address()) << m_server->m_nodes.count(id) << (m_server->m_nodes.count(id) ? isPrivateAddress(m_server->m_nodes.at(id)->address.address()) : -1); + + clogS(NetAllDetail) << "Checking: " << ep << "(" << id.abridged() << ")"; +// clogS(NetAllDetail) << "Checking: " << ep << "(" << id.abridged() << ")" << isPrivateAddress(peerAddress) << this->id().abridged() << isPrivateAddress(endpoint().address()) << m_server->m_peers.count(id) << (m_server->m_peers.count(id) ? isPrivateAddress(m_server->m_peers.at(id)->address.address()) : -1); - if (isPrivateAddress(peerAddress) && !m_server->m_netPrefs.localNetworking) + // ignore if dist(us,item) - dist(us,them) > 1 + + // TODO: isPrivate + if (!m_server->m_netPrefs.localNetworking && isPrivateAddress(peerAddress)) goto CONTINUE; // Private address. Ignore. if (!id) - goto CONTINUE; // Null identity. Ignore. + goto LAMEPEER; // Null identity. Ignore. if (m_server->id() == id) - goto CONTINUE; // Just our info - we already have that. + goto LAMEPEER; // Just our info - we already have that. if (id == this->id()) - goto CONTINUE; // Just their info - we already have that. + goto LAMEPEER; // Just their info - we already have that. + // we don't worry about m_peers.count(id) now because node table will handle this and + // by default we will not blindly connect to nodes received via tcp; instead they will + // be pinged, as-is standard, by the node table and added if appropriate. unless flagged + // as required, nodes aren't connected to unless they respond via discovery; no matter if + // a node is relayed via udp or tcp. // check that it's not us or one we already know: - if (m_server->m_nodes.count(id)) - { - /* MEH. Far from an ideal solution. Leave alone for now. - // Already got this node. - // See if it's any better that ours or not... - // This could be the public address of a known node. - // SECURITY: remove this in beta - it's only for lazy connections and presents an easy attack vector. - if (m_server->m_nodes.count(id) && isPrivateAddress(m_server->m_nodes.at(id)->address.address()) && ep.port() != 0) - // Update address if the node if we now have a public IP for it. - m_server->m_nodes[id]->address = ep; - */ - goto CONTINUE; - } +// if (m_server->m_peers.count(id)) +// { +// /* MEH. Far from an ideal solution. Leave alone for now. +// // Already got this node. +// // See if it's any better that ours or not... +// // This could be the public address of a known node. +// // SECURITY: remove this in beta - it's only for lazy connections and presents an easy attack vector. +// if (m_server->m_peers.count(id) && isPrivateAddress(m_server->m_peers.at(id)->address.address()) && ep.port() != 0) +// // Update address if the node if we now have a public IP for it. +// m_server->m_peers[id]->address = ep; +// */ +// goto CONTINUE; +// } if (!ep.port()) - goto CONTINUE; // Zero port? Don't think so. + goto LAMEPEER; // Zero port? Don't think so. if (ep.port() >= /*49152*/32768) - goto CONTINUE; // Private port according to IANA. + goto LAMEPEER; // Private port according to IANA. - // TODO: PoC-7: - // Technically fine, but ignore for now to avoid peers passing on incoming ports until we can be sure that doesn't happen any more. -// if (ep.port() < 30300 || ep.port() > 30305) -// goto CONTINUE; // Wierd port. - - // Avoid our random other addresses that they might end up giving us. - for (auto i: m_server->m_peerAddresses) - if (ep.address() == i && ep.port() == m_server->listenPort()) - goto CONTINUE; - - // Check that we don't already know about this addr:port combination. If we are, assume the original is best. - // SECURITY: Not a valid assumption in general. Should compare ID origins and pick the best or note uncertainty and weight each equally. - for (auto const& i: m_server->m_nodes) - if (i.second->address == ep) - goto CONTINUE; // Same address but a different node. + // node table handles another node giving us a node which represents one of our other local network interfaces + // node table handles another node giving us a node we already know about // OK passed all our checks. Assume it's good. addRating(1000); // TODO: P2P change to addNode() -// m_server->noteNode(id, ep); + m_server->addNode(Node(id, NodeIPEndpoint(bi::udp::endpoint(ep.address(), 30303), ep))); + clogS(NetTriviaDetail) << "New peer: " << ep << "(" << id .abridged()<< ")"; CONTINUE:; + LAMEPEER:; } break; default: @@ -469,15 +477,15 @@ void Session::drop(DisconnectReason _reason) } catch (...) {} - if (m_node) + if (m_peer) { - if (_reason != m_node->lastDisconnect || _reason == NoDisconnect || _reason == ClientQuit || _reason == DisconnectRequested) - m_node->failedAttempts = 0; - m_node->lastDisconnect = _reason; + if (_reason != m_peer->lastDisconnect || _reason == NoDisconnect || _reason == ClientQuit || _reason == DisconnectRequested) + m_peer->failedAttempts = 0; + m_peer->lastDisconnect = _reason; if (_reason == BadProtocol) { - m_node->rating /= 2; - m_node->score /= 2; + m_peer->rating /= 2; + m_peer->score /= 2; } } m_dropped = true; diff --git a/libp2p/Session.h b/libp2p/Session.h index 19dc60a28..9c0472a81 100644 --- a/libp2p/Session.h +++ b/libp2p/Session.h @@ -39,7 +39,7 @@ namespace dev namespace p2p { -struct NodeInfo; +struct PeerInfo; /** * @brief The Session class @@ -51,7 +51,7 @@ class Session: public std::enable_shared_from_this friend class HostCapabilityFace; public: - Session(Host* _server, bi::tcp::socket _socket, std::shared_ptr const& _n); + Session(Host* _server, bi::tcp::socket _socket, std::shared_ptr const& _n); virtual ~Session(); void start(); @@ -80,7 +80,7 @@ public: void addNote(std::string const& _k, std::string const& _v) { m_info.notes[_k] = _v; } - PeerInfo const& info() const { return m_info; } + PeerSessionInfo const& info() const { return m_info; } void ensureNodesRequested(); void serviceNodesRequest(); @@ -109,10 +109,10 @@ private: std::array m_data; ///< Buffer for ingress packet data. bytes m_incoming; ///< Read buffer for ingress bytes. - PeerInfo m_info; ///< Dynamic information about this peer. + PeerSessionInfo m_info; ///< Dynamic information about this peer. unsigned m_protocolVersion = 0; ///< The protocol version of the peer. - std::shared_ptr m_node; ///< The NodeInfo object. + std::shared_ptr m_peer; ///< The PeerInfo object. bi::tcp::endpoint m_manualEndpoint; ///< The endpoint as specified by the constructor. bool m_dropped = false; ///< If true, we've already divested ourselves of this peer. We're just waiting for the reads & writes to fail before the shared_ptr goes OOS and the destructor kicks in. diff --git a/libwebthree/WebThree.cpp b/libwebthree/WebThree.cpp index fd51b5947..40e878858 100644 --- a/libwebthree/WebThree.cpp +++ b/libwebthree/WebThree.cpp @@ -75,7 +75,7 @@ void WebThreeDirect::setNetworkPreferences(p2p::NetworkPreferences const& _n) startNetwork(); } -std::vector WebThreeDirect::peers() +std::vector WebThreeDirect::peers() { return m_net.peers(); } @@ -102,5 +102,5 @@ void WebThreeDirect::restoreNodes(bytesConstRef _saved) void WebThreeDirect::connect(std::string const& _seedHost, unsigned short _port) { - m_net.connect(NodeId(), _seedHost, _port); + m_net.addNode(NodeId(), _seedHost, _port, _port); } diff --git a/libwebthree/WebThree.h b/libwebthree/WebThree.h index ec7bf2406..3a9fbbb30 100644 --- a/libwebthree/WebThree.h +++ b/libwebthree/WebThree.h @@ -84,7 +84,7 @@ public: // Network stuff: /// Get information on the current peer set. - std::vector peers(); + std::vector peers(); /// Same as peers().size(), but more efficient. size_t peerCount() const; @@ -195,7 +195,7 @@ public: // Peer network stuff - forward through RPCSlave, probably with P2PNetworkSlave/Master classes like Whisper & Ethereum. /// Get information on the current peer set. - std::vector peers(); + std::vector peers(); /// Same as peers().size(), but more efficient. size_t peerCount() const; diff --git a/neth/main.cpp b/neth/main.cpp index 8552177ce..2bcbb5e04 100644 --- a/neth/main.cpp +++ b/neth/main.cpp @@ -913,7 +913,7 @@ int main(int argc, char** argv) // Peers y = 1; - for (PeerInfo const& i: web3.peers()) + for (PeerSessionInfo const& i: web3.peers()) { auto s = boost::format("%1% ms - %2%:%3% - %4%") % toString(chrono::duration_cast(i.lastPing).count()) % diff --git a/test/peer.cpp b/test/peer.cpp index d13ea97f3..5c11d4cfb 100644 --- a/test/peer.cpp +++ b/test/peer.cpp @@ -49,14 +49,14 @@ int peerTest(int argc, char** argv) Host ph("Test", NetworkPreferences(listenPort)); if (!remoteHost.empty()) - ph.connect(NodeId(), remoteHost, remotePort); - - for (int i = 0; ; ++i) - { - this_thread::sleep_for(chrono::milliseconds(100)); - if (!(i % 10)) - ph.keepAlivePeers(); - } + ph.addNode(NodeId(), remoteHost, remotePort, remotePort); + +// for (int i = 0; ; ++i) +// { +// this_thread::sleep_for(chrono::milliseconds(100)); +// if (!(i % 10)) +// ph.keepAlivePeers(); +// } return 0; } diff --git a/test/whisperTopic.cpp b/test/whisperTopic.cpp index 8ecabde9b..ea46b162c 100644 --- a/test/whisperTopic.cpp +++ b/test/whisperTopic.cpp @@ -72,7 +72,7 @@ BOOST_AUTO_TEST_CASE(topic) this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); this_thread::sleep_for(chrono::milliseconds(500)); - ph.connect(NodeId(), "127.0.0.1", 50303); + ph.addNode(NodeId(), "127.0.0.1", 50303, 50303); KeyPair us = KeyPair::create(); for (int i = 0; i < 10; ++i) From 77316d19dc35a88b4b7390ed5ef418fb51640745 Mon Sep 17 00:00:00 2001 From: subtly Date: Sat, 10 Jan 2015 20:12:47 +0100 Subject: [PATCH 07/71] Don't setup nodetable until network is started. --- libp2p/Host.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 243738918..dd38ff5bd 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -52,8 +52,8 @@ Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n, bool m_ifAddresses(Network::getInterfaceAddresses()), m_ioService(2), m_tcp4Acceptor(m_ioService), - m_key(move(getHostIdentifier())), - m_nodeTable(new NodeTable(m_ioService, m_key)) + m_key(move(getHostIdentifier())) +// m_nodeTable(new NodeTable(m_ioService, m_key)) { for (auto address: m_ifAddresses) if (address.is_v4()) @@ -443,7 +443,7 @@ unsigned PeerInfo::fallbackSeconds() const } } -// TODO: P2P rebuild noetable when localNetworking is enabled/disabled +// TODO: P2P rebuild nodetable when localNetworking is enabled/disabled // TODO: P2P migrate grow/prunePeers into 'maintainPeers' & evaluate reputation instead of availability. schedule via deadline timer. //void Host::growPeers() //{ From 75f231419aad339028c92024ea55bd741b0e63e7 Mon Sep 17 00:00:00 2001 From: subtly Date: Mon, 12 Jan 2015 04:58:52 +0100 Subject: [PATCH 08/71] Connectivity and nodetable callbacks. Disable stale code. --- alethzero/MainWin.cpp | 4 ++-- libp2p/Host.cpp | 20 +++++++++++++++++++- libp2p/Host.h | 6 +++--- libp2p/NodeTable.cpp | 14 ++++++++++++-- libp2p/NodeTable.h | 20 ++++++++++++++++++-- libp2p/Session.cpp | 44 ++++++++++++++++++++++--------------------- libp2p/Session.h | 7 ++----- test/peer.cpp | 23 ++++++++++++++++++++++ 8 files changed, 102 insertions(+), 36 deletions(-) diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index f2437c6d2..f4750326d 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -930,7 +930,7 @@ void Main::refreshNetwork() .arg(QString::fromStdString(i.id.abridged()))); auto ns = web3()->nodes(); - for (p2p::Node const& i: ns) + for (p2p::PeerInfo const& i: ns) if (!i.dead) ui->nodes->insertItem(clients.count(i.id) ? 0 : ui->nodes->count(), QString("[%1 %3] %2 - ( =%5s | /%4s%6 ) - *%7 $%8") .arg(QString::fromStdString(i.id.abridged())) @@ -940,7 +940,7 @@ void Main::refreshNetwork() .arg(i.secondsSinceLastConnected()) .arg(i.isOffline() ? " | " + QString::fromStdString(reasonOf(i.lastDisconnect)) + " | " + QString::number(i.failedAttempts) + "x" : "") .arg(i.rating) - .arg((int)i.idOrigin) + .arg(0 /* (int)i.idOrigin */) ); } } diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index dd38ff5bd..61f30b548 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -177,6 +177,8 @@ void Host::onNodeTableEvent(NodeId _n, NodeTableEventType _e) { if (_e == NodeEntryAdded) { + clog(NetNote) << "p2p.host.nodeTable.events.nodeEntryAdded " << _n; + auto n = (*m_nodeTable)[_n]; if (n) { @@ -196,6 +198,8 @@ void Host::onNodeTableEvent(NodeId _n, NodeTableEventType _e) } else if (_e == NodeEntryRemoved) { + clog(NetNote) << "p2p.host.nodeTable.events.nodeEntryRemoved " << _n; + RecursiveGuard l(x_sessions); m_peers.erase(_n); } @@ -294,7 +298,17 @@ void Host::runAcceptor() { try { - doHandshake(s); +// RecursiveGuard l(x_sessions); +// auto p = m_peers[_n]; +// if (!p) +// { +// m_peers[_n] = make_shared(); +// p = m_peers[_n]; +// p->id = _n; +// } +// p->address = n.endpoint.tcp; + + doHandshake(s, NodeId()); success = true; } catch (Exception const& _e) @@ -329,6 +343,7 @@ void Host::doHandshake(bi::tcp::socket* _socket, NodeId _nodeId) clog(NetConnect) << "Accepting connection for " << _socket->remote_endpoint(); } catch (...){} + // auto p = std::make_shared(this, std::move(*_socket), m_peers[_nodeId]); p->start(); } @@ -538,6 +553,8 @@ void Host::run(boost::system::error_code const&) return; } + m_nodeTable->processEvents(); + for (auto p: m_sessions) if (auto pp = p.second.lock()) pp->serviceNodesRequest(); @@ -585,6 +602,7 @@ void Host::startedWorking() m_nodeTable.reset(new NodeTable(m_ioService, m_key, m_listenPort)); else m_nodeTable.reset(new NodeTable(m_ioService, m_key, m_listenPort > 0 ? m_listenPort : 30303)); + m_nodeTable->setEventHandler(new HostNodeTableHandler(*this)); } clog(NetNote) << "p2p.started id:" << id().abridged(); diff --git a/libp2p/Host.h b/libp2p/Host.h index 097b07433..3d386fe8c 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -110,10 +110,10 @@ struct PeerInfo using Nodes = std::vector; - -class Host; + class HostNodeTableHandler: public NodeTableEventHandler { + friend class Host; HostNodeTableHandler(Host& _host); virtual void processEvent(NodeId _n, NodeTableEventType _e); Host& m_host; @@ -213,7 +213,7 @@ private: void runAcceptor(); /// Handler for verifying handshake siganture before creating session. _egressNodeId is passed for outbound connections. - void doHandshake(bi::tcp::socket* _socket, NodeId _egressNodeId = NodeId()); + void doHandshake(bi::tcp::socket* _socket, NodeId _nodeId); void seal(bytes& _b); diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index b384c42d2..635f91bfb 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -65,6 +65,10 @@ shared_ptr NodeTable::addNode(Node const& _node) shared_ptr ret = m_nodes[_node.id]; if (!ret) { + clog(NodeTableNote) << "p2p.nodes.add " << _node.id.abridged(); + if (m_nodeEvents) + m_nodeEvents->appendEvent(_node.id, NodeEntryAdded); + ret.reset(new NodeEntry(m_node, _node.id, NodeIPEndpoint(_node.endpoint.udp, _node.endpoint.tcp))); m_nodes[_node.id] = ret; PingNode p(_node.endpoint.udp, m_node.endpoint.udp.address().to_string(), m_node.endpoint.udp.port()); @@ -317,8 +321,14 @@ void NodeTable::dropNode(shared_ptr _n) Guard l(x_state); s.nodes.remove_if([&_n](weak_ptr n) { return n.lock() == _n; }); } - Guard l(x_nodes); - m_nodes.erase(_n->id); + { + Guard l(x_nodes); + m_nodes.erase(_n->id); + } + + clog(NodeTableNote) << "p2p.nodes.drop " << _n->id.abridged(); + if (m_nodeEvents) + m_nodeEvents->appendEvent(_n->id, NodeEntryRemoved); } NodeTable::NodeBucket& NodeTable::bucket(NodeEntry const* _n) diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 7f75bf27e..75042dffc 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -94,7 +94,20 @@ public: protected: /// Called by NodeTable on behalf of an implementation (Host) to process new events without blocking nodetable. - void processEvents() { std::list> events; { Guard l(x_events); if (!m_nodeEvents.size()) return; m_nodeEvents.unique(); for (auto const& n: m_nodeEvents) events.push_back(std::make_pair(n,m_events[n])); m_nodeEvents.empty(); m_events.empty(); } for (auto const& e: events) processEvent(e.first, e.second); } + void processEvents() { + std::list> events; + { + Guard l(x_events); + if (!m_nodeEvents.size()) + return; + m_nodeEvents.unique(); + for (auto const& n: m_nodeEvents) events.push_back(std::make_pair(n,m_events[n])); + m_nodeEvents.empty(); + m_events.empty(); + } + for (auto const& e: events) + processEvent(e.first, e.second); + } /// Called by NodeTable to append event. virtual void appendEvent(NodeId _n, NodeTableEventType _e) { Guard l(x_events); m_nodeEvents.push_back(_n); m_events[_n] = _e; } @@ -173,7 +186,10 @@ public: void setEventHandler(NodeTableEventHandler* _handler) { m_nodeEvents.reset(_handler); } /// Called by implementation which provided handler to process NodeEntryAdded/NodeEntryRemoved events. Events are coalesced by type whereby old events are ignored. - void processEvents() { if (m_nodeEvents) m_nodeEvents->processEvents(); } + void processEvents() { + if (m_nodeEvents) + m_nodeEvents->processEvents(); + } /// Add node. Node will be pinged if it's not already known. std::shared_ptr addNode(Public const& _pubk, bi::udp::endpoint const& _udp, bi::tcp::endpoint const& _tcp = bi::tcp::endpoint()); diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index 0ef06ab50..80c538b24 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -39,9 +39,8 @@ using namespace dev::p2p; Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr const& _n): m_server(_s), m_socket(std::move(_socket)), - m_info({m_peer->id, "?", _n->address.address().to_string(), _n->address.port(), std::chrono::steady_clock::duration(0), CapDescSet(), 0, map()}), m_peer(_n), - m_manualEndpoint(_n->address) + m_info({NodeId(), "?", m_socket.remote_endpoint().address().to_string(), 0, std::chrono::steady_clock::duration(0), CapDescSet(), 0, map()}) { m_lastReceived = m_connect = std::chrono::steady_clock::now(); } @@ -87,21 +86,21 @@ int Session::rating() const return m_peer->rating; } -// TODO: P2P integration: session->? should be unavailable when socket isn't open -bi::tcp::endpoint Session::endpoint() const -{ - if (m_socket.is_open() && m_peer) - try - { - return bi::tcp::endpoint(m_socket.remote_endpoint().address(), m_peer->address.port()); - } - catch (...) {} - - if (m_peer) - return m_peer->address; - - return m_manualEndpoint; -} +//// TODO: P2P integration: session->? should be unavailable when socket isn't open +//bi::tcp::endpoint Session::endpoint() const +//{ +// if (m_socket.is_open() && m_peer) +// try +// { +// return bi::tcp::endpoint(m_socket.remote_endpoint().address(), m_peer->address.port()); +// } +// catch (...) {} +// +// if (m_peer) +// return m_peer->address; +// +// return m_manualEndpoint; +//} template vector randomSelection(vector const& _t, unsigned _n) { @@ -190,14 +189,11 @@ bool Session::interpret(RLP const& _r) if (m_server->id() == id) { // Already connected. - clogS(NetWarn) << "Connected to ourself under a false pretext. We were told this peer was id" << m_info.id.abridged(); + clogS(NetWarn) << "Connected to ourself under a false pretext. We were told this peer was id" << id.abridged(); disconnect(LocalIdentity); return true; } - assert(!!m_peer); - assert(!!m_peer->id); - // TODO: P2P ensure disabled logic is covered if (false /* m_server->havePeer(id) */) { @@ -218,7 +214,13 @@ bool Session::interpret(RLP const& _r) // TODO: P2P Move all node-lifecycle information into Host. Determine best way to handle peer-lifecycle properties vs node lifecycle. // TODO: P2P remove oldid // TODO: P2P with encrypted transport the handshake will fail and we won't get here + + // if peer is missing this is incoming connection and we need to tell host about new potential peer + + // m_peer = m_server->noteNode(m_peer->id, bi::tcp::endpoint(m_socket.remote_endpoint().address(), listenPort)); + assert(!!m_peer); + assert(!!m_peer->id); if (m_peer->isOffline()) m_peer->lastConnected = chrono::system_clock::now(); // diff --git a/libp2p/Session.h b/libp2p/Session.h index 9c0472a81..55330c93c 100644 --- a/libp2p/Session.h +++ b/libp2p/Session.h @@ -64,8 +64,6 @@ public: NodeId id() const; unsigned socketId() const { return m_socket.native_handle(); } - bi::tcp::endpoint endpoint() const; ///< for other peers to connect to. - template std::shared_ptr cap() const { try { return std::static_pointer_cast(m_capabilities.at(std::make_pair(PeerCap::name(), PeerCap::version()))); } catch (...) { return nullptr; } } @@ -109,13 +107,12 @@ private: std::array m_data; ///< Buffer for ingress packet data. bytes m_incoming; ///< Read buffer for ingress bytes. - PeerSessionInfo m_info; ///< Dynamic information about this peer. - unsigned m_protocolVersion = 0; ///< The protocol version of the peer. std::shared_ptr m_peer; ///< The PeerInfo object. - bi::tcp::endpoint m_manualEndpoint; ///< The endpoint as specified by the constructor. bool m_dropped = false; ///< If true, we've already divested ourselves of this peer. We're just waiting for the reads & writes to fail before the shared_ptr goes OOS and the destructor kicks in. + PeerSessionInfo m_info; ///< Dynamic information about this peer. + bool m_theyRequestedNodes = false; ///< Has the peer requested nodes from us without receiveing an answer from us? bool m_weRequestedNodes = false; ///< Have we requested nodes from the peer and not received an answer yet? diff --git a/test/peer.cpp b/test/peer.cpp index 5c11d4cfb..c3a617ce6 100644 --- a/test/peer.cpp +++ b/test/peer.cpp @@ -20,6 +20,7 @@ * Peer Network test functions. */ +#include #include #include #include @@ -27,6 +28,28 @@ using namespace std; using namespace dev; using namespace dev::p2p; +BOOST_AUTO_TEST_SUITE(p2p) + +BOOST_AUTO_TEST_CASE(host) +{ + NetworkPreferences host1prefs(30301, "127.0.0.1", true, true); + NetworkPreferences host2prefs(30302, "127.0.0.1", true, true); + + Host host1("Test", host1prefs); + NodeId node1 = host1.id(); + host1.start(); + + Host host2("Test", host2prefs); + auto node2 = host2.id(); + host2.start(); + + host1.addNode(node2, "127.0.0.1", host2prefs.listenPort, host2prefs.listenPort); + + this_thread::sleep_for(chrono::seconds(2)); +} + +BOOST_AUTO_TEST_SUITE_END() + int peerTest(int argc, char** argv) { short listenPort = 30303; From 7ac5c12978ae5b892cac234b6da77140b1ca1d08 Mon Sep 17 00:00:00 2001 From: subtly Date: Wed, 14 Jan 2015 12:32:12 -0500 Subject: [PATCH 09/71] peerinfo created closer to authentication (when it's known). fixes missing m_peer on connect. --- libp2p/Host.cpp | 36 ++++++++++++++++++---------------- libp2p/Host.h | 7 +++---- libp2p/Session.cpp | 49 ++++++++++++++++++---------------------------- 3 files changed, 41 insertions(+), 51 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 61f30b548..d7546bb11 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -157,13 +157,15 @@ unsigned Host::protocolVersion() const void Host::registerPeer(std::shared_ptr _s, CapDescs const& _caps) { - assert(!!_s->m_peer); - assert(!!_s->m_peer->id); + asserts(!!_s->m_peer->id); { RecursiveGuard l(x_sessions); + if (!m_peers.count(_s->m_peer->id)) + m_peers[_s->m_peer->id] = _s->m_peer; m_sessions[_s->m_peer->id] = _s; } + unsigned o = (unsigned)UserPacket; for (auto const& i: _caps) if (haveCapability(i)) @@ -298,16 +300,7 @@ void Host::runAcceptor() { try { -// RecursiveGuard l(x_sessions); -// auto p = m_peers[_n]; -// if (!p) -// { -// m_peers[_n] = make_shared(); -// p = m_peers[_n]; -// p->id = _n; -// } -// p->address = n.endpoint.tcp; - + // incoming connection so we don't yet know nodeid doHandshake(s, NodeId()); success = true; } @@ -343,9 +336,18 @@ void Host::doHandshake(bi::tcp::socket* _socket, NodeId _nodeId) clog(NetConnect) << "Accepting connection for " << _socket->remote_endpoint(); } catch (...){} - // - auto p = std::make_shared(this, std::move(*_socket), m_peers[_nodeId]); - p->start(); + shared_ptr p; + if (_nodeId) + p = m_peers[_nodeId]; + + if (!p) + { + p = make_shared(); + p->address.address(_socket->remote_endpoint().address()); + } + + auto ps = std::make_shared(this, std::move(*_socket), p); + ps->start(); } string Host::pocHost() @@ -426,8 +428,8 @@ void Host::connect(std::shared_ptr const& _n) clog(NetConnect) << "Connected to" << _n->id.abridged() << "@" << _n->address; _n->lastConnected = std::chrono::system_clock::now(); - auto p = make_shared(this, std::move(*s), _n); - p->start(); + auto ps = make_shared(this, std::move(*s), _n); + ps->start(); } delete s; diff --git a/libp2p/Host.h b/libp2p/Host.h index 3d386fe8c..0e86710ec 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -212,8 +212,8 @@ private: /// Called only from startedWorking(). void runAcceptor(); - /// Handler for verifying handshake siganture before creating session. _egressNodeId is passed for outbound connections. - void doHandshake(bi::tcp::socket* _socket, NodeId _nodeId); + /// Handler for verifying handshake siganture before creating session. _nodeId is passed for outbound connections. + void doHandshake(bi::tcp::socket* _socket, NodeId _nodeId = NodeId()); void seal(bytes& _b); @@ -261,11 +261,10 @@ private: std::map> m_peers; - mutable RecursiveMutex x_sessions; - /// The nodes to which we are currently connected. /// Mutable because we flush zombie entries (null-weakptrs) as regular maintenance from a const method. mutable std::map> m_sessions; + mutable RecursiveMutex x_sessions; unsigned m_idealPeerCount = 5; ///< Ideal number of peers to be connected to. diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index 80c538b24..bafaed7e3 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -86,22 +86,6 @@ int Session::rating() const return m_peer->rating; } -//// TODO: P2P integration: session->? should be unavailable when socket isn't open -//bi::tcp::endpoint Session::endpoint() const -//{ -// if (m_socket.is_open() && m_peer) -// try -// { -// return bi::tcp::endpoint(m_socket.remote_endpoint().address(), m_peer->address.port()); -// } -// catch (...) {} -// -// if (m_peer) -// return m_peer->address; -// -// return m_manualEndpoint; -//} - template vector randomSelection(vector const& _t, unsigned _n) { if (_t.size() <= _n) @@ -172,6 +156,11 @@ bool Session::interpret(RLP const& _r) { case HelloPacket: { + // TODO: P2P first pass, implement signatures. if signature fails, drop connection. if egress, flag node's endpoint as stale. + // Move auth to Host so we consolidate authentication logic and eschew peer deduplication logic. + // Move all node-lifecycle information into Host. + // Finalize peer-lifecycle properties vs node lifecycle. + m_protocolVersion = _r[1].toInt(); auto clientVersion = _r[2].toString(); auto caps = _r[3].toVector(); @@ -194,7 +183,7 @@ bool Session::interpret(RLP const& _r) return true; } - // TODO: P2P ensure disabled logic is covered + // TODO: P2P ensure disabled logic is considered if (false /* m_server->havePeer(id) */) { // Already connected. @@ -203,27 +192,28 @@ bool Session::interpret(RLP const& _r) return true; } + // if peer and connection have id, check for UnexpectedIdentity if (!id) { disconnect(NullIdentity); return true; } - - // TODO: P2P first pass, implement signatures. if signature fails, drop connection. if egress, flag node's endpoint as stale. - // Discussion: Most this to Host so we consolidate authentication logic and eschew peer deduplication logic. - // TODO: P2P Move all node-lifecycle information into Host. Determine best way to handle peer-lifecycle properties vs node lifecycle. - // TODO: P2P remove oldid - // TODO: P2P with encrypted transport the handshake will fail and we won't get here - - // if peer is missing this is incoming connection and we need to tell host about new potential peer - + else if (!m_peer->id) + { + m_peer->id = id; + m_peer->address.port(listenPort); + } + else if (m_peer->id != id) + { + disconnect(UnexpectedIdentity); + return true; + } -// m_peer = m_server->noteNode(m_peer->id, bi::tcp::endpoint(m_socket.remote_endpoint().address(), listenPort)); assert(!!m_peer); assert(!!m_peer->id); if (m_peer->isOffline()) m_peer->lastConnected = chrono::system_clock::now(); -// + // // TODO: P2P introduce map of nodes we've given to this node (if GetPeers/Peers stays in TCP) m_knownNodes.extendAll(m_peer->index); m_knownNodes.unionWith(m_peer->index); @@ -234,7 +224,6 @@ bool Session::interpret(RLP const& _r) return true; } - // TODO: P2P migrate auth to Host and Handshake to constructor m_info.clientVersion = clientVersion; m_info.host = m_socket.remote_endpoint().address().to_string(); m_info.port = listenPort; @@ -347,7 +336,7 @@ bool Session::interpret(RLP const& _r) // OK passed all our checks. Assume it's good. addRating(1000); - // TODO: P2P change to addNode() + // TODO: P2P test m_server->addNode(Node(id, NodeIPEndpoint(bi::udp::endpoint(ep.address(), 30303), ep))); clogS(NetTriviaDetail) << "New peer: " << ep << "(" << id .abridged()<< ")"; From b8da68e77405e25ec83ce14961a9fccd0ec0e3b0 Mon Sep 17 00:00:00 2001 From: subtly Date: Wed, 14 Jan 2015 12:41:42 -0500 Subject: [PATCH 10/71] empty events via clear --- libp2p/NodeTable.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 75042dffc..a68d42426 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -102,8 +102,8 @@ protected: return; m_nodeEvents.unique(); for (auto const& n: m_nodeEvents) events.push_back(std::make_pair(n,m_events[n])); - m_nodeEvents.empty(); - m_events.empty(); + m_nodeEvents.clear(); + m_events.clear(); } for (auto const& e: events) processEvent(e.first, e.second); From d13f69da5985e353ce63ad544b5966a67ea6c434 Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 15 Jan 2015 17:17:20 -0500 Subject: [PATCH 11/71] fix merge conflict --- libp2p/Host.h | 1 + libwebthree/WebThree.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/libp2p/Host.h b/libp2p/Host.h index 0e86710ec..d04470efd 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -125,6 +125,7 @@ class HostNodeTableHandler: public NodeTableEventHandler * @todo gracefully disconnect peer if peer already connected * @todo determinePublic: ipv6, udp * @todo handle conflict if addNode/requireNode called and Node already exists w/conflicting tcp or udp port + * @todo writing host identifier */ class Host: public Worker { diff --git a/libwebthree/WebThree.h b/libwebthree/WebThree.h index c5655b581..f2130d3f4 100644 --- a/libwebthree/WebThree.h +++ b/libwebthree/WebThree.h @@ -54,7 +54,7 @@ class WebThreeNetworkFace { public: /// Get information on the current peer set. - virtual std::vector peers() = 0; + virtual std::vector peers() = 0; /// Same as peers().size(), but more efficient. virtual size_t peerCount() const = 0; From f0a06fa1153db6f5b53c86a513f7604daa78af19 Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 16 Jan 2015 21:37:21 -0500 Subject: [PATCH 12/71] evictions logic --- libp2p/NodeTable.cpp | 14 +++++++++++++- libp2p/NodeTable.h | 13 +++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 635f91bfb..1e6d54682 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -374,7 +374,19 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes // clog(NodeTableMessageSummary) << "Received Pong from " << _from.address().to_string() << ":" << _from.port(); Pong in = Pong::fromBytesConstRef(_from, rlpBytes); - // whenever a pong is received, first check if it's in m_evictions + // whenever a pong is received, check if it's in m_evictions + Guard le(x_evictions); + for (auto it = m_evictions.begin(); it != m_evictions.end(); it++) + if (it->first.first == nodeid && it->first.second > std::chrono::steady_clock::now()) + { + if (auto n = getNodeEntry(it->second)) + dropNode(n); + + if (auto n = (*this)[it->first.first]) + addNode(n); + + m_evictions.erase(it); + } break; } diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index a68d42426..66f445827 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -125,19 +125,19 @@ protected: * shared_ptr replacement instead of mutating values. * * [Integration] - * @todo deadline-timer which maintains tcp/peer connections * @todo restore nodes: affects refreshbuckets * @todo TCP endpoints * @todo makeRequired: don't try to evict node if node isRequired. * @todo makeRequired: exclude bucket from refresh if we have node as peer. * * [Optimization] + * @todo serialize evictions per-bucket + * @todo store evictions in map, unit-test eviction logic + * @todo store root node in table * @todo encapsulate doFindNode into NetworkAlgorithm (task) * @todo Pong to include ip:port where ping was received * @todo expiration and sha3(id) 'to' for messages which are replies (prevents replay) - * @todo std::shared_ptr m_cachedPingPacket; - * @todo std::shared_ptr m_cachedFindSelfPacket; - * @todo store root node in table? + * @todo cache Ping and FindSelf * * [Networking] * @todo TCP endpoints @@ -145,11 +145,8 @@ protected: * @todo firewall * * [Protocol] - * @todo post-eviction pong * @todo optimize knowledge at opposite edges; eg, s_bitsPerStep lookups. (Can be done via pointers to NodeBucket) - * @todo ^ s_bitsPerStep = 5; // Denoted by b in [Kademlia]. Bits by which address space is divided. - * @todo optimize (use tree for state and/or custom compare for cache) - * @todo reputation (aka universal siblings lists) + * @todo ^ s_bitsPerStep = 8; // Denoted by b in [Kademlia]. Bits by which address space is divided. */ class NodeTable: UDPSocketEvents, public std::enable_shared_from_this { From f63e53f7351692c9c03fc45d635b0d8c729995a7 Mon Sep 17 00:00:00 2001 From: subtly Date: Sat, 17 Jan 2015 00:51:27 -0500 Subject: [PATCH 13/71] cleanup, sanity checks, and last pass of noting todos. --- libp2p/Host.cpp | 2 +- libp2p/Host.h | 8 ++---- libp2p/NodeTable.cpp | 10 ++++++- libp2p/Session.cpp | 63 ++++++++++++-------------------------------- libp2p/Session.h | 1 - 5 files changed, 29 insertions(+), 55 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index f566af6e1..d4ef54795 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -644,7 +644,7 @@ bytes Host::saveNodes() const { PeerInfo const& n = *(i.second); // TODO: PoC-7: Figure out why it ever shares these ports.//n.address.port() >= 30300 && n.address.port() <= 30305 && - if (!n.dead && chrono::system_clock::now() - n.lastConnected < chrono::seconds(3600 * 48) && n.address.port() > 0 && n.address.port() < /*49152*/32768 && n.id != id() && !isPrivateAddress(n.address.address())) + if (chrono::system_clock::now() - n.lastConnected < chrono::seconds(3600 * 48) && n.address.port() > 0 && n.address.port() < /*49152*/32768 && n.id != id() && !isPrivateAddress(n.address.address())) { nodes.appendList(10); if (n.address.address().is_v4()) diff --git a/libp2p/Host.h b/libp2p/Host.h index d04470efd..f979ceebc 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -54,7 +54,6 @@ class Host; struct PeerInfo { NodeId id; ///< Their id/public key. - unsigned index; ///< Index into m_nodesList // p2p: move to NodeIPEndpoint bi::tcp::endpoint address; ///< As reported from the node itself. @@ -65,10 +64,6 @@ struct PeerInfo unsigned failedAttempts = 0; DisconnectReason lastDisconnect = NoDisconnect; ///< Reason for disconnect that happened last. - // p2p: just remove node - // p2p: revisit logic in see Session.cpp#210 - bool dead = false; ///< If true, we believe this node is permanently dead - forget all about it. - // p2p: move to protocol-specific map int score = 0; ///< All time cumulative. int rating = 0; ///< Trending. @@ -125,7 +120,8 @@ class HostNodeTableHandler: public NodeTableEventHandler * @todo gracefully disconnect peer if peer already connected * @todo determinePublic: ipv6, udp * @todo handle conflict if addNode/requireNode called and Node already exists w/conflicting tcp or udp port - * @todo writing host identifier + * @todo write host identifier to disk along w/nodes + * @todo move Session::addRating into Host and implement via sender-tagged events */ class Host: public Worker { diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 1e6d54682..90916450d 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -63,7 +63,15 @@ shared_ptr NodeTable::addNode(Node const& _node) { Guard l(x_nodes); shared_ptr ret = m_nodes[_node.id]; - if (!ret) + if (ret) + { + // TODO: p2p robust percolation of node-endpoint changes +// // SECURITY: remove this in beta - it's only for lazy connections and presents an easy attack vector. +// if (m_server->m_peers.count(id) && isPrivateAddress(m_server->m_peers.at(id)->address.address()) && ep.port() != 0) +// // Update address if the node if we now have a public IP for it. +// m_server->m_peers[id]->address = ep; + } + else { clog(NodeTableNote) << "p2p.nodes.add " << _node.id.abridged(); if (m_nodeEvents) diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index bafaed7e3..d57a16fa2 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -47,9 +47,7 @@ Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr co Session::~Session() { - // TODO: P2P revisit (refactored from previous logic) - if (m_peer && !(id() && !isPermanentProblem(m_peer->lastDisconnect) && !m_peer->dead)) - m_peer->lastConnected = m_peer->lastAttempted - chrono::seconds(1); + m_peer->lastConnected = m_peer->lastAttempted - chrono::seconds(1); // Read-chain finished for one reason or another. for (auto& i: m_capabilities) @@ -136,8 +134,6 @@ void Session::serviceNodesRequest() s.appendList(3) << bytesConstRef(i.address.address().to_v4().to_bytes().data(), 4) << i.address.port() << i.id; else// if (i.second.address().is_v6()) - assumed s.appendList(3) << bytesConstRef(i.address.address().to_v6().to_bytes().data(), 16) << i.address.port() << i.id; - m_knownNodes.extendAll(i.index); - m_knownNodes.unionWith(i.index); } sealAndSend(s); m_theyRequestedNodes = false; @@ -183,15 +179,6 @@ bool Session::interpret(RLP const& _r) return true; } - // TODO: P2P ensure disabled logic is considered - if (false /* m_server->havePeer(id) */) - { - // Already connected. - clogS(NetWarn) << "Already connected to a peer with id" << id.abridged(); - disconnect(DuplicatePeer); - return true; - } - // if peer and connection have id, check for UnexpectedIdentity if (!id) { @@ -208,16 +195,18 @@ bool Session::interpret(RLP const& _r) disconnect(UnexpectedIdentity); return true; } + + if (m_server->havePeerSession(id)) + { + // Already connected. + clogS(NetWarn) << "Already connected to a peer with id" << id.abridged(); + disconnect(DuplicatePeer); + return true; + } - assert(!!m_peer); - assert(!!m_peer->id); if (m_peer->isOffline()) m_peer->lastConnected = chrono::system_clock::now(); -// // TODO: P2P introduce map of nodes we've given to this node (if GetPeers/Peers stays in TCP) - m_knownNodes.extendAll(m_peer->index); - m_knownNodes.unionWith(m_peer->index); - if (m_protocolVersion != m_server->protocolVersion()) { disconnect(IncompatibleProtocol); @@ -262,12 +251,20 @@ bool Session::interpret(RLP const& _r) break; case GetPeersPacket: { + // Disabled for interop testing. + // GetPeers/PeersPacket will be modified to only exchange new nodes which it's peers are interested in. + break; + clogS(NetTriviaSummary) << "GetPeers"; m_theyRequestedNodes = true; serviceNodesRequest(); break; } case PeersPacket: + // Disabled for interop testing. + // GetPeers/PeersPacket will be modified to only exchange new nodes which it's peers are interested in. + break; + clogS(NetTriviaSummary) << "Peers (" << dec << (_r.itemCount() - 1) << " entries)"; m_weRequestedNodes = false; for (unsigned i = 1; i < _r.itemCount(); ++i) @@ -304,41 +301,15 @@ bool Session::interpret(RLP const& _r) if (id == this->id()) goto LAMEPEER; // Just their info - we already have that. - // we don't worry about m_peers.count(id) now because node table will handle this and - // by default we will not blindly connect to nodes received via tcp; instead they will - // be pinged, as-is standard, by the node table and added if appropriate. unless flagged - // as required, nodes aren't connected to unless they respond via discovery; no matter if - // a node is relayed via udp or tcp. - // check that it's not us or one we already know: -// if (m_server->m_peers.count(id)) -// { -// /* MEH. Far from an ideal solution. Leave alone for now. -// // Already got this node. -// // See if it's any better that ours or not... -// // This could be the public address of a known node. -// // SECURITY: remove this in beta - it's only for lazy connections and presents an easy attack vector. -// if (m_server->m_peers.count(id) && isPrivateAddress(m_server->m_peers.at(id)->address.address()) && ep.port() != 0) -// // Update address if the node if we now have a public IP for it. -// m_server->m_peers[id]->address = ep; -// */ -// goto CONTINUE; -// } - if (!ep.port()) goto LAMEPEER; // Zero port? Don't think so. if (ep.port() >= /*49152*/32768) goto LAMEPEER; // Private port according to IANA. - // node table handles another node giving us a node which represents one of our other local network interfaces - // node table handles another node giving us a node we already know about - // OK passed all our checks. Assume it's good. addRating(1000); - - // TODO: P2P test m_server->addNode(Node(id, NodeIPEndpoint(bi::udp::endpoint(ep.address(), 30303), ep))); - clogS(NetTriviaDetail) << "New peer: " << ep << "(" << id .abridged()<< ")"; CONTINUE:; LAMEPEER:; diff --git a/libp2p/Session.h b/libp2p/Session.h index 55330c93c..68f9e5fa6 100644 --- a/libp2p/Session.h +++ b/libp2p/Session.h @@ -121,7 +121,6 @@ private: std::chrono::steady_clock::time_point m_lastReceived; ///< Time point of last message. std::map> m_capabilities; ///< The peer's capability set. - RangeMask m_knownNodes; ///< Nodes we already know about as indices into Host's nodesList. These shouldn't be resent to peer. }; } From 94ca6ca088d1c416df81c1b7bebb4d9eb752a01f Mon Sep 17 00:00:00 2001 From: subtly Date: Sat, 17 Jan 2015 00:56:36 -0500 Subject: [PATCH 14/71] cleanup --- libp2p/Host.cpp | 64 +------------------------------------------------ test/peer.cpp | 1 - 2 files changed, 1 insertion(+), 64 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index d4ef54795..a63b47b76 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -462,69 +462,7 @@ unsigned PeerInfo::fallbackSeconds() const } // TODO: P2P rebuild nodetable when localNetworking is enabled/disabled -// TODO: P2P migrate grow/prunePeers into 'maintainPeers' & evaluate reputation instead of availability. schedule via deadline timer. -//void Host::growPeers() -//{ -// RecursiveGuard l(x_peers); -// int morePeers = (int)m_idealPeerCount - m_peers.size(); -// if (morePeers > 0) -// { -// auto toTry = m_ready; -// if (!m_netPrefs.localNetworking) -// toTry -= m_private; -// set ns; -// for (auto i: toTry) -// if (m_nodes[m_nodesList[i]]->shouldReconnect()) -// ns.insert(*m_nodes[m_nodesList[i]]); -// -// if (ns.size()) -// for (Node const& i: ns) -// { -// connect(m_nodes[i.id]); -// if (!--morePeers) -// return; -// } -// else -// for (auto const& i: m_peers) -// if (auto p = i.second.lock()) -// p->ensureNodesRequested(); -// } -//} -// -//void Host::prunePeers() -//{ -// RecursiveGuard l(x_peers); -// // We'll keep at most twice as many as is ideal, halfing what counts as "too young to kill" until we get there. -// set dc; -// for (unsigned old = 15000; m_peers.size() - dc.size() > m_idealPeerCount * 2 && old > 100; old /= 2) -// if (m_peers.size() - dc.size() > m_idealPeerCount) -// { -// // look for worst peer to kick off -// // first work out how many are old enough to kick off. -// shared_ptr worst; -// unsigned agedPeers = 0; -// for (auto i: m_peers) -// if (!dc.count(i.first)) -// if (auto p = i.second.lock()) -// if (chrono::steady_clock::now() > p->m_connect + chrono::milliseconds(old)) // don't throw off new peers; peer-servers should never kick off other peer-servers. -// { -// ++agedPeers; -// if ((!worst || p->rating() < worst->rating() || (p->rating() == worst->rating() && p->m_connect > worst->m_connect))) // kill older ones -// worst = p; -// } -// if (!worst || agedPeers <= m_idealPeerCount) -// break; -// dc.insert(worst->id()); -// worst->disconnect(TooManyPeers); -// } -// -// // Remove dead peers from list. -// for (auto i = m_peers.begin(); i != m_peers.end();) -// if (i->second.lock().get()) -// ++i; -// else -// i = m_peers.erase(i); -//} +// TODO: P2P implement 'maintainPeers' & evaluate reputation instead of availability. schedule via deadline timer. PeerSessionInfos Host::peers() const { diff --git a/test/peer.cpp b/test/peer.cpp index c3a617ce6..43ddc792a 100644 --- a/test/peer.cpp +++ b/test/peer.cpp @@ -36,7 +36,6 @@ BOOST_AUTO_TEST_CASE(host) NetworkPreferences host2prefs(30302, "127.0.0.1", true, true); Host host1("Test", host1prefs); - NodeId node1 = host1.id(); host1.start(); Host host2("Test", host2prefs); From c01b6da4345731cf8a6af8ce9480abc11923a28e Mon Sep 17 00:00:00 2001 From: subtly Date: Sat, 17 Jan 2015 01:10:50 -0500 Subject: [PATCH 15/71] cleanup --- libp2p/Host.cpp | 5 +++-- libp2p/Host.h | 5 ++--- libp2p/NodeTable.cpp | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index a63b47b76..b330386fb 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -374,7 +374,8 @@ void Host::addNode(NodeId const& _node, std::string const& _addr, unsigned short if (ec) { bi::tcp::resolver r(m_ioService); - r.async_resolve({_addr, toString(_tcpPeerPort)}, [=](boost::system::error_code const& _ec, bi::tcp::resolver::iterator _epIt) { + r.async_resolve({_addr, toString(_tcpPeerPort)}, [=](boost::system::error_code const& _ec, bi::tcp::resolver::iterator _epIt) + { if (_ec) return; bi::tcp::endpoint tcp = *_epIt; @@ -503,7 +504,7 @@ void Host::run(boost::system::error_code const&) if (chrono::steady_clock::now() - m_lastPing >= chrono::seconds(30)) // ping every 30s. keepAlivePeers(); - auto runcb = [this](boost::system::error_code const& error) -> void { run(error); }; + auto runcb = [this](boost::system::error_code const& error) { run(error); }; m_timer->expires_from_now(boost::posix_time::milliseconds(c_timerInterval)); m_timer->async_wait(runcb); } diff --git a/libp2p/Host.h b/libp2p/Host.h index f979ceebc..3b1825cca 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -78,11 +78,10 @@ struct PeerInfo // p2p: move to NodeIPEndpoint bool shouldReconnect() const { return std::chrono::system_clock::now() > lastAttempted + std::chrono::seconds(fallbackSeconds()); } - // p2p: This has two meanings now. It's possible UDP works but TPC is down (unable to punch hole). - // p2p: Rename to isConnect() and move to endpoint. revisit Session.cpp#245, MainWin.cpp#877 + // p2p: This has two meanings now. It's possible UDP works but TPC is down or vice-versa. bool isOffline() const { return lastAttempted > lastConnected; } - // p2p: Remove (in favor of lru eviction and sub-protocol ratings). + // p2p: Remove (in favor of lr-seen eviction and ratings). bool operator<(PeerInfo const& _n) const { if (isOffline() != _n.isOffline()) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 90916450d..75e642f2e 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -506,7 +506,7 @@ void NodeTable::doRefreshBuckets(boost::system::error_code const& _ec) } unsigned nextRefresh = connected ? (refreshed ? 200 : c_bucketRefresh.count()*1000) : 10000; - auto runcb = [this](boost::system::error_code const& error) -> void { doRefreshBuckets(error); }; + auto runcb = [this](boost::system::error_code const& error) { doRefreshBuckets(error); }; m_bucketRefreshTimer.expires_from_now(boost::posix_time::milliseconds(nextRefresh)); m_bucketRefreshTimer.async_wait(runcb); } From 8845967a1d536031ab9d771e42de59e183db58a3 Mon Sep 17 00:00:00 2001 From: subtly Date: Sat, 17 Jan 2015 01:30:52 -0500 Subject: [PATCH 16/71] coding standards --- libp2p/Host.cpp | 3 --- libp2p/NodeTable.cpp | 6 ++++++ libp2p/NodeTable.h | 8 +++----- test/peer.cpp | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index b330386fb..8829b728e 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -53,7 +53,6 @@ Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n, bool m_ioService(2), m_tcp4Acceptor(m_ioService), m_key(move(getHostIdentifier())) -// m_nodeTable(new NodeTable(m_ioService, m_key)) { for (auto address: m_ifAddresses) if (address.is_v4()) @@ -157,8 +156,6 @@ unsigned Host::protocolVersion() const void Host::registerPeer(std::shared_ptr _s, CapDescs const& _caps) { - asserts(!!_s->m_peer->id); - { RecursiveGuard l(x_sessions); if (!m_peers.count(_s->m_peer->id)) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 75e642f2e..242c1b325 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -53,6 +53,12 @@ NodeTable::~NodeTable() m_socketPtr->disconnect(); } +void NodeTable::processEvents() +{ + if (m_nodeEvents) + m_nodeEvents->processEvents(); +} + shared_ptr NodeTable::addNode(Public const& _pubk, bi::udp::endpoint const& _udp, bi::tcp::endpoint const& _tcp) { auto node = Node(_pubk, NodeIPEndpoint(_udp, _tcp)); diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 66f445827..b68a9e346 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -94,7 +94,8 @@ public: protected: /// Called by NodeTable on behalf of an implementation (Host) to process new events without blocking nodetable. - void processEvents() { + void processEvents() + { std::list> events; { Guard l(x_events); @@ -183,10 +184,7 @@ public: void setEventHandler(NodeTableEventHandler* _handler) { m_nodeEvents.reset(_handler); } /// Called by implementation which provided handler to process NodeEntryAdded/NodeEntryRemoved events. Events are coalesced by type whereby old events are ignored. - void processEvents() { - if (m_nodeEvents) - m_nodeEvents->processEvents(); - } + void processEvents(); /// Add node. Node will be pinged if it's not already known. std::shared_ptr addNode(Public const& _pubk, bi::udp::endpoint const& _udp, bi::tcp::endpoint const& _tcp = bi::tcp::endpoint()); diff --git a/test/peer.cpp b/test/peer.cpp index 43ddc792a..51e777a56 100644 --- a/test/peer.cpp +++ b/test/peer.cpp @@ -44,7 +44,7 @@ BOOST_AUTO_TEST_CASE(host) host1.addNode(node2, "127.0.0.1", host2prefs.listenPort, host2prefs.listenPort); - this_thread::sleep_for(chrono::seconds(2)); + this_thread::sleep_for(chrono::seconds(1)); } BOOST_AUTO_TEST_SUITE_END() From 97f6145b939e43fa32ab4cd4dc521ff5a976ac79 Mon Sep 17 00:00:00 2001 From: subtly Date: Sat, 17 Jan 2015 02:45:59 -0500 Subject: [PATCH 17/71] compile fix --- alethzero/MainWin.cpp | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index 0a6622e7c..87704eb04 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -943,17 +943,16 @@ void Main::refreshNetwork() auto ns = web3()->nodes(); for (p2p::PeerInfo const& i: ns) - if (!i.dead) - ui->nodes->insertItem(clients.count(i.id) ? 0 : ui->nodes->count(), QString("[%1 %3] %2 - ( =%5s | /%4s%6 ) - *%7 $%8") - .arg(QString::fromStdString(i.id.abridged())) - .arg(QString::fromStdString(toString(i.address))) - .arg(i.id == web3()->id() ? "self" : clients.count(i.id) ? clients[i.id] : i.secondsSinceLastAttempted() == -1 ? "session-fail" : i.secondsSinceLastAttempted() >= (int)i.fallbackSeconds() ? "retrying..." : "retry-" + QString::number(i.fallbackSeconds() - i.secondsSinceLastAttempted()) + "s") - .arg(i.secondsSinceLastAttempted()) - .arg(i.secondsSinceLastConnected()) - .arg(i.isOffline() ? " | " + QString::fromStdString(reasonOf(i.lastDisconnect)) + " | " + QString::number(i.failedAttempts) + "x" : "") - .arg(i.rating) - .arg(0 /* (int)i.idOrigin */) - ); + ui->nodes->insertItem(clients.count(i.id) ? 0 : ui->nodes->count(), QString("[%1 %3] %2 - ( =%5s | /%4s%6 ) - *%7 $%8") + .arg(QString::fromStdString(i.id.abridged())) + .arg(QString::fromStdString(toString(i.address))) + .arg(i.id == web3()->id() ? "self" : clients.count(i.id) ? clients[i.id] : i.secondsSinceLastAttempted() == -1 ? "session-fail" : i.secondsSinceLastAttempted() >= (int)i.fallbackSeconds() ? "retrying..." : "retry-" + QString::number(i.fallbackSeconds() - i.secondsSinceLastAttempted()) + "s") + .arg(i.secondsSinceLastAttempted()) + .arg(i.secondsSinceLastConnected()) + .arg(i.isOffline() ? " | " + QString::fromStdString(reasonOf(i.lastDisconnect)) + " | " + QString::number(i.failedAttempts) + "x" : "") + .arg(i.rating) + .arg(0 /* (int)i.idOrigin */) + ); } } From a4bafa86eb51f8dcb0163e147895470ac9d63e30 Mon Sep 17 00:00:00 2001 From: subtly Date: Sat, 17 Jan 2015 14:13:59 -0500 Subject: [PATCH 18/71] header include path --- libp2p/Host.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 8829b728e..5eb948077 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include "Session.h" #include "Common.h" #include "Capability.h" From d7e3065f97c19381fff678fc15bb182819b31f49 Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 22 Jan 2015 18:32:12 -0500 Subject: [PATCH 19/71] Remove unused code paths with confidence. Rename key/identity to alias. Inherit Peer from Node and update Host/Session to use Node::endpoint instead of previous Peer::address. --- alethzero/MainWin.cpp | 2 +- libp2p/Common.h | 47 ++++++++++++++++-- libp2p/Host.cpp | 89 ++++++++++++++------------------- libp2p/Host.h | 110 +++++++++++++++++++---------------------- libp2p/NodeTable.h | 40 +-------------- libp2p/Session.cpp | 17 +++---- libp2p/Session.h | 6 +-- libwebthree/WebThree.h | 4 +- 8 files changed, 148 insertions(+), 167 deletions(-) diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index 87704eb04..6a44a2b24 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -942,7 +942,7 @@ void Main::refreshNetwork() .arg(QString::fromStdString(i.id.abridged()))); auto ns = web3()->nodes(); - for (p2p::PeerInfo const& i: ns) + for (p2p::Peer const& i: ns) ui->nodes->insertItem(clients.count(i.id) ? 0 : ui->nodes->count(), QString("[%1 %3] %2 - ( =%5s | /%4s%6 ) - *%7 $%8") .arg(QString::fromStdString(i.id.abridged())) .arg(QString::fromStdString(toString(i.address))) diff --git a/libp2p/Common.h b/libp2p/Common.h index 059e1a64e..38012ec76 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -16,9 +16,10 @@ */ /** @file Common.h * @author Gav Wood + * @author Alex Leverington * @date 2014 * - * Miscellanea required for the Host/Session classes. + * Miscellanea required for the Host/Session/NodeTable classes. */ #pragma once @@ -29,9 +30,8 @@ #include #include #include -#include +#include #include -#include namespace ba = boost::asio; namespace bi = boost::asio::ip; @@ -116,6 +116,11 @@ typedef std::pair CapDesc; typedef std::set CapDescSet; typedef std::vector CapDescs; +/* + * Used by Host to pass negotiated information about a connection to a + * new Peer Session; PeerSessionInfo is then maintained by Session and can + * be queried for point-in-time status information via Host. + */ struct PeerSessionInfo { NodeId id; @@ -130,5 +135,41 @@ struct PeerSessionInfo using PeerSessionInfos = std::vector; +/** + * @brief IPv4,UDP/TCP endpoints. + */ +struct NodeIPEndpoint +{ + NodeIPEndpoint(): udp(bi::udp::endpoint()), tcp(bi::tcp::endpoint()) {} + NodeIPEndpoint(bi::udp::endpoint _udp): udp(_udp) {} + NodeIPEndpoint(bi::tcp::endpoint _tcp): tcp(_tcp) {} + NodeIPEndpoint(bi::udp::endpoint _udp, bi::tcp::endpoint _tcp): udp(_udp), tcp(_tcp) {} + + bi::udp::endpoint udp; + bi::tcp::endpoint tcp; + + operator bool() const { return udp.address().is_unspecified() && tcp.address().is_unspecified(); } +}; + +struct Node +{ + Node(): endpoint(NodeIPEndpoint()) {}; + Node(Public _pubk, NodeIPEndpoint _ip, bool _required = false): id(_pubk), endpoint(_ip), required(_required) {} + Node(Public _pubk, bi::udp::endpoint _udp, bool _required = false): Node(_pubk, NodeIPEndpoint(_udp), _required) {} + + virtual NodeId const& address() const { return id; } + virtual Public const& publicKey() const { return id; } + + NodeId id; + + /// Endpoints by which we expect to reach node. + NodeIPEndpoint endpoint; + + /// If true, node will not be removed from Node list. + bool required = false; + + operator bool() const { return (bool)id; } +}; + } } diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 5eb948077..38929278d 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -52,7 +52,7 @@ Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n, bool m_ifAddresses(Network::getInterfaceAddresses()), m_ioService(2), m_tcp4Acceptor(m_ioService), - m_key(move(getHostIdentifier())) + m_alias(move(getHostIdentifier())) { for (auto address: m_ifAddresses) if (address.is_v4()) @@ -186,12 +186,20 @@ void Host::onNodeTableEvent(NodeId _n, NodeTableEventType _e) auto p = m_peers[_n]; if (!p) { - m_peers[_n] = make_shared(); + m_peers[_n] = make_shared(); p = m_peers[_n]; p->id = _n; } - p->address = n.endpoint.tcp; + p->endpoint.tcp = n.endpoint.tcp; + // TODO: Implement similar to doFindNode. Attempt connecting to nodes + // until ideal peer count is reached; if all nodes are tried, + // repeat. Notably, this is an integrated process such that + // when onNodeTableEvent occurs we should also update +/- + // the list of nodes to be tried. Thus: + // 1) externalize connection attempts + // 2) attempt copies potentialPeer list + // 3) replace this logic w/maintenance of potentialPeers if (peerCount() < m_idealPeerCount) connect(p); } @@ -334,14 +342,14 @@ void Host::doHandshake(bi::tcp::socket* _socket, NodeId _nodeId) clog(NetConnect) << "Accepting connection for " << _socket->remote_endpoint(); } catch (...){} - shared_ptr p; + shared_ptr p; if (_nodeId) p = m_peers[_nodeId]; if (!p) { - p = make_shared(); - p->address.address(_socket->remote_endpoint().address()); + p = make_shared(); + p->endpoint.tcp.address(_socket->remote_endpoint().address()); } auto ps = std::make_shared(this, std::move(*_socket), p); @@ -357,6 +365,8 @@ string Host::pocHost() void Host::addNode(NodeId const& _node, std::string const& _addr, unsigned short _tcpPeerPort, unsigned short _udpNodePort) { + + if (_tcpPeerPort < 30300 || _tcpPeerPort > 30305) cwarn << "Non-standard port being recorded: " << _tcpPeerPort; @@ -383,7 +393,7 @@ void Host::addNode(NodeId const& _node, std::string const& _addr, unsigned short addNode(Node(_node, NodeIPEndpoint(bi::udp::endpoint(addr, _udpNodePort), bi::tcp::endpoint(addr, _tcpPeerPort)))); } -void Host::connect(std::shared_ptr const& _n) +void Host::connect(std::shared_ptr const& _n) { if (!m_run) return; @@ -401,7 +411,7 @@ void Host::connect(std::shared_ptr const& _n) } // prevent concurrently connecting to a node - PeerInfo *nptr = _n.get(); + Peer *nptr = _n.get(); { Guard l(x_pendingNodeConns); if (m_pendingNodeConns.count(nptr)) @@ -409,22 +419,19 @@ void Host::connect(std::shared_ptr const& _n) m_pendingNodeConns.insert(nptr); } - clog(NetConnect) << "Attempting connection to node" << _n->id.abridged() << "@" << _n->address << "from" << id().abridged(); - _n->lastAttempted = std::chrono::system_clock::now(); - _n->failedAttempts++; - + clog(NetConnect) << "Attempting connection to node" << _n->id.abridged() << "@" << _n->peerEndpoint() << "from" << id().abridged(); bi::tcp::socket* s = new bi::tcp::socket(m_ioService); - s->async_connect(_n->address, [=](boost::system::error_code const& ec) + s->async_connect(_n->peerEndpoint(), [=](boost::system::error_code const& ec) { if (ec) { - clog(NetConnect) << "Connection refused to node" << _n->id.abridged() << "@" << _n->address << "(" << ec.message() << ")"; + clog(NetConnect) << "Connection refused to node" << _n->id.abridged() << "@" << _n->peerEndpoint() << "(" << ec.message() << ")"; _n->lastDisconnect = TCPError; _n->lastAttempted = std::chrono::system_clock::now(); } else { - clog(NetConnect) << "Connected to" << _n->id.abridged() << "@" << _n->address; + clog(NetConnect) << "Connected to" << _n->id.abridged() << "@" << _n->peerEndpoint(); _n->lastConnected = std::chrono::system_clock::now(); auto ps = make_shared(this, std::move(*s), _n); @@ -437,31 +444,6 @@ void Host::connect(std::shared_ptr const& _n) }); } -unsigned PeerInfo::fallbackSeconds() const -{ - switch (lastDisconnect) - { - case BadProtocol: - return 30 * (failedAttempts + 1); - case UselessPeer: - case TooManyPeers: - case ClientQuit: - return 15 * (failedAttempts + 1); - case NoDisconnect: - return 0; - default: - if (failedAttempts < 5) - return failedAttempts * 5; - else if (failedAttempts < 15) - return 25 + (failedAttempts - 5) * 10; - else - return 25 + 100 + (failedAttempts - 15) * 20; - } -} - -// TODO: P2P rebuild nodetable when localNetworking is enabled/disabled -// TODO: P2P implement 'maintainPeers' & evaluate reputation instead of availability. schedule via deadline timer. - PeerSessionInfos Host::peers() const { if (!m_run) @@ -538,12 +520,14 @@ void Host::startedWorking() if (!m_tcpPublic.address().is_unspecified()) // TODO: add m_tcpPublic endpoint; sort out endpoint stuff for nodetable - m_nodeTable.reset(new NodeTable(m_ioService, m_key, m_listenPort)); + m_nodeTable.reset(new NodeTable(m_ioService, m_alias, m_listenPort)); else - m_nodeTable.reset(new NodeTable(m_ioService, m_key, m_listenPort > 0 ? m_listenPort : 30303)); + m_nodeTable.reset(new NodeTable(m_ioService, m_alias, m_listenPort > 0 ? m_listenPort : 30303)); m_nodeTable->setEventHandler(new HostNodeTableHandler(*this)); } - + else + clog(NetNote) << "p2p.start.notice id:" << id().abridged() << "Invalid listen-port. Node Table Disabled."; + clog(NetNote) << "p2p.started id:" << id().abridged(); run(boost::system::error_code()); @@ -578,16 +562,16 @@ bytes Host::saveNodes() const RecursiveGuard l(x_sessions); for (auto const& i: m_peers) { - PeerInfo const& n = *(i.second); + Peer const& n = *(i.second); // TODO: PoC-7: Figure out why it ever shares these ports.//n.address.port() >= 30300 && n.address.port() <= 30305 && - if (chrono::system_clock::now() - n.lastConnected < chrono::seconds(3600 * 48) && n.address.port() > 0 && n.address.port() < /*49152*/32768 && n.id != id() && !isPrivateAddress(n.address.address())) + if (chrono::system_clock::now() - n.lastConnected < chrono::seconds(3600 * 48) && n.peerEndpoint().port() > 0 && n.peerEndpoint().port() < /*49152*/32768 && n.id != id() && !isPrivateAddress(n.peerEndpoint().address())) { nodes.appendList(10); - if (n.address.address().is_v4()) - nodes << n.address.address().to_v4().to_bytes(); + if (n.peerEndpoint().address().is_v4()) + nodes << n.peerEndpoint().address().to_v4().to_bytes(); else - nodes << n.address.address().to_v6().to_bytes(); - nodes << n.address.port() << n.id /* << (int)n.idOrigin */ << 0 + nodes << n.peerEndpoint().address().to_v6().to_bytes(); + nodes << n.peerEndpoint().port() << n.id /* << (int)n.idOrigin */ << 0 << chrono::duration_cast(n.lastConnected.time_since_epoch()).count() << chrono::duration_cast(n.lastAttempted.time_since_epoch()).count() << n.failedAttempts << (unsigned)n.lastDisconnect << n.score << n.rating; @@ -596,11 +580,12 @@ bytes Host::saveNodes() const } } RLPStream ret(3); - ret << 0 << m_key.secret(); + ret << 0 << m_alias.secret(); ret.appendList(count).appendRaw(nodes.out(), count); return ret.out(); } +// TODO: p2p Import-ant void Host::restoreNodes(bytesConstRef _b) { RecursiveGuard l(x_sessions); @@ -610,7 +595,7 @@ void Host::restoreNodes(bytesConstRef _b) { case 0: { - m_key = KeyPair(r[1].toHash()); + m_alias = KeyPair(r[1].toHash()); // noteNode(id(), m_tcpPublic); for (auto i: r[2]) @@ -623,7 +608,7 @@ void Host::restoreNodes(bytesConstRef _b) auto id = (NodeId)i[2]; if (!m_peers.count(id)) { -//// auto o = (Origin)i[3].toInt(); + // TODO: p2p Important :) // auto n = noteNode(id, ep); // n->lastConnected = chrono::system_clock::time_point(chrono::seconds(i[4].toInt())); // n->lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[5].toInt())); diff --git a/libp2p/Host.h b/libp2p/Host.h index 3b1825cca..1719bac6a 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -51,58 +51,50 @@ namespace p2p class Host; -struct PeerInfo +/** + * @brief Representation of connectivity state and all other pertinent Peer metadata. + * A Peer represents connectivity between two nodes, which in this case, are the host + * and remote nodes. + * + * State information necessary for loading network topology is maintained by NodeTable. + * + * @todo Implement 'bool required' + * @todo reputation: Move score, rating to capability-specific map (&& remove friend class) + * @todo reputation: implement via origin-tagged events + * @todo Populate metadata upon construction; save when destroyed. + * @todo Metadata for peers needs to be handled via a storage backend. + * Specifically, peers can be utilized in a variety of + * many-to-many relationships while also needing to modify shared instances of + * those peers. Modifying these properties via a storage backend alleviates + * Host of the responsibility. (&& remove save/restoreNodes) + * @todo reimplement recording of historical session information on per-transport basis + * @todo rebuild nodetable when localNetworking is enabled/disabled + */ +class Peer: public Node { - NodeId id; ///< Their id/public key. + friend class Session; /// Allows Session to update score and rating. + friend class Host; /// For Host: saveNodes(), restoreNodes() +public: + bool isOffline() const { return !m_session.lock(); } + + bi::tcp::endpoint const& peerEndpoint() const { return endpoint.tcp; } + +protected: + + /// Used by isOffline() and (todo) for peer to emit session information. + std::weak_ptr m_session; + + int score = 0; ///< All time cumulative. + int rating = 0; ///< Trending. - // p2p: move to NodeIPEndpoint - bi::tcp::endpoint address; ///< As reported from the node itself. + /// Network Availability - // p2p: This information is relevant to the network-stack, ex: firewall, rather than node itself std::chrono::system_clock::time_point lastConnected; std::chrono::system_clock::time_point lastAttempted; unsigned failedAttempts = 0; DisconnectReason lastDisconnect = NoDisconnect; ///< Reason for disconnect that happened last. - - // p2p: move to protocol-specific map - int score = 0; ///< All time cumulative. - int rating = 0; ///< Trending. - - // p2p: move to NodeIPEndpoint - int secondsSinceLastConnected() const { return lastConnected == std::chrono::system_clock::time_point() ? -1 : (int)std::chrono::duration_cast(std::chrono::system_clock::now() - lastConnected).count(); } - // p2p: move to NodeIPEndpoint - int secondsSinceLastAttempted() const { return lastAttempted == std::chrono::system_clock::time_point() ? -1 : (int)std::chrono::duration_cast(std::chrono::system_clock::now() - lastAttempted).count(); } - - // p2p: move to NodeIPEndpoint - unsigned fallbackSeconds() const; - // p2p: move to NodeIPEndpoint - bool shouldReconnect() const { return std::chrono::system_clock::now() > lastAttempted + std::chrono::seconds(fallbackSeconds()); } - - // p2p: This has two meanings now. It's possible UDP works but TPC is down or vice-versa. - bool isOffline() const { return lastAttempted > lastConnected; } - - // p2p: Remove (in favor of lr-seen eviction and ratings). - bool operator<(PeerInfo const& _n) const - { - if (isOffline() != _n.isOffline()) - return isOffline(); - else if (isOffline()) - if (lastAttempted == _n.lastAttempted) - return failedAttempts < _n.failedAttempts; - else - return lastAttempted < _n.lastAttempted; - else - if (score == _n.score) - if (rating == _n.rating) - return failedAttempts < _n.failedAttempts; - else - return rating < _n.rating; - else - return score < _n.score; - } }; - -using Nodes = std::vector; +using Peers = std::vector; class HostNodeTableHandler: public NodeTableEventHandler @@ -116,18 +108,18 @@ class HostNodeTableHandler: public NodeTableEventHandler /** * @brief The Host class * Capabilities should be registered prior to startNetwork, since m_capabilities is not thread-safe. - * @todo gracefully disconnect peer if peer already connected + * + * @todo onNodeTableEvent: move peer-connection logic into ensurePeers + * @todo handshake: gracefully disconnect peer if peer already connected * @todo determinePublic: ipv6, udp * @todo handle conflict if addNode/requireNode called and Node already exists w/conflicting tcp or udp port - * @todo write host identifier to disk along w/nodes - * @todo move Session::addRating into Host and implement via sender-tagged events + * @todo write host identifier to disk w/nodes */ class Host: public Worker { friend class HostNodeTableHandler; friend class Session; friend class HostCapabilityFace; - friend struct PeerInfo; public: /// Start server, listening for connections on the given port. @@ -163,6 +155,9 @@ public: /// Get number of peers connected; equivalent to, but faster than, peers().size(). size_t peerCount() const { RecursiveGuard l(x_sessions); return m_peers.size(); } + /// Get the address we're listening on currently. + std::string listenAddress() const { return m_tcpPublic.address().to_string(); } + /// Get the port we're listening on currently. unsigned short listenPort() const { return m_tcpPublic.port(); } @@ -173,7 +168,7 @@ public: void restoreNodes(bytesConstRef _b); // TODO: P2P this should be combined with peers into a HostStat object of some kind; coalesce data, as it's only used for status information. - Nodes nodes() const { RecursiveGuard l(x_sessions); Nodes ret; for (auto const& i: m_peers) ret.push_back(*i.second); return ret; } + Peers nodes() const { RecursiveGuard l(x_sessions); Peers ret; for (auto const& i: m_peers) ret.push_back(*i.second); return ret; } void setNetworkPreferences(NetworkPreferences const& _p) { auto had = isStarted(); if (had) stop(); m_netPrefs = _p; if (had) start(); } @@ -187,12 +182,10 @@ public: /// @returns if network is running. bool isStarted() const { return m_run; } - NodeId id() const { return m_key.pub(); } + NodeId id() const { return m_alias.pub(); } void registerPeer(std::shared_ptr _s, CapDescs const& _caps); -// std::shared_ptr node(NodeId _id) const { if (m_nodes.count(_id)) return m_nodes.at(_id); return std::shared_ptr(); } - protected: void onNodeTableEvent(NodeId _n, NodeTableEventType _e); @@ -200,7 +193,7 @@ private: /// Populate m_peerAddresses with available public addresses. void determinePublic(std::string const& _publicAddress, bool _upnp); - void connect(std::shared_ptr const& _n); + void connect(std::shared_ptr const& _n); /// Ping the peers to update the latency information and disconnect peers which have timed out. void keepAlivePeers(); @@ -225,7 +218,7 @@ private: virtual void doneWorking(); /// Add node - void addNode(Node const& _node) { m_nodeTable->addNode(_node); } + void addNode(Node const& _node) { if (m_nodeTable) m_nodeTable->addNode(_node); } /// Get or create host identifier (KeyPair). KeyPair getHostIdentifier(); @@ -248,16 +241,17 @@ private: std::unique_ptr m_timer; ///< Timer which, when network is running, calls scheduler() every c_timerInterval ms. static const unsigned c_timerInterval = 100; ///< Interval which m_timer is run when network is connected. - std::set m_pendingNodeConns; /// Used only by connect(PeerInfo&) to limit concurrently connecting to same node. See connect(shared_ptrconst&). + std::set m_pendingNodeConns; /// Used only by connect(Peer&) to limit concurrently connecting to same node. See connect(shared_ptrconst&). Mutex x_pendingNodeConns; bi::tcp::endpoint m_tcpPublic; ///< Our public listening endpoint. - KeyPair m_key; ///< Our unique ID. + KeyPair m_alias; ///< Alias for network communication. Network address is k*G. k is key material. TODO: Replace KeyPair. std::shared_ptr m_nodeTable; ///< Node table (uses kademlia-like discovery). - std::map> m_peers; + /// Shared storage of Peer objects. Peers are created or destroyed on demand by the Host. Active sessions maintain a shared_ptr to a Peer; + std::map> m_peers; - /// The nodes to which we are currently connected. + /// The nodes to which we are currently connected. Used by host to service peer requests and keepAlivePeers and for shutdown. (see run()) /// Mutable because we flush zombie entries (null-weakptrs) as regular maintenance from a const method. mutable std::map> m_sessions; mutable RecursiveMutex x_sessions; diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index b68a9e346..dee10c7ae 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -24,51 +24,14 @@ #include #include #include -#include #include +#include "Common.h" namespace dev { namespace p2p { -/** - * @brief IPv4,UDP/TCP endpoints. - */ -struct NodeIPEndpoint -{ - NodeIPEndpoint(): udp(bi::udp::endpoint()), tcp(bi::tcp::endpoint()) {} - NodeIPEndpoint(bi::udp::endpoint _udp): udp(_udp) {} - NodeIPEndpoint(bi::tcp::endpoint _tcp): tcp(_tcp) {} - NodeIPEndpoint(bi::udp::endpoint _udp, bi::tcp::endpoint _tcp): udp(_udp), tcp(_tcp) {} - - bi::udp::endpoint udp; - bi::tcp::endpoint tcp; - - operator bool() const { return udp.address().is_unspecified() && tcp.address().is_unspecified(); } -}; - -struct Node -{ - Node(): endpoint(NodeIPEndpoint()) {}; - Node(Public _pubk, NodeIPEndpoint _ip, bool _required = false): id(_pubk), endpoint(_ip), required(_required) {} - Node(Public _pubk, bi::udp::endpoint _udp, bool _required = false): Node(_pubk, NodeIPEndpoint(_udp), _required) {} - - virtual NodeId const& address() const { return id; } - virtual Public const& publicKey() const { return id; } - - NodeId id; - - /// Endpoints by which we expect to reach node. - NodeIPEndpoint endpoint; - - /// If true, node will not be removed from Node list. - bool required = false; - - operator bool() const { return (bool)id; } -}; - - /** * NodeEntry * @brief Entry in Node Table @@ -151,7 +114,6 @@ protected: */ class NodeTable: UDPSocketEvents, public std::enable_shared_from_this { - friend struct Neighbours; using NodeSocket = UDPSocket; using TimePoint = std::chrono::steady_clock::time_point; using EvictionTimeout = std::pair,NodeId>; ///< First NodeId may be evicted and replaced with second NodeId. diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index d57a16fa2..9f68f3ef0 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -36,7 +36,7 @@ using namespace dev::p2p; #endif #define clogS(X) dev::LogOutputStream(false) << "| " << std::setw(2) << m_socket.native_handle() << "] " -Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr const& _n): +Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr const& _n): m_server(_s), m_socket(std::move(_socket)), m_peer(_n), @@ -114,9 +114,9 @@ void Session::serviceNodesRequest() if (!m_theyRequestedNodes) return; -// TODO: P2P +// TODO: P2P reimplement, as per TCP "close nodes" gossip specifications (WiP) // auto peers = m_server->potentialPeers(m_knownNodes); - Nodes peers; + Peers peers; if (peers.empty()) { addNote("peers", "requested"); @@ -129,11 +129,11 @@ void Session::serviceNodesRequest() auto rs = randomSelection(peers, 10); for (auto const& i: rs) { - clogS(NetTriviaDetail) << "Sending peer " << i.id.abridged() << i.address; - if (i.address.address().is_v4()) - s.appendList(3) << bytesConstRef(i.address.address().to_v4().to_bytes().data(), 4) << i.address.port() << i.id; + clogS(NetTriviaDetail) << "Sending peer " << i.id.abridged() << i.peerEndpoint(); + if (i.peerEndpoint().address().is_v4()) + s.appendList(3) << bytesConstRef(i.peerEndpoint().address().to_v4().to_bytes().data(), 4) << i.peerEndpoint().port() << i.id; else// if (i.second.address().is_v6()) - assumed - s.appendList(3) << bytesConstRef(i.address.address().to_v6().to_bytes().data(), 16) << i.address.port() << i.id; + s.appendList(3) << bytesConstRef(i.peerEndpoint().address().to_v6().to_bytes().data(), 16) << i.peerEndpoint().port() << i.id; } sealAndSend(s); m_theyRequestedNodes = false; @@ -188,7 +188,7 @@ bool Session::interpret(RLP const& _r) else if (!m_peer->id) { m_peer->id = id; - m_peer->address.port(listenPort); + m_peer->endpoint.tcp.port(listenPort); } else if (m_peer->id != id) { @@ -386,7 +386,6 @@ void Session::send(bytes&& _msg) if (!checkPacket(bytesConstRef(&_msg))) clogS(NetWarn) << "INVALID PACKET CONSTRUCTED!"; -// cerr << (void*)this << " writeImpl" << endl; if (!m_socket.is_open()) return; diff --git a/libp2p/Session.h b/libp2p/Session.h index 68f9e5fa6..87620a604 100644 --- a/libp2p/Session.h +++ b/libp2p/Session.h @@ -39,7 +39,7 @@ namespace dev namespace p2p { -struct PeerInfo; +class Peer; /** * @brief The Session class @@ -51,7 +51,7 @@ class Session: public std::enable_shared_from_this friend class HostCapabilityFace; public: - Session(Host* _server, bi::tcp::socket _socket, std::shared_ptr const& _n); + Session(Host* _server, bi::tcp::socket _socket, std::shared_ptr const& _n); virtual ~Session(); void start(); @@ -108,7 +108,7 @@ private: bytes m_incoming; ///< Read buffer for ingress bytes. unsigned m_protocolVersion = 0; ///< The protocol version of the peer. - std::shared_ptr m_peer; ///< The PeerInfo object. + std::shared_ptr m_peer; ///< The Peer object. bool m_dropped = false; ///< If true, we've already divested ourselves of this peer. We're just waiting for the reads & writes to fail before the shared_ptr goes OOS and the destructor kicks in. PeerSessionInfo m_info; ///< Dynamic information about this peer. diff --git a/libwebthree/WebThree.h b/libwebthree/WebThree.h index f2130d3f4..3b97a0763 100644 --- a/libwebthree/WebThree.h +++ b/libwebthree/WebThree.h @@ -78,7 +78,7 @@ public: virtual p2p::NodeId id() const = 0; /// Gets the nodes. - virtual p2p::Nodes nodes() const = 0; + virtual p2p::Peers nodes() const = 0; /// Start the network subsystem. virtual void startNetwork() = 0; @@ -148,7 +148,7 @@ public: p2p::NodeId id() const override { return m_net.id(); } /// Gets the nodes. - p2p::Nodes nodes() const override { return m_net.nodes(); } + p2p::Peers nodes() const override { return m_net.nodes(); } /// Start the network subsystem. void startNetwork() override { m_net.start(); } From 26f8d3b62dcee0ca958420ececce9aa913bd85d8 Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 22 Jan 2015 19:50:33 -0500 Subject: [PATCH 20/71] update whisper test. move ports out of private ranges and specify ip address. --- test/whisperTopic.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/whisperTopic.cpp b/test/whisperTopic.cpp index ea46b162c..9f0f34630 100644 --- a/test/whisperTopic.cpp +++ b/test/whisperTopic.cpp @@ -36,15 +36,16 @@ BOOST_AUTO_TEST_CASE(topic) auto oldLogVerbosity = g_logVerbosity; g_logVerbosity = 0; + Host ph1("Test", NetworkPreferences(30303, "127.0.0.1", true, true)); + bool started = false; unsigned result = 0; std::thread listener([&]() { setThreadName("other"); - Host ph("Test", NetworkPreferences(50303, "", false, true)); - auto wh = ph.registerCapability(new WhisperHost()); - ph.start(); + auto wh = ph1.registerCapability(new WhisperHost()); + ph1.start(); started = true; @@ -67,12 +68,12 @@ BOOST_AUTO_TEST_CASE(topic) while (!started) this_thread::sleep_for(chrono::milliseconds(50)); - Host ph("Test", NetworkPreferences(50300, "", false, true)); + Host ph("Test", NetworkPreferences(30300, "127.0.0.1", true, true)); auto wh = ph.registerCapability(new WhisperHost()); this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); this_thread::sleep_for(chrono::milliseconds(500)); - ph.addNode(NodeId(), "127.0.0.1", 50303, 50303); + ph.addNode(ph1.id(), "127.0.0.1", 30303, 30303); KeyPair us = KeyPair::create(); for (int i = 0; i < 10; ++i) From a73f1fa7c55923cf13550efe8809e0ac26efaa8f Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 23 Jan 2015 04:23:55 -0500 Subject: [PATCH 21/71] add packet-type. prep for node-discovery interop. --- alethzero/MainWin.cpp | 13 +++++-------- libp2p/Host.h | 11 +++++------ libp2p/NodeTable.cpp | 25 +++++++++++++++---------- libp2p/NodeTable.h | 22 +++++++++++----------- libp2p/UDP.cpp | 14 +++++++++++--- libp2p/UDP.h | 3 ++- 6 files changed, 49 insertions(+), 39 deletions(-) diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index 7a28948f4..d96c55917 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -938,13 +938,13 @@ void Main::refreshNetwork() if (web3()->haveNetwork()) { - map clients; + map sessions; for (PeerSessionInfo const& i: ps) ui->peers->addItem(QString("[%8 %7] %3 ms - %1:%2 - %4 %5 %6") .arg(QString::fromStdString(i.host)) .arg(i.port) .arg(chrono::duration_cast(i.lastPing).count()) - .arg(clients[i.id] = QString::fromStdString(i.clientVersion)) + .arg(sessions[i.id] = QString::fromStdString(i.clientVersion)) .arg(QString::fromStdString(toString(i.caps))) .arg(QString::fromStdString(toString(i.notes))) .arg(i.socket) @@ -952,15 +952,12 @@ void Main::refreshNetwork() auto ns = web3()->nodes(); for (p2p::Peer const& i: ns) - ui->nodes->insertItem(clients.count(i.id) ? 0 : ui->nodes->count(), QString("[%1 %3] %2 - ( =%5s | /%4s%6 ) - *%7 $%8") + ui->nodes->insertItem(sessions.count(i.id) ? 0 : ui->nodes->count(), QString("[%1 %3] %2 - ( =%5s | /%4s%6 ) - *%7 $%8") .arg(QString::fromStdString(i.id.abridged())) - .arg(QString::fromStdString(toString(i.address))) - .arg(i.id == web3()->id() ? "self" : clients.count(i.id) ? clients[i.id] : i.secondsSinceLastAttempted() == -1 ? "session-fail" : i.secondsSinceLastAttempted() >= (int)i.fallbackSeconds() ? "retrying..." : "retry-" + QString::number(i.fallbackSeconds() - i.secondsSinceLastAttempted()) + "s") - .arg(i.secondsSinceLastAttempted()) - .arg(i.secondsSinceLastConnected()) + .arg(QString::fromStdString(i.peerEndpoint().address().to_string())) + .arg(i.id == web3()->id() ? "self" : sessions.count(i.id) ? sessions[i.id] : "disconnected") .arg(i.isOffline() ? " | " + QString::fromStdString(reasonOf(i.lastDisconnect)) + " | " + QString::number(i.failedAttempts) + "x" : "") .arg(i.rating) - .arg(0 /* (int)i.idOrigin */) ); } } diff --git a/libp2p/Host.h b/libp2p/Host.h index 1719bac6a..bffc6c39f 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -69,6 +69,7 @@ class Host; * Host of the responsibility. (&& remove save/restoreNodes) * @todo reimplement recording of historical session information on per-transport basis * @todo rebuild nodetable when localNetworking is enabled/disabled + * @todo move attributes into protected */ class Peer: public Node { @@ -79,11 +80,6 @@ public: bi::tcp::endpoint const& peerEndpoint() const { return endpoint.tcp; } -protected: - - /// Used by isOffline() and (todo) for peer to emit session information. - std::weak_ptr m_session; - int score = 0; ///< All time cumulative. int rating = 0; ///< Trending. @@ -93,6 +89,10 @@ protected: std::chrono::system_clock::time_point lastAttempted; unsigned failedAttempts = 0; DisconnectReason lastDisconnect = NoDisconnect; ///< Reason for disconnect that happened last. + +protected: + /// Used by isOffline() and (todo) for peer to emit session information. + std::weak_ptr m_session; }; using Peers = std::vector; @@ -260,7 +260,6 @@ private: std::set m_peerAddresses; ///< Public addresses that peers (can) know us by. - // Our capabilities. std::map> m_capabilities; ///< Each of the capabilities we support. std::chrono::steady_clock::time_point m_lastPing; ///< Time we sent the last ping to all peers. diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 242c1b325..40688f4a5 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -352,24 +352,24 @@ NodeTable::NodeBucket& NodeTable::bucket(NodeEntry const* _n) void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytesConstRef _packet) { - // h256 + Signature + RLP (smallest possible packet is empty neighbours packet which is 3 bytes) - if (_packet.size() < h256::size + Signature::size + 3) + // h256 + Signature + type + RLP (smallest possible packet is empty neighbours packet which is 3 bytes) + if (_packet.size() < h256::size + Signature::size + 1 + 3) { clog(NodeTableMessageSummary) << "Invalid Message size from " << _from.address().to_string() << ":" << _from.port(); return; } - bytesConstRef signedBytes(_packet.cropped(h256::size, _packet.size() - h256::size)); - h256 hashSigned(sha3(signedBytes)); + bytesConstRef hashedBytes(_packet.cropped(h256::size, _packet.size() - h256::size)); + h256 hashSigned(sha3(hashedBytes)); if (!_packet.cropped(0, h256::size).contentsEqual(hashSigned.asBytes())) { clog(NodeTableMessageSummary) << "Invalid Message hash from " << _from.address().to_string() << ":" << _from.port(); return; } + + bytesConstRef rlpBytes(hashedBytes.cropped(Signature::size, hashedBytes.size() - Signature::size)); - bytesConstRef rlpBytes(signedBytes.cropped(Signature::size, signedBytes.size() - Signature::size)); - RLP rlp(rlpBytes); - unsigned itemCount = rlp.itemCount(); + // todo: verify sig via known-nodeid and MDC, or, do ping/pong auth if node/endpoint is unknown/untrusted bytesConstRef sigBytes(_packet.cropped(h256::size, Signature::size)); Public nodeid(dev::recover(*(Signature const*)sigBytes.data(), sha3(rlpBytes))); @@ -378,8 +378,13 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes clog(NodeTableMessageSummary) << "Invalid Message signature from " << _from.address().to_string() << ":" << _from.port(); return; } - noteNode(nodeid, _from); + if (rlpBytes[0] && rlpBytes[0] < 4) + noteNode(nodeid, _from); + + // todo: switch packet-type + RLP rlp(rlpBytes.cropped(1, rlpBytes.size() - 1)); + unsigned itemCount = rlp.itemCount(); try { switch (itemCount) { @@ -435,14 +440,14 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes PingNode in = PingNode::fromBytesConstRef(_from, rlpBytes); Pong p(_from); - p.replyTo = sha3(rlpBytes); + p.echo = sha3(rlpBytes); p.sign(m_secret); m_socketPtr->send(p); break; } default: - clog(NodeTableMessageSummary) << "Invalid Message received from " << _from.address().to_string() << ":" << _from.port(); + clog(NodeTableMessageSummary) << "Invalid Message, " << std::hex << rlpBytes[0] << ", received from " << _from.address().to_string() << ":" << _from.port(); return; } } diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index dee10c7ae..acf735f74 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -122,7 +122,7 @@ public: NodeTable(ba::io_service& _io, KeyPair _alias, uint16_t _udpPort = 30303); ~NodeTable(); - /// Constants for Kademlia, mostly derived from address space. + /// Constants for Kademlia, derived from address space. static unsigned const s_addressByteSize = sizeof(NodeId); ///< Size of address type in bytes. static unsigned const s_bits = 8 * s_addressByteSize; ///< Denoted by n in [Kademlia]. @@ -269,6 +269,9 @@ struct PingNode: RLPXDatagram PingNode(bi::udp::endpoint _ep): RLPXDatagram(_ep) {} PingNode(bi::udp::endpoint _ep, std::string _src, uint16_t _srcPort, std::chrono::seconds _expiration = std::chrono::seconds(60)): RLPXDatagram(_ep), ipAddress(_src), port(_srcPort), expiration(futureFromEpoch(_expiration)) {} + uint8_t packetType() { return 1; } + + unsigned version = 1; std::string ipAddress; unsigned port; unsigned expiration; @@ -283,20 +286,17 @@ struct PingNode: RLPXDatagram * RLP Encoded Items: 1 * Minimum Encoded Size: 33 bytes * Maximum Encoded Size: 33 bytes - * - * @todo expiration - * @todo value of replyTo - * @todo create from PingNode (reqs RLPXDatagram verify flag) */ struct Pong: RLPXDatagram { Pong(bi::udp::endpoint _ep): RLPXDatagram(_ep) {} - h256 replyTo; // hash of rlp of PingNode + uint8_t packetType() { return 2; } + h256 echo; ///< MCD of PingNode unsigned expiration; - void streamRLP(RLPStream& _s) const { _s.appendList(1); _s << replyTo; } - void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); replyTo = (h256)r[0]; } + void streamRLP(RLPStream& _s) const { _s.appendList(1); _s << echo; } + void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); echo = (h256)r[0]; } }; /** @@ -317,6 +317,7 @@ struct FindNode: RLPXDatagram FindNode(bi::udp::endpoint _ep): RLPXDatagram(_ep) {} FindNode(bi::udp::endpoint _ep, NodeId _target, std::chrono::seconds _expiration = std::chrono::seconds(30)): RLPXDatagram(_ep), target(_target), expiration(futureFromEpoch(_expiration)) {} + uint8_t packetType() { return 3; } h512 target; unsigned expiration; @@ -329,8 +330,6 @@ struct FindNode: RLPXDatagram * * RLP Encoded Items: 2 (first item is list) * Minimum Encoded Size: 10 bytes - * - * @todo nonce: Should be replaced with expiration. */ struct Neighbours: RLPXDatagram { @@ -346,7 +345,7 @@ struct Neighbours: RLPXDatagram }; Neighbours(bi::udp::endpoint _ep): RLPXDatagram(_ep) {} - Neighbours(bi::udp::endpoint _to, std::vector> const& _nearest, unsigned _offset = 0, unsigned _limit = 0): RLPXDatagram(_to) + Neighbours(bi::udp::endpoint _to, std::vector> const& _nearest, unsigned _offset = 0, unsigned _limit = 0): RLPXDatagram(_to), expiration(futureFromEpoch(std::chrono::seconds(30))) { auto limit = _limit ? std::min(_nearest.size(), (size_t)(_offset + _limit)) : _nearest.size(); for (auto i = _offset; i < limit; i++) @@ -359,6 +358,7 @@ struct Neighbours: RLPXDatagram } } + uint8_t packetType() { return 4; } std::list nodes; unsigned expiration = 1; diff --git a/libp2p/UDP.cpp b/libp2p/UDP.cpp index b1f87e409..23a3531ac 100644 --- a/libp2p/UDP.cpp +++ b/libp2p/UDP.cpp @@ -25,9 +25,13 @@ using namespace dev::p2p; h256 RLPXDatagramFace::sign(Secret const& _k) { - RLPStream rlpstream; - streamRLP(rlpstream); - bytes rlpBytes(rlpstream.out()); + assert(packetType()); + + RLPStream rlpxstream; +// rlpxstream.appendRaw(toPublic(_k).asBytes()); // for mdc-based signature + rlpxstream.appendRaw(bytes(1, packetType())); + streamRLP(rlpxstream); + bytes rlpBytes(rlpxstream.out()); bytesConstRef rlp(&rlpBytes); h256 hash(dev::sha3(rlp)); @@ -35,12 +39,16 @@ h256 RLPXDatagramFace::sign(Secret const& _k) data.resize(h256::size + Signature::size + rlp.size()); bytesConstRef packetHash(&data[0], h256::size); + bytesConstRef signedPayload(&data[h256::size], Signature::size + rlp.size()); bytesConstRef payloadSig(&data[h256::size], Signature::size); bytesConstRef payload(&data[h256::size + Signature::size], rlp.size()); sig.ref().copyTo(payloadSig); +// rlp.cropped(Public::size, rlp.size() - Public::size).copyTo(payload); rlp.copyTo(payload); + +// hash.ref().copyTo(packetHash); // for mdc-based signature dev::sha3(signedPayload).ref().copyTo(packetHash); return std::move(hash); diff --git a/libp2p/UDP.h b/libp2p/UDP.h index d048e697c..9451e43f0 100644 --- a/libp2p/UDP.h +++ b/libp2p/UDP.h @@ -63,7 +63,8 @@ struct RLPXDatagramFace: public UDPDatagram static uint64_t futureFromEpoch(std::chrono::milliseconds _ms) { return std::chrono::duration_cast((std::chrono::system_clock::now() + _ms).time_since_epoch()).count(); } static uint64_t futureFromEpoch(std::chrono::seconds _sec) { return std::chrono::duration_cast((std::chrono::system_clock::now() + _sec).time_since_epoch()).count(); } static Public authenticate(bytesConstRef _sig, bytesConstRef _rlp); - + + virtual uint8_t packetType() =0; RLPXDatagramFace(bi::udp::endpoint const& _ep): UDPDatagram(_ep) {} virtual h256 sign(Secret const& _from); From 98a2d193c2d1abbee760e8856a7dda2ea61ab928 Mon Sep 17 00:00:00 2001 From: subtly Date: Sun, 25 Jan 2015 14:08:34 -0800 Subject: [PATCH 22/71] updates and fixes for code review --- libp2p/Common.h | 2 +- libp2p/Host.cpp | 62 +++++++++++++++++++++++++++---------------- libp2p/Host.h | 22 ++++++++++----- libp2p/NodeTable.cpp | 28 ++++++++++--------- libp2p/NodeTable.h | 26 +++++++++--------- libp2p/Session.cpp | 4 +-- libp2p/UDP.cpp | 31 ++++++++++------------ test/peer.cpp | 16 +++++------ test/whisperTopic.cpp | 28 +++++++++---------- 9 files changed, 120 insertions(+), 99 deletions(-) diff --git a/libp2p/Common.h b/libp2p/Common.h index 38012ec76..784b2f513 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -148,7 +148,7 @@ struct NodeIPEndpoint bi::udp::endpoint udp; bi::tcp::endpoint tcp; - operator bool() const { return udp.address().is_unspecified() && tcp.address().is_unspecified(); } + operator bool() const { return !udp.address().is_unspecified() || !tcp.address().is_unspecified(); } }; struct Node diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 38929278d..8c73a79f2 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -40,7 +40,7 @@ using namespace dev::p2p; HostNodeTableHandler::HostNodeTableHandler(Host& _host): m_host(_host) {} -void HostNodeTableHandler::processEvent(NodeId _n, NodeTableEventType _e) +void HostNodeTableHandler::processEvent(NodeId const& _n, NodeTableEventType const& _e) { m_host.onNodeTableEvent(_n, _e); } @@ -158,6 +158,8 @@ void Host::registerPeer(std::shared_ptr _s, CapDescs const& _caps) { { RecursiveGuard l(x_sessions); + // TODO: temporary loose-coupling; if m_peers already has peer, + // it is same as _s->m_peer. (fixing next PR) if (!m_peers.count(_s->m_peer->id)) m_peers[_s->m_peer->id] = _s->m_peer; m_sessions[_s->m_peer->id] = _s; @@ -172,7 +174,7 @@ void Host::registerPeer(std::shared_ptr _s, CapDescs const& _caps) } } -void Host::onNodeTableEvent(NodeId _n, NodeTableEventType _e) +void Host::onNodeTableEvent(NodeId const& _n, NodeTableEventType const& _e) { if (_e == NodeEntryAdded) @@ -320,6 +322,7 @@ void Host::runAcceptor() } } + // asio doesn't close socket on error if (!success && s->is_open()) { boost::system::error_code ec; @@ -393,25 +396,25 @@ void Host::addNode(NodeId const& _node, std::string const& _addr, unsigned short addNode(Node(_node, NodeIPEndpoint(bi::udp::endpoint(addr, _udpNodePort), bi::tcp::endpoint(addr, _tcpPeerPort)))); } -void Host::connect(std::shared_ptr const& _n) +void Host::connect(std::shared_ptr const& _p) { if (!m_run) return; - if (havePeerSession(_n->id)) + if (havePeerSession(_p->id)) { clog(NetWarn) << "Aborted connect. Node already connected."; return; } - if (!m_nodeTable->haveNode(_n->id)) + if (!m_nodeTable->haveNode(_p->id)) { clog(NetWarn) << "Aborted connect. Node not in node table."; return; } // prevent concurrently connecting to a node - Peer *nptr = _n.get(); + Peer *nptr = _p.get(); { Guard l(x_pendingNodeConns); if (m_pendingNodeConns.count(nptr)) @@ -419,22 +422,22 @@ void Host::connect(std::shared_ptr const& _n) m_pendingNodeConns.insert(nptr); } - clog(NetConnect) << "Attempting connection to node" << _n->id.abridged() << "@" << _n->peerEndpoint() << "from" << id().abridged(); + clog(NetConnect) << "Attempting connection to node" << _p->id.abridged() << "@" << _p->peerEndpoint() << "from" << id().abridged(); bi::tcp::socket* s = new bi::tcp::socket(m_ioService); - s->async_connect(_n->peerEndpoint(), [=](boost::system::error_code const& ec) + s->async_connect(_p->peerEndpoint(), [=](boost::system::error_code const& ec) { if (ec) { - clog(NetConnect) << "Connection refused to node" << _n->id.abridged() << "@" << _n->peerEndpoint() << "(" << ec.message() << ")"; - _n->lastDisconnect = TCPError; - _n->lastAttempted = std::chrono::system_clock::now(); + clog(NetConnect) << "Connection refused to node" << _p->id.abridged() << "@" << _p->peerEndpoint() << "(" << ec.message() << ")"; + _p->lastDisconnect = TCPError; + _p->lastAttempted = std::chrono::system_clock::now(); } else { - clog(NetConnect) << "Connected to" << _n->id.abridged() << "@" << _n->peerEndpoint(); + clog(NetConnect) << "Connected to" << _p->id.abridged() << "@" << _p->peerEndpoint(); - _n->lastConnected = std::chrono::system_clock::now(); - auto ps = make_shared(this, std::move(*s), _n); + _p->lastConnected = std::chrono::system_clock::now(); + auto ps = make_shared(this, std::move(*s), _p); ps->start(); } @@ -480,8 +483,8 @@ void Host::run(boost::system::error_code const&) if (auto pp = p.second.lock()) pp->serviceNodesRequest(); - if (chrono::steady_clock::now() - m_lastPing >= chrono::seconds(30)) // ping every 30s. - keepAlivePeers(); + disconnectLatePeers(); + keepAlivePeers(); auto runcb = [this](boost::system::error_code const& error) { run(error); }; m_timer->expires_from_now(boost::posix_time::milliseconds(c_timerInterval)); @@ -541,19 +544,29 @@ void Host::doWork() void Host::keepAlivePeers() { + if (chrono::steady_clock::now() < m_lastPing + c_keepAliveInterval) + return; + RecursiveGuard l(x_sessions); for (auto p: m_sessions) if (auto pp = p.second.lock()) - { - if (chrono::steady_clock::now() - pp->m_lastReceived >= chrono::seconds(60)) - pp->disconnect(PingTimeout); - else pp->ping(); - } m_lastPing = chrono::steady_clock::now(); } +void Host::disconnectLatePeers() +{ + if (chrono::steady_clock::now() < m_lastPing + c_keepAliveTimeOut) + return; + + RecursiveGuard l(x_sessions); + for (auto p: m_sessions) + if (auto pp = p.second.lock()) + if (pp->m_lastReceived < m_lastPing + c_keepAliveTimeOut) + pp->disconnect(PingTimeout); +} + bytes Host::saveNodes() const { RLPStream nodes; @@ -563,7 +576,9 @@ bytes Host::saveNodes() const for (auto const& i: m_peers) { Peer const& n = *(i.second); - // TODO: PoC-7: Figure out why it ever shares these ports.//n.address.port() >= 30300 && n.address.port() <= 30305 && + // TODO: alpha: Figure out why it ever shares these ports.//n.address.port() >= 30300 && n.address.port() <= 30305 && + // TODO: alpha: if/how to save private addresses + // Only save peers which have connected within 2 days, with properly-advertised port and public IP address if (chrono::system_clock::now() - n.lastConnected < chrono::seconds(3600 * 48) && n.peerEndpoint().port() > 0 && n.peerEndpoint().port() < /*49152*/32768 && n.id != id() && !isPrivateAddress(n.peerEndpoint().address())) { nodes.appendList(10); @@ -571,7 +586,8 @@ bytes Host::saveNodes() const nodes << n.peerEndpoint().address().to_v4().to_bytes(); else nodes << n.peerEndpoint().address().to_v6().to_bytes(); - nodes << n.peerEndpoint().port() << n.id /* << (int)n.idOrigin */ << 0 + // TODO: alpha: replace 0 with trust-state of node + nodes << n.peerEndpoint().port() << n.id << 0 << chrono::duration_cast(n.lastConnected.time_since_epoch()).count() << chrono::duration_cast(n.lastAttempted.time_since_epoch()).count() << n.failedAttempts << (unsigned)n.lastDisconnect << n.score << n.rating; diff --git a/libp2p/Host.h b/libp2p/Host.h index bffc6c39f..bb3e214c4 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -101,7 +101,7 @@ class HostNodeTableHandler: public NodeTableEventHandler { friend class Host; HostNodeTableHandler(Host& _host); - virtual void processEvent(NodeId _n, NodeTableEventType _e); + virtual void processEvent(NodeId const& _n, NodeTableEventType const& _e); Host& m_host; }; @@ -111,9 +111,11 @@ class HostNodeTableHandler: public NodeTableEventHandler * * @todo onNodeTableEvent: move peer-connection logic into ensurePeers * @todo handshake: gracefully disconnect peer if peer already connected + * @todo abstract socket -> IPConnection * @todo determinePublic: ipv6, udp * @todo handle conflict if addNode/requireNode called and Node already exists w/conflicting tcp or udp port * @todo write host identifier to disk w/nodes + * @todo per-session keepalive/ping instead of broadcast; set ping-timeout via median-latency */ class Host: public Worker { @@ -128,6 +130,12 @@ public: /// Will block on network process events. virtual ~Host(); + /// Interval at which Host::run will call keepAlivePeers to ping peers. + std::chrono::seconds const c_keepAliveInterval = std::chrono::seconds(30); + + /// Disconnect timeout after failure to respond to keepAlivePeers ping. + std::chrono::seconds const c_keepAliveTimeOut = std::chrono::seconds(1); + /// Default host for current version of client. static std::string pocHost(); @@ -141,9 +149,8 @@ public: CapDescs caps() const { CapDescs ret; for (auto const& i: m_capabilities) ret.push_back(i.first); return ret; } template std::shared_ptr cap() const { try { return std::static_pointer_cast(m_capabilities.at(std::make_pair(T::staticName(), T::staticVersion()))); } catch (...) { return nullptr; } } - bool havePeerSession(NodeId _id) { RecursiveGuard l(x_sessions); if (m_sessions.count(_id)) return !!m_sessions[_id].lock(); else return false; } + bool havePeerSession(NodeId _id) { RecursiveGuard l(x_sessions); return m_sessions.count(_id) ? !!m_sessions[_id].lock() : false; } - /// Add node. void addNode(NodeId const& _node, std::string const& _addr, unsigned short _tcpPort, unsigned short _udpPort); /// Set ideal number of peers. @@ -187,21 +194,24 @@ public: void registerPeer(std::shared_ptr _s, CapDescs const& _caps); protected: - void onNodeTableEvent(NodeId _n, NodeTableEventType _e); + void onNodeTableEvent(NodeId const& _n, NodeTableEventType const& _e); private: /// Populate m_peerAddresses with available public addresses. void determinePublic(std::string const& _publicAddress, bool _upnp); - void connect(std::shared_ptr const& _n); + void connect(std::shared_ptr const& _p); /// Ping the peers to update the latency information and disconnect peers which have timed out. void keepAlivePeers(); + /// Disconnect peers which didn't respond to keepAlivePeers ping prior to c_keepAliveTimeOut. + void disconnectLatePeers(); + /// Called only from startedWorking(). void runAcceptor(); - /// Handler for verifying handshake siganture before creating session. _nodeId is passed for outbound connections. + /// Handler for verifying handshake siganture before creating session. _nodeId is passed for outbound connections. If successful, socket is moved to Session via std::move. void doHandshake(bi::tcp::socket* _socket, NodeId _nodeId = NodeId()); void seal(bytes& _b); diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 40688f4a5..ae32cc81d 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -55,8 +55,8 @@ NodeTable::~NodeTable() void NodeTable::processEvents() { - if (m_nodeEvents) - m_nodeEvents->processEvents(); + if (m_nodeEventHandler) + m_nodeEventHandler->processEvents(); } shared_ptr NodeTable::addNode(Public const& _pubk, bi::udp::endpoint const& _udp, bi::tcp::endpoint const& _tcp) @@ -80,8 +80,8 @@ shared_ptr NodeTable::addNode(Node const& _node) else { clog(NodeTableNote) << "p2p.nodes.add " << _node.id.abridged(); - if (m_nodeEvents) - m_nodeEvents->appendEvent(_node.id, NodeEntryAdded); + if (m_nodeEventHandler) + m_nodeEventHandler->appendEvent(_node.id, NodeEntryAdded); ret.reset(new NodeEntry(m_node, _node.id, NodeIPEndpoint(_node.endpoint.udp, _node.endpoint.tcp))); m_nodes[_node.id] = ret; @@ -341,8 +341,8 @@ void NodeTable::dropNode(shared_ptr _n) } clog(NodeTableNote) << "p2p.nodes.drop " << _n->id.abridged(); - if (m_nodeEvents) - m_nodeEvents->appendEvent(_n->id, NodeEntryRemoved); + if (m_nodeEventHandler) + m_nodeEventHandler->appendEvent(_n->id, NodeEntryRemoved); } NodeTable::NodeBucket& NodeTable::bucket(NodeEntry const* _n) @@ -367,23 +367,25 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes return; } - bytesConstRef rlpBytes(hashedBytes.cropped(Signature::size, hashedBytes.size() - Signature::size)); + bytesConstRef signedBytes(hashedBytes.cropped(Signature::size, hashedBytes.size() - Signature::size)); // todo: verify sig via known-nodeid and MDC, or, do ping/pong auth if node/endpoint is unknown/untrusted bytesConstRef sigBytes(_packet.cropped(h256::size, Signature::size)); - Public nodeid(dev::recover(*(Signature const*)sigBytes.data(), sha3(rlpBytes))); + Public nodeid(dev::recover(*(Signature const*)sigBytes.data(), sha3(signedBytes))); if (!nodeid) { clog(NodeTableMessageSummary) << "Invalid Message signature from " << _from.address().to_string() << ":" << _from.port(); return; } - if (rlpBytes[0] && rlpBytes[0] < 4) + unsigned packetType = signedBytes[0]; + if (packetType && packetType < 4) noteNode(nodeid, _from); // todo: switch packet-type - RLP rlp(rlpBytes.cropped(1, rlpBytes.size() - 1)); + bytesConstRef rlpBytes(_packet.cropped(h256::size + Signature::size + 1)); + RLP rlp(rlpBytes); unsigned itemCount = rlp.itemCount(); try { switch (itemCount) @@ -404,7 +406,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes if (auto n = (*this)[it->first.first]) addNode(n); - m_evictions.erase(it); + it = m_evictions.erase(it); } break; @@ -447,7 +449,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes } default: - clog(NodeTableMessageSummary) << "Invalid Message, " << std::hex << rlpBytes[0] << ", received from " << _from.address().to_string() << ":" << _from.port(); + clog(NodeTableWarn) << "Invalid Message, " << hex << packetType << ", received from " << _from.address().to_string() << ":" << dec << _from.port(); return; } } @@ -456,7 +458,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes clog(NodeTableWarn) << "Exception processing message from " << _from.address().to_string() << ":" << _from.port(); } } - + void NodeTable::doCheckEvictions(boost::system::error_code const& _ec) { if (_ec || !m_socketPtr->isOpen()) diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index acf735f74..79adba62b 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -41,7 +41,7 @@ struct NodeEntry: public Node NodeEntry(Node _src, Public _pubk, NodeIPEndpoint _gw); NodeEntry(Node _src, Public _pubk, bi::udp::endpoint _udp); - const unsigned distance; ///< Node's distance (xor of _src as integer). + unsigned const distance; ///< Node's distance (xor of _src as integer). }; enum NodeTableEventType { @@ -53,20 +53,20 @@ class NodeTableEventHandler { friend class NodeTable; public: - virtual void processEvent(NodeId _n, NodeTableEventType _e) =0; + virtual void processEvent(NodeId const& _n, NodeTableEventType const& _e) =0; protected: /// Called by NodeTable on behalf of an implementation (Host) to process new events without blocking nodetable. void processEvents() { - std::list> events; + std::list> events; { Guard l(x_events); - if (!m_nodeEvents.size()) + if (!m_nodeEventHandler.size()) return; - m_nodeEvents.unique(); - for (auto const& n: m_nodeEvents) events.push_back(std::make_pair(n,m_events[n])); - m_nodeEvents.clear(); + m_nodeEventHandler.unique(); + for (auto const& n: m_nodeEventHandler) events.push_back(std::make_pair(n,m_events[n])); + m_nodeEventHandler.clear(); m_events.clear(); } for (auto const& e: events) @@ -74,11 +74,11 @@ protected: } /// Called by NodeTable to append event. - virtual void appendEvent(NodeId _n, NodeTableEventType _e) { Guard l(x_events); m_nodeEvents.push_back(_n); m_events[_n] = _e; } + virtual void appendEvent(NodeId _n, NodeTableEventType _e) { Guard l(x_events); m_nodeEventHandler.push_back(_n); m_events[_n] = _e; } Mutex x_events; - std::list m_nodeEvents; - std::map m_events; + std::list m_nodeEventHandler; + std::map m_events; }; /** @@ -143,7 +143,7 @@ public: static unsigned dist(NodeId const& _a, NodeId const& _b) { u512 d = _a ^ _b; unsigned ret; for (ret = 0; d >>= 1; ++ret) {}; return ret; } /// Set event handler for NodeEntryAdded and NodeEntryRemoved events. - void setEventHandler(NodeTableEventHandler* _handler) { m_nodeEvents.reset(_handler); } + void setEventHandler(NodeTableEventHandler* _handler) { m_nodeEventHandler.reset(_handler); } /// Called by implementation which provided handler to process NodeEntryAdded/NodeEntryRemoved events. Events are coalesced by type whereby old events are ignored. void processEvents(); @@ -160,7 +160,7 @@ public: std::list nodes() const; std::list state() const; - bool haveNode(NodeId _id) { Guard l(x_nodes); return !!m_nodes[_id]; } + bool haveNode(NodeId _id) { Guard l(x_nodes); return m_nodes.count(_id); } Node operator[](NodeId _id); std::shared_ptr getNodeEntry(NodeId _id); @@ -212,7 +212,7 @@ protected: /// Sends FindNeighbor packet. See doFindNode. void requestNeighbours(NodeEntry const& _node, NodeId _target) const; - std::unique_ptr m_nodeEvents; ///< Event handler for node events. + std::unique_ptr m_nodeEventHandler; ///< Event handler for node events. Node m_node; ///< This node. Secret m_secret; ///< This nodes secret key. diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index 9f68f3ef0..d23bf2d89 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -286,7 +286,7 @@ bool Session::interpret(RLP const& _r) clogS(NetAllDetail) << "Checking: " << ep << "(" << id.abridged() << ")"; // clogS(NetAllDetail) << "Checking: " << ep << "(" << id.abridged() << ")" << isPrivateAddress(peerAddress) << this->id().abridged() << isPrivateAddress(endpoint().address()) << m_server->m_peers.count(id) << (m_server->m_peers.count(id) ? isPrivateAddress(m_server->m_peers.at(id)->address.address()) : -1); - // ignore if dist(us,item) - dist(us,them) > 1 + // todo: draft spec: ignore if dist(us,item) - dist(us,them) > 1 // TODO: isPrivate if (!m_server->m_netPrefs.localNetworking && isPrivateAddress(peerAddress)) @@ -309,7 +309,7 @@ bool Session::interpret(RLP const& _r) // OK passed all our checks. Assume it's good. addRating(1000); - m_server->addNode(Node(id, NodeIPEndpoint(bi::udp::endpoint(ep.address(), 30303), ep))); + m_server->addNode(Node(id, NodeIPEndpoint(bi::udp::endpoint(ep.address(), ep.port()), ep))); clogS(NetTriviaDetail) << "New peer: " << ep << "(" << id .abridged()<< ")"; CONTINUE:; LAMEPEER:; diff --git a/libp2p/UDP.cpp b/libp2p/UDP.cpp index 23a3531ac..e27b6e57a 100644 --- a/libp2p/UDP.cpp +++ b/libp2p/UDP.cpp @@ -29,29 +29,26 @@ h256 RLPXDatagramFace::sign(Secret const& _k) RLPStream rlpxstream; // rlpxstream.appendRaw(toPublic(_k).asBytes()); // for mdc-based signature - rlpxstream.appendRaw(bytes(1, packetType())); + rlpxstream.appendRaw(bytes(1, packetType())); // prefix by 1 byte for type streamRLP(rlpxstream); - bytes rlpBytes(rlpxstream.out()); + bytes rlpxBytes(rlpxstream.out()); - bytesConstRef rlp(&rlpBytes); - h256 hash(dev::sha3(rlp)); - Signature sig = dev::sign(_k, hash); + bytesConstRef rlpx(&rlpxBytes); + h256 sighash(dev::sha3(rlpx)); // H(type||data) + Signature sig = dev::sign(_k, sighash); // S(H(type||data)) - data.resize(h256::size + Signature::size + rlp.size()); - bytesConstRef packetHash(&data[0], h256::size); - - bytesConstRef signedPayload(&data[h256::size], Signature::size + rlp.size()); - bytesConstRef payloadSig(&data[h256::size], Signature::size); - bytesConstRef payload(&data[h256::size + Signature::size], rlp.size()); + data.resize(h256::size + Signature::size + rlpx.size()); + bytesConstRef rlpxHash(&data[0], h256::size); + bytesConstRef rlpxSig(&data[h256::size], Signature::size); + bytesConstRef rlpxPayload(&data[h256::size + Signature::size], rlpx.size()); - sig.ref().copyTo(payloadSig); -// rlp.cropped(Public::size, rlp.size() - Public::size).copyTo(payload); - rlp.copyTo(payload); + sig.ref().copyTo(rlpxSig); + rlpx.copyTo(rlpxPayload); -// hash.ref().copyTo(packetHash); // for mdc-based signature - dev::sha3(signedPayload).ref().copyTo(packetHash); + bytesConstRef signedRLPx(&data[h256::size], data.size() - h256::size); + dev::sha3(signedRLPx).ref().copyTo(rlpxHash); - return std::move(hash); + return std::move(sighash); }; Public RLPXDatagramFace::authenticate(bytesConstRef _sig, bytesConstRef _rlp) diff --git a/test/peer.cpp b/test/peer.cpp index 51e777a56..5c3e677f6 100644 --- a/test/peer.cpp +++ b/test/peer.cpp @@ -51,10 +51,11 @@ BOOST_AUTO_TEST_SUITE_END() int peerTest(int argc, char** argv) { + Public remoteAlias; short listenPort = 30303; string remoteHost; short remotePort = 30303; - + for (int i = 1; i < argc; ++i) { string arg = argv[i]; @@ -64,21 +65,18 @@ int peerTest(int argc, char** argv) remoteHost = argv[++i]; else if (arg == "-p" && i + 1 < argc) remotePort = (short)atoi(argv[++i]); + else if (arg == "-ra" && i + 1 < argc) + remoteAlias = Public(dev::fromHex(argv[++i])); else remoteHost = argv[i]; } Host ph("Test", NetworkPreferences(listenPort)); - if (!remoteHost.empty()) - ph.addNode(NodeId(), remoteHost, remotePort, remotePort); + if (!remoteHost.empty() && !remoteAlias) + ph.addNode(remoteAlias, remoteHost, remotePort, remotePort); -// for (int i = 0; ; ++i) -// { -// this_thread::sleep_for(chrono::milliseconds(100)); -// if (!(i % 10)) -// ph.keepAlivePeers(); -// } + this_thread::sleep_for(chrono::milliseconds(200)); return 0; } diff --git a/test/whisperTopic.cpp b/test/whisperTopic.cpp index 9f0f34630..02fccbc9f 100644 --- a/test/whisperTopic.cpp +++ b/test/whisperTopic.cpp @@ -34,29 +34,27 @@ BOOST_AUTO_TEST_CASE(topic) { cnote << "Testing Whisper..."; auto oldLogVerbosity = g_logVerbosity; - g_logVerbosity = 0; + g_logVerbosity = 5; - Host ph1("Test", NetworkPreferences(30303, "127.0.0.1", true, true)); + Host phOther("Test", NetworkPreferences(30303, "127.0.0.1", true, true)); + auto whOther = phOther.registerCapability(new WhisperHost()); + phOther.start(); bool started = false; unsigned result = 0; std::thread listener([&]() { setThreadName("other"); - - auto wh = ph1.registerCapability(new WhisperHost()); - ph1.start(); - started = true; /// Only interested in odd packets - auto w = wh->installWatch(BuildTopicMask("odd")); + auto w = whOther->installWatch(BuildTopicMask("odd")); for (int i = 0, last = 0; i < 200 && last < 81; ++i) { - for (auto i: wh->checkWatch(w)) + for (auto i: whOther->checkWatch(w)) { - Message msg = wh->envelope(i).open(); + Message msg = whOther->envelope(i).open(); last = RLP(msg.payload()).toInt(); cnote << "New message from:" << msg.from().abridged() << RLP(msg.payload()).toInt(); result += last; @@ -65,16 +63,16 @@ BOOST_AUTO_TEST_CASE(topic) } }); - while (!started) - this_thread::sleep_for(chrono::milliseconds(50)); - Host ph("Test", NetworkPreferences(30300, "127.0.0.1", true, true)); auto wh = ph.registerCapability(new WhisperHost()); - this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); - this_thread::sleep_for(chrono::milliseconds(500)); - ph.addNode(ph1.id(), "127.0.0.1", 30303, 30303); + ph.addNode(phOther.id(), "127.0.0.1", 30303, 30303); + this_thread::sleep_for(chrono::milliseconds(300)); + while (!started) + this_thread::sleep_for(chrono::milliseconds(25)); + + KeyPair us = KeyPair::create(); for (int i = 0; i < 10; ++i) { From 6381706c665ffae061c847c7a309b18167e32ad4 Mon Sep 17 00:00:00 2001 From: subtly Date: Sun, 25 Jan 2015 19:54:15 -0800 Subject: [PATCH 23/71] fix ping-timeouts --- libp2p/Host.cpp | 12 +++++++----- libp2p/Host.h | 2 +- libp2p/Session.cpp | 3 ++- test/peer.cpp | 2 +- test/whisperTopic.cpp | 22 +++++++++++++--------- 5 files changed, 24 insertions(+), 17 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 8c73a79f2..11320e463 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -52,7 +52,8 @@ Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n, bool m_ifAddresses(Network::getInterfaceAddresses()), m_ioService(2), m_tcp4Acceptor(m_ioService), - m_alias(move(getHostIdentifier())) + m_alias(move(getHostIdentifier())), + m_lastPing(chrono::time_point::min()) { for (auto address: m_ifAddresses) if (address.is_v4()) @@ -483,8 +484,8 @@ void Host::run(boost::system::error_code const&) if (auto pp = p.second.lock()) pp->serviceNodesRequest(); - disconnectLatePeers(); keepAlivePeers(); + disconnectLatePeers(); auto runcb = [this](boost::system::error_code const& error) { run(error); }; m_timer->expires_from_now(boost::posix_time::milliseconds(c_timerInterval)); @@ -544,7 +545,7 @@ void Host::doWork() void Host::keepAlivePeers() { - if (chrono::steady_clock::now() < m_lastPing + c_keepAliveInterval) + if (chrono::steady_clock::now() - c_keepAliveInterval < m_lastPing) return; RecursiveGuard l(x_sessions); @@ -557,13 +558,14 @@ void Host::keepAlivePeers() void Host::disconnectLatePeers() { - if (chrono::steady_clock::now() < m_lastPing + c_keepAliveTimeOut) + auto now = chrono::steady_clock::now(); + if (now - c_keepAliveTimeOut < m_lastPing) return; RecursiveGuard l(x_sessions); for (auto p: m_sessions) if (auto pp = p.second.lock()) - if (pp->m_lastReceived < m_lastPing + c_keepAliveTimeOut) + if (now - c_keepAliveTimeOut > m_lastPing && pp->m_lastReceived < m_lastPing) pp->disconnect(PingTimeout); } diff --git a/libp2p/Host.h b/libp2p/Host.h index bb3e214c4..d770b81b1 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -134,7 +134,7 @@ public: std::chrono::seconds const c_keepAliveInterval = std::chrono::seconds(30); /// Disconnect timeout after failure to respond to keepAlivePeers ping. - std::chrono::seconds const c_keepAliveTimeOut = std::chrono::seconds(1); + std::chrono::milliseconds const c_keepAliveTimeOut = std::chrono::milliseconds(1000); /// Default host for current version of client. static std::string pocHost(); diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index d23bf2d89..7e618fb3b 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -40,7 +40,8 @@ Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr const& m_server(_s), m_socket(std::move(_socket)), m_peer(_n), - m_info({NodeId(), "?", m_socket.remote_endpoint().address().to_string(), 0, std::chrono::steady_clock::duration(0), CapDescSet(), 0, map()}) + m_info({NodeId(), "?", m_socket.remote_endpoint().address().to_string(), 0, std::chrono::steady_clock::duration(0), CapDescSet(), 0, map()}), + m_ping(chrono::time_point::max()) { m_lastReceived = m_connect = std::chrono::steady_clock::now(); } diff --git a/test/peer.cpp b/test/peer.cpp index 5c3e677f6..7c128276e 100644 --- a/test/peer.cpp +++ b/test/peer.cpp @@ -44,7 +44,7 @@ BOOST_AUTO_TEST_CASE(host) host1.addNode(node2, "127.0.0.1", host2prefs.listenPort, host2prefs.listenPort); - this_thread::sleep_for(chrono::seconds(1)); + this_thread::sleep_for(chrono::seconds(3)); } BOOST_AUTO_TEST_SUITE_END() diff --git a/test/whisperTopic.cpp b/test/whisperTopic.cpp index 02fccbc9f..eccdf16c0 100644 --- a/test/whisperTopic.cpp +++ b/test/whisperTopic.cpp @@ -34,12 +34,21 @@ BOOST_AUTO_TEST_CASE(topic) { cnote << "Testing Whisper..."; auto oldLogVerbosity = g_logVerbosity; - g_logVerbosity = 5; + g_logVerbosity = 0; - Host phOther("Test", NetworkPreferences(30303, "127.0.0.1", true, true)); + Host phOther("Test", NetworkPreferences(30303, "127.0.0.1", false, true)); auto whOther = phOther.registerCapability(new WhisperHost()); phOther.start(); + Host ph("Test", NetworkPreferences(30300, "127.0.0.1", false, true)); + auto wh = ph.registerCapability(new WhisperHost()); + ph.start(); + + this_thread::sleep_for(chrono::milliseconds(100)); + ph.addNode(phOther.id(), "127.0.0.1", 30303, 30303); + + this_thread::sleep_for(chrono::milliseconds(500)); + bool started = false; unsigned result = 0; std::thread listener([&]() @@ -61,16 +70,11 @@ BOOST_AUTO_TEST_CASE(topic) } this_thread::sleep_for(chrono::milliseconds(50)); } + }); - Host ph("Test", NetworkPreferences(30300, "127.0.0.1", true, true)); - auto wh = ph.registerCapability(new WhisperHost()); - ph.start(); - ph.addNode(phOther.id(), "127.0.0.1", 30303, 30303); - - this_thread::sleep_for(chrono::milliseconds(300)); while (!started) - this_thread::sleep_for(chrono::milliseconds(25)); + this_thread::sleep_for(chrono::milliseconds(1)); KeyPair us = KeyPair::create(); From 2e7ea3564c10e498fdf8bbb3054c722b8f7845ac Mon Sep 17 00:00:00 2001 From: subtly Date: Sun, 25 Jan 2015 21:43:53 -0800 Subject: [PATCH 24/71] add test-require to p2p/host --- libp2p/Host.cpp | 2 +- test/peer.cpp | 5 ++++- test/whisperTopic.cpp | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 11320e463..5bf0b937d 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -626,7 +626,7 @@ void Host::restoreNodes(bytesConstRef _b) auto id = (NodeId)i[2]; if (!m_peers.count(id)) { - // TODO: p2p Important :) + // TODO: p2p import/export :) // auto n = noteNode(id, ep); // n->lastConnected = chrono::system_clock::time_point(chrono::seconds(i[4].toInt())); // n->lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[5].toInt())); diff --git a/test/peer.cpp b/test/peer.cpp index 7c128276e..a4b07e0b3 100644 --- a/test/peer.cpp +++ b/test/peer.cpp @@ -44,7 +44,10 @@ BOOST_AUTO_TEST_CASE(host) host1.addNode(node2, "127.0.0.1", host2prefs.listenPort, host2prefs.listenPort); - this_thread::sleep_for(chrono::seconds(3)); + this_thread::sleep_for(chrono::seconds(1)); + + BOOST_REQUIRE_EQUAL(host1.peerCount(), 1); + BOOST_REQUIRE_EQUAL(host2.peerCount(), host1.peerCount()); } BOOST_AUTO_TEST_SUITE_END() diff --git a/test/whisperTopic.cpp b/test/whisperTopic.cpp index eccdf16c0..5bd8f0f88 100644 --- a/test/whisperTopic.cpp +++ b/test/whisperTopic.cpp @@ -75,7 +75,6 @@ BOOST_AUTO_TEST_CASE(topic) while (!started) this_thread::sleep_for(chrono::milliseconds(1)); - KeyPair us = KeyPair::create(); for (int i = 0; i < 10; ++i) From 5566f335e8671830bad3a572ab9d80aabb7bc08c Mon Sep 17 00:00:00 2001 From: subtly Date: Mon, 26 Jan 2015 10:02:08 -0800 Subject: [PATCH 25/71] Comments for socket in acceptor. Remove start arg from Host constructor; it is not used and conflicts with restoreNodes being used to set node credentials. --- libp2p/Host.cpp | 26 +++++++++++++++++++++----- libp2p/Host.h | 2 +- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 5bf0b937d..f2f2b6a89 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -45,7 +45,7 @@ void HostNodeTableHandler::processEvent(NodeId const& _n, NodeTableEventType con m_host.onNodeTableEvent(_n, _e); } -Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n, bool _start): +Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n): Worker("p2p", 0), m_clientVersion(_clientVersion), m_netPrefs(_n), @@ -60,8 +60,6 @@ Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n, bool clog(NetNote) << "IP Address: " << address << " = " << (isPrivateAddress(address) ? "[LOCAL]" : "[PEER]"); clog(NetNote) << "Id:" << id(); - if (_start) - start(); } Host::~Host() @@ -300,16 +298,34 @@ void Host::runAcceptor() { clog(NetConnect) << "Listening on local port " << m_listenPort << " (public: " << m_tcpPublic << ")"; m_accepting = true; + + // socket is created outside of acceptor-callback + // An allocated socket is necessary as asio can use the socket + // until the callback succeeds or fails. + // + // Until callback succeeds or fails, we can't dealloc it. + // + // Callback is guaranteed to be called via asio or when + // m_tcp4Acceptor->stop() is called by Host. + // + // All exceptions are caught so they don't halt asio and so the + // socket is deleted. + // + // It's possible for an accepted connection to return an error in which + // case the socket may be open and must be closed to prevent asio from + // processing socket events after socket is deallocated. - bi::tcp::socket* s = new bi::tcp::socket(m_ioService); + bi::tcp::socket *s = new bi::tcp::socket(m_ioService); m_tcp4Acceptor.async_accept(*s, [=](boost::system::error_code ec) { + // if no error code, doHandshake takes ownership bool success = false; if (!ec) { try { - // incoming connection so we don't yet know nodeid + // doHandshake takes ownersihp of *s via std::move + // incoming connection; we don't yet know nodeid doHandshake(s, NodeId()); success = true; } diff --git a/libp2p/Host.h b/libp2p/Host.h index d770b81b1..a4493f9bf 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -125,7 +125,7 @@ class Host: public Worker public: /// Start server, listening for connections on the given port. - Host(std::string const& _clientVersion, NetworkPreferences const& _n = NetworkPreferences(), bool _start = false); + Host(std::string const& _clientVersion, NetworkPreferences const& _n = NetworkPreferences()); /// Will block on network process events. virtual ~Host(); From 2f3ab3a92d286727ce30847f5646048ed23819ef Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 29 Jan 2015 10:53:30 -0800 Subject: [PATCH 26/71] add expiration to pong --- libp2p/NodeTable.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 79adba62b..7f9e3877f 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -88,6 +88,9 @@ protected: * Thread-safety is ensured by modifying NodeEntry details via * shared_ptr replacement instead of mutating values. * + * NodeTable accepts a port for UDP and will listen to the port on all available + * interfaces. + * * [Integration] * @todo restore nodes: affects refreshbuckets * @todo TCP endpoints @@ -289,7 +292,7 @@ struct PingNode: RLPXDatagram */ struct Pong: RLPXDatagram { - Pong(bi::udp::endpoint _ep): RLPXDatagram(_ep) {} + Pong(bi::udp::endpoint _ep): RLPXDatagram(_ep), expiration(futureFromEpoch(std::chrono::seconds(60))) {} uint8_t packetType() { return 2; } h256 echo; ///< MCD of PingNode From 3d2e72ce774a70bc2b8913d14ad6c0a42db8a577 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 30 Jan 2015 00:50:28 +0100 Subject: [PATCH 27/71] libp2p: distinguish discovery packets by packet type --- libp2p/NodeTable.cpp | 52 +++++++++++++++++++++----------------------- libp2p/NodeTable.h | 10 +++++---- libp2p/UDP.h | 1 + 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index ae32cc81d..dffa0933b 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -383,14 +383,12 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes if (packetType && packetType < 4) noteNode(nodeid, _from); - // todo: switch packet-type bytesConstRef rlpBytes(_packet.cropped(h256::size + Signature::size + 1)); RLP rlp(rlpBytes); - unsigned itemCount = rlp.itemCount(); try { - switch (itemCount) + switch (packetType) { - case 1: + case Pong::type: { // clog(NodeTableMessageSummary) << "Received Pong from " << _from.address().to_string() << ":" << _from.port(); Pong in = Pong::fromBytesConstRef(_from, rlpBytes); @@ -408,35 +406,35 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes it = m_evictions.erase(it); } - break; } - case 2: - if (rlp[0].isList()) - { - Neighbours in = Neighbours::fromBytesConstRef(_from, rlpBytes); -// clog(NodeTableMessageSummary) << "Received " << in.nodes.size() << " Neighbours from " << _from.address().to_string() << ":" << _from.port(); - for (auto n: in.nodes) - noteNode(n.node, bi::udp::endpoint(bi::address::from_string(n.ipAddress), n.port)); - } - else + case Neighbours::type: + { + Neighbours in = Neighbours::fromBytesConstRef(_from, rlpBytes); +// clog(NodeTableMessageSummary) << "Received " << in.nodes.size() << " Neighbours from " << _from.address().to_string() << ":" << _from.port(); + for (auto n: in.nodes) + noteNode(n.node, bi::udp::endpoint(bi::address::from_string(n.ipAddress), n.port)); + break; + } + + case FindNode::type: + { +// clog(NodeTableMessageSummary) << "Received FindNode from " << _from.address().to_string() << ":" << _from.port(); + FindNode in = FindNode::fromBytesConstRef(_from, rlpBytes); + + vector> nearest = findNearest(in.target); + static unsigned const nlimit = (m_socketPtr->maxDatagramSize - 11) / 86; + for (unsigned offset = 0; offset < nearest.size(); offset += nlimit) { -// clog(NodeTableMessageSummary) << "Received FindNode from " << _from.address().to_string() << ":" << _from.port(); - FindNode in = FindNode::fromBytesConstRef(_from, rlpBytes); - - vector> nearest = findNearest(in.target); - static unsigned const nlimit = (m_socketPtr->maxDatagramSize - 11) / 86; - for (unsigned offset = 0; offset < nearest.size(); offset += nlimit) - { - Neighbours out(_from, nearest, offset, nlimit); - out.sign(m_secret); - m_socketPtr->send(out); - } + Neighbours out(_from, nearest, offset, nlimit); + out.sign(m_secret); + m_socketPtr->send(out); } break; - - case 3: + } + + case PingNode::type: { // clog(NodeTableMessageSummary) << "Received PingNode from " << _from.address().to_string() << ":" << _from.port(); PingNode in = PingNode::fromBytesConstRef(_from, rlpBytes); diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 7f9e3877f..61241a333 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -272,7 +272,7 @@ struct PingNode: RLPXDatagram PingNode(bi::udp::endpoint _ep): RLPXDatagram(_ep) {} PingNode(bi::udp::endpoint _ep, std::string _src, uint16_t _srcPort, std::chrono::seconds _expiration = std::chrono::seconds(60)): RLPXDatagram(_ep), ipAddress(_src), port(_srcPort), expiration(futureFromEpoch(_expiration)) {} - uint8_t packetType() { return 1; } + static const uint8_t type = 1; unsigned version = 1; std::string ipAddress; @@ -294,7 +294,8 @@ struct Pong: RLPXDatagram { Pong(bi::udp::endpoint _ep): RLPXDatagram(_ep), expiration(futureFromEpoch(std::chrono::seconds(60))) {} - uint8_t packetType() { return 2; } + static const uint8_t type = 2; + h256 echo; ///< MCD of PingNode unsigned expiration; @@ -319,8 +320,9 @@ struct FindNode: RLPXDatagram { FindNode(bi::udp::endpoint _ep): RLPXDatagram(_ep) {} FindNode(bi::udp::endpoint _ep, NodeId _target, std::chrono::seconds _expiration = std::chrono::seconds(30)): RLPXDatagram(_ep), target(_target), expiration(futureFromEpoch(_expiration)) {} + + static const uint8_t type = 3; - uint8_t packetType() { return 3; } h512 target; unsigned expiration; @@ -361,7 +363,7 @@ struct Neighbours: RLPXDatagram } } - uint8_t packetType() { return 4; } + static const uint8_t type = 4; std::list nodes; unsigned expiration = 1; diff --git a/libp2p/UDP.h b/libp2p/UDP.h index 9451e43f0..cc0dca330 100644 --- a/libp2p/UDP.h +++ b/libp2p/UDP.h @@ -77,6 +77,7 @@ struct RLPXDatagram: public RLPXDatagramFace { RLPXDatagram(bi::udp::endpoint const& _ep): RLPXDatagramFace(_ep) {} static T fromBytesConstRef(bi::udp::endpoint const& _ep, bytesConstRef _bytes) { T t(_ep); t.interpretRLP(_bytes); return std::move(t); } + uint8_t packetType() { return T::type; } }; /** From a1911f34952c81c37101680e8a95d720b4ca2398 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 29 Jan 2015 22:29:03 +0100 Subject: [PATCH 28/71] libp2p: add expiration to serialized Pong packets --- libp2p/NodeTable.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 61241a333..8127c3597 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -286,7 +286,7 @@ struct PingNode: RLPXDatagram /** * Pong packet: response to ping * - * RLP Encoded Items: 1 + * RLP Encoded Items: 2 * Minimum Encoded Size: 33 bytes * Maximum Encoded Size: 33 bytes */ @@ -299,8 +299,8 @@ struct Pong: RLPXDatagram h256 echo; ///< MCD of PingNode unsigned expiration; - void streamRLP(RLPStream& _s) const { _s.appendList(1); _s << echo; } - void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); echo = (h256)r[0]; } + void streamRLP(RLPStream& _s) const { _s.appendList(2); _s << echo << expiration; } + void interpretRLP(bytesConstRef _bytes) { RLP r(_bytes); echo = (h256)r[0]; expiration = r[1].toInt(); } }; /** From 683a5da114c191c2b46b9355bbe001ba12acf2e7 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 30 Jan 2015 00:51:30 +0100 Subject: [PATCH 29/71] libp2p: use actual unix timestamps for discover packets --- libp2p/UDP.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libp2p/UDP.h b/libp2p/UDP.h index cc0dca330..bf9a6a372 100644 --- a/libp2p/UDP.h +++ b/libp2p/UDP.h @@ -60,8 +60,8 @@ protected: */ struct RLPXDatagramFace: public UDPDatagram { - static uint64_t futureFromEpoch(std::chrono::milliseconds _ms) { return std::chrono::duration_cast((std::chrono::system_clock::now() + _ms).time_since_epoch()).count(); } - static uint64_t futureFromEpoch(std::chrono::seconds _sec) { return std::chrono::duration_cast((std::chrono::system_clock::now() + _sec).time_since_epoch()).count(); } + static uint64_t futureFromEpoch(std::chrono::milliseconds _ms) { return std::chrono::duration_cast((std::chrono::system_clock::now() + _ms).time_since_epoch()).count(); } + static uint64_t futureFromEpoch(std::chrono::seconds _sec) { return std::chrono::duration_cast((std::chrono::system_clock::now() + _sec).time_since_epoch()).count(); } static Public authenticate(bytesConstRef _sig, bytesConstRef _rlp); virtual uint8_t packetType() =0; From 92f9ec8c42fece1c133ec89d2a33e0683df76da8 Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 30 Jan 2015 17:53:27 -0800 Subject: [PATCH 30/71] import/export peers and nodes --- alethzero/MainWin.cpp | 13 +-- alethzero/MainWin.h | 2 +- eth/main.cpp | 10 +- libp2p/Common.h | 5 +- libp2p/Host.cpp | 198 +++++++++++++++++++++------------------ libp2p/Host.h | 53 ++++++++--- libp2p/NodeTable.h | 3 +- libp2p/Session.cpp | 4 +- libp2p/Session.h | 2 +- libwebthree/WebThree.cpp | 13 +-- libwebthree/WebThree.h | 13 +-- neth/main.cpp | 9 +- test/net.cpp | 11 ++- test/peer.cpp | 37 ++++++++ third/MainWin.cpp | 19 ++-- third/MainWin.h | 2 +- 16 files changed, 235 insertions(+), 159 deletions(-) diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index 33089244d..55c32fb8e 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -660,10 +660,10 @@ void Main::writeSettings() s.setValue("verbosity", ui->verbosity->value()); s.setValue("jitvm", ui->jitvm->isChecked()); - bytes d = m_webThree->saveNodes(); + bytes d = m_webThree->saveNetwork(); if (d.size()) - m_peers = QByteArray((char*)d.data(), (int)d.size()); - s.setValue("peers", m_peers); + m_networkConfig = QByteArray((char*)d.data(), (int)d.size()); + s.setValue("peers", m_networkConfig); s.setValue("nameReg", ui->nameReg->text()); s.setValue("geometry", saveGeometry()); @@ -712,7 +712,7 @@ void Main::readSettings(bool _skipGeometry) } } - m_peers = s.value("peers").toByteArray(); + m_networkConfig = s.value("peers").toByteArray(); ui->upnp->setChecked(s.value("upnp", true).toBool()); ui->forceAddress->setText(s.value("forceAddress", "").toString()); ui->usePast->setChecked(s.value("usePast", true).toBool()); @@ -1859,8 +1859,9 @@ void Main::on_net_triggered() web3()->setIdealPeerCount(ui->idealPeers->value()); web3()->setNetworkPreferences(netPrefs()); ethereum()->setNetworkId(m_privateChain.size() ? sha3(m_privateChain.toStdString()) : 0); - if (m_peers.size()/* && ui->usePast->isChecked()*/) - web3()->restoreNodes(bytesConstRef((byte*)m_peers.data(), m_peers.size())); + // TODO: p2p + if (m_networkConfig.size()/* && ui->usePast->isChecked()*/) + web3()->restoreNetwork(bytesConstRef((byte*)m_networkConfig.data(), m_networkConfig.size())); web3()->startNetwork(); ui->downloadView->setDownloadMan(ethereum()->downloadMan()); } diff --git a/alethzero/MainWin.h b/alethzero/MainWin.h index b5639fc68..4e21d493f 100644 --- a/alethzero/MainWin.h +++ b/alethzero/MainWin.h @@ -251,7 +251,7 @@ private: unsigned m_currenciesFilter = (unsigned)-1; unsigned m_balancesFilter = (unsigned)-1; - QByteArray m_peers; + QByteArray m_networkConfig; QStringList m_servers; QList m_myKeys; QList m_myIdentities; diff --git a/eth/main.cpp b/eth/main.cpp index 515197ddc..f3a6a3941 100644 --- a/eth/main.cpp +++ b/eth/main.cpp @@ -332,13 +332,14 @@ int main(int argc, char** argv) VMFactory::setKind(jit ? VMKind::JIT : VMKind::Interpreter); NetworkPreferences netPrefs(listenPort, publicIP, upnp, useLocal); + auto nodesState = contents((dbPath.size() ? dbPath : getDataDir()) + "/network.rlp"); dev::WebThreeDirect web3( "Ethereum(++)/" + clientName + "v" + dev::Version + "/" DEV_QUOTED(ETH_BUILD_TYPE) "/" DEV_QUOTED(ETH_BUILD_PLATFORM) + (jit ? "/JIT" : ""), dbPath, false, mode == NodeMode::Full ? set{"eth", "shh"} : set(), - netPrefs - ); + netPrefs, + &nodesState); web3.setIdealPeerCount(peers); eth::Client* c = mode == NodeMode::Full ? web3.ethereum() : nullptr; @@ -348,9 +349,6 @@ int main(int argc, char** argv) c->setAddress(coinbase); } - auto nodesState = contents((dbPath.size() ? dbPath : getDataDir()) + "/nodeState.rlp"); - web3.restoreNodes(&nodesState); - cout << "Address: " << endl << toHex(us.address().asArray()) << endl; web3.startNetwork(); @@ -824,7 +822,7 @@ int main(int argc, char** argv) while (!g_exit) this_thread::sleep_for(chrono::milliseconds(1000)); - writeFile((dbPath.size() ? dbPath : getDataDir()) + "/nodeState.rlp", web3.saveNodes()); + writeFile((dbPath.size() ? dbPath : getDataDir()) + "/network.rlp", web3.saveNetwork()); return 0; } diff --git a/libp2p/Common.h b/libp2p/Common.h index 784b2f513..19cf447ce 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -32,6 +32,7 @@ #include #include #include +#include namespace ba = boost::asio; namespace bi = boost::asio::ip; @@ -54,6 +55,8 @@ class Capability; class Host; class Session; +struct NetworkStartRequired: virtual dev::Exception {}; + struct NetWarn: public LogChannel { static const char* name() { return "!N!"; } static const int verbosity = 0; }; struct NetNote: public LogChannel { static const char* name() { return "*N*"; } static const int verbosity = 1; }; struct NetMessageSummary: public LogChannel { static const char* name() { return "-N-"; } static const int verbosity = 2; }; @@ -168,7 +171,7 @@ struct Node /// If true, node will not be removed from Node list. bool required = false; - operator bool() const { return (bool)id; } + virtual operator bool() const { return (bool)id; } }; } diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index f2f2b6a89..a526db259 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -45,14 +45,15 @@ void HostNodeTableHandler::processEvent(NodeId const& _n, NodeTableEventType con m_host.onNodeTableEvent(_n, _e); } -Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n): +Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n, bytesConstRef _restoreNetwork): Worker("p2p", 0), + m_restoreNetwork(_restoreNetwork.toBytes()), m_clientVersion(_clientVersion), m_netPrefs(_n), m_ifAddresses(Network::getInterfaceAddresses()), m_ioService(2), m_tcp4Acceptor(m_ioService), - m_alias(move(getHostIdentifier())), + m_alias(getNetworkAlias(_restoreNetwork)), m_lastPing(chrono::time_point::min()) { for (auto address: m_ifAddresses) @@ -124,7 +125,7 @@ void Host::doneWorking() RecursiveGuard l(x_sessions); for (auto i: m_sessions) if (auto p = i.second.lock()) - if (p->isOpen()) + if (p->isConnected()) { p->disconnect(ClientQuit); n++; @@ -156,6 +157,7 @@ unsigned Host::protocolVersion() const void Host::registerPeer(std::shared_ptr _s, CapDescs const& _caps) { { + clog(NetNote) << "p2p.host.peer.register" << _s->m_peer->id.abridged(); RecursiveGuard l(x_sessions); // TODO: temporary loose-coupling; if m_peers already has peer, // it is same as _s->m_peer. (fixing next PR) @@ -385,8 +387,6 @@ string Host::pocHost() void Host::addNode(NodeId const& _node, std::string const& _addr, unsigned short _tcpPeerPort, unsigned short _udpNodePort) { - - if (_tcpPeerPort < 30300 || _tcpPeerPort > 30305) cwarn << "Non-standard port being recorded: " << _tcpPeerPort; @@ -406,11 +406,11 @@ void Host::addNode(NodeId const& _node, std::string const& _addr, unsigned short if (_ec) return; bi::tcp::endpoint tcp = *_epIt; - addNode(Node(_node, NodeIPEndpoint(bi::udp::endpoint(tcp.address(), _udpNodePort), tcp))); + if (m_nodeTable) m_nodeTable->addNode(Node(_node, NodeIPEndpoint(bi::udp::endpoint(tcp.address(), _udpNodePort), tcp))); }); } else - addNode(Node(_node, NodeIPEndpoint(bi::udp::endpoint(addr, _udpNodePort), bi::tcp::endpoint(addr, _tcpPeerPort)))); + if (m_nodeTable) m_nodeTable->addNode(Node(_node, NodeIPEndpoint(bi::udp::endpoint(addr, _udpNodePort), bi::tcp::endpoint(addr, _tcpPeerPort)))); } void Host::connect(std::shared_ptr const& _p) @@ -473,11 +473,22 @@ PeerSessionInfos Host::peers() const RecursiveGuard l(x_sessions); for (auto& i: m_sessions) if (auto j = i.second.lock()) - if (j->m_socket.is_open()) + if (j->isConnected()) ret.push_back(j->m_info); return ret; } +size_t Host::peerCount() const +{ + unsigned retCount = 0; + RecursiveGuard l(x_sessions); + for (auto& i: m_sessions) + if (std::shared_ptr j = i.second.lock()) + if (j->isConnected()) + retCount++; + return retCount; +} + void Host::run(boost::system::error_code const&) { if (!m_run) @@ -537,16 +548,14 @@ void Host::startedWorking() if (m_listenPort > 0) runAcceptor(); - - if (!m_tcpPublic.address().is_unspecified()) - // TODO: add m_tcpPublic endpoint; sort out endpoint stuff for nodetable - m_nodeTable.reset(new NodeTable(m_ioService, m_alias, m_listenPort)); - else - m_nodeTable.reset(new NodeTable(m_ioService, m_alias, m_listenPort > 0 ? m_listenPort : 30303)); - m_nodeTable->setEventHandler(new HostNodeTableHandler(*this)); } else - clog(NetNote) << "p2p.start.notice id:" << id().abridged() << "Invalid listen-port. Node Table Disabled."; + clog(NetNote) << "p2p.start.notice id:" << id().abridged() << "Listen port is invalid or unavailable. Node Table using default port (30303)."; + + // TODO: add m_tcpPublic endpoint; sort out endpoint stuff for nodetable + m_nodeTable.reset(new NodeTable(m_ioService, m_alias, m_listenPort > 0 ? m_listenPort : 30303)); + m_nodeTable->setEventHandler(new HostNodeTableHandler(*this)); + restoreNetwork(&m_restoreNetwork); clog(NetNote) << "p2p.started id:" << id().abridged(); @@ -585,112 +594,117 @@ void Host::disconnectLatePeers() pp->disconnect(PingTimeout); } -bytes Host::saveNodes() const +bytes Host::saveNetwork() const { - RLPStream nodes; + std::list peers; + { + RecursiveGuard l(x_sessions); + for (auto p: m_peers) + if (p.second) + peers.push_back(*p.second); + } + peers.sort(); + + RLPStream network; int count = 0; { RecursiveGuard l(x_sessions); - for (auto const& i: m_peers) + for (auto const& p: peers) { - Peer const& n = *(i.second); - // TODO: alpha: Figure out why it ever shares these ports.//n.address.port() >= 30300 && n.address.port() <= 30305 && + // TODO: alpha: Figure out why it ever shares these ports.//p.address.port() >= 30300 && p.address.port() <= 30305 && // TODO: alpha: if/how to save private addresses // Only save peers which have connected within 2 days, with properly-advertised port and public IP address - if (chrono::system_clock::now() - n.lastConnected < chrono::seconds(3600 * 48) && n.peerEndpoint().port() > 0 && n.peerEndpoint().port() < /*49152*/32768 && n.id != id() && !isPrivateAddress(n.peerEndpoint().address())) + if (chrono::system_clock::now() - p.lastConnected < chrono::seconds(3600 * 48) && p.peerEndpoint().port() > 0 && p.peerEndpoint().port() < /*49152*/32768 && p.id != id() && !isPrivateAddress(p.peerEndpoint().address())) { - nodes.appendList(10); - if (n.peerEndpoint().address().is_v4()) - nodes << n.peerEndpoint().address().to_v4().to_bytes(); + network.appendList(10); + if (p.peerEndpoint().address().is_v4()) + network << p.peerEndpoint().address().to_v4().to_bytes(); else - nodes << n.peerEndpoint().address().to_v6().to_bytes(); + network << p.peerEndpoint().address().to_v6().to_bytes(); // TODO: alpha: replace 0 with trust-state of node - nodes << n.peerEndpoint().port() << n.id << 0 - << chrono::duration_cast(n.lastConnected.time_since_epoch()).count() - << chrono::duration_cast(n.lastAttempted.time_since_epoch()).count() - << n.failedAttempts << (unsigned)n.lastDisconnect << n.score << n.rating; + network << p.peerEndpoint().port() << p.id << 0 + << chrono::duration_cast(p.lastConnected.time_since_epoch()).count() + << chrono::duration_cast(p.lastAttempted.time_since_epoch()).count() + << p.failedAttempts << (unsigned)p.lastDisconnect << p.score << p.rating; count++; } } } + + auto state = m_nodeTable->state(); + state.sort(); + for (auto const& s: state) + { + network.appendList(3); + if (s.endpoint.tcp.address().is_v4()) + network << s.endpoint.tcp.address().to_v4().to_bytes(); + else + network << s.endpoint.tcp.address().to_v6().to_bytes(); + network << s.endpoint.tcp.port() << s.id; + count++; + } + RLPStream ret(3); - ret << 0 << m_alias.secret(); - ret.appendList(count).appendRaw(nodes.out(), count); + ret << 1 << m_alias.secret(); + ret.appendList(count).appendRaw(network.out(), count); return ret.out(); } -// TODO: p2p Import-ant -void Host::restoreNodes(bytesConstRef _b) +void Host::restoreNetwork(bytesConstRef _b) { + // nodes can only be added if network is added + if (!isStarted()) + BOOST_THROW_EXCEPTION(NetworkStartRequired()); + RecursiveGuard l(x_sessions); RLP r(_b); - if (r.itemCount() > 0 && r[0].isInt()) - switch (r[0].toInt()) - { - case 0: - { - m_alias = KeyPair(r[1].toHash()); -// noteNode(id(), m_tcpPublic); + if (r.itemCount() > 0 && r[0].isInt() && r[0].toInt() == 1) + { + // r[0] = version + // r[1] = key + // r[2] = nodes - for (auto i: r[2]) + for (auto i: r[2]) + { + bi::tcp::endpoint tcp; + bi::udp::endpoint udp; + if (i[0].itemCount() == 4) { - bi::tcp::endpoint ep; - if (i[0].itemCount() == 4) - ep = bi::tcp::endpoint(bi::address_v4(i[0].toArray()), i[1].toInt()); - else - ep = bi::tcp::endpoint(bi::address_v6(i[0].toArray()), i[1].toInt()); - auto id = (NodeId)i[2]; - if (!m_peers.count(id)) - { - // TODO: p2p import/export :) -// auto n = noteNode(id, ep); -// n->lastConnected = chrono::system_clock::time_point(chrono::seconds(i[4].toInt())); -// n->lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[5].toInt())); -// n->failedAttempts = i[6].toInt(); -// n->lastDisconnect = (DisconnectReason)i[7].toInt(); -// n->score = (int)i[8].toInt(); -// n->rating = (int)i[9].toInt(); - } + tcp = bi::tcp::endpoint(bi::address_v4(i[0].toArray()), i[1].toInt()); + udp = bi::udp::endpoint(bi::address_v4(i[0].toArray()), i[1].toInt()); + } + else + { + tcp = bi::tcp::endpoint(bi::address_v6(i[0].toArray()), i[1].toInt()); + udp = bi::udp::endpoint(bi::address_v6(i[0].toArray()), i[1].toInt()); } - } - default:; - } - else - for (auto i: r) - { auto id = (NodeId)i[2]; - if (!m_peers.count(id)) + if (i.itemCount() == 3) + m_nodeTable->addNode(id, udp, tcp); + else if (i.itemCount() == 10) { - bi::tcp::endpoint ep; - if (i[0].itemCount() == 4) - ep = bi::tcp::endpoint(bi::address_v4(i[0].toArray()), i[1].toInt()); - else - ep = bi::tcp::endpoint(bi::address_v6(i[0].toArray()), i[1].toInt()); -// auto n = noteNode(id, ep); + shared_ptr p = make_shared(); + p->id = id; + p->lastConnected = chrono::system_clock::time_point(chrono::seconds(i[4].toInt())); + p->lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[5].toInt())); + p->failedAttempts = i[6].toInt(); + p->lastDisconnect = (DisconnectReason)i[7].toInt(); + p->score = (int)i[8].toInt(); + p->rating = (int)i[9].toInt(); + p->endpoint.tcp = tcp; + p->endpoint.udp = udp; + m_peers[p->id] = p; + m_nodeTable->addNode(*p.get()); } } + } } -KeyPair Host::getHostIdentifier() +KeyPair Host::getNetworkAlias(bytesConstRef _b) { - static string s_file(getDataDir() + "/host"); - static mutex s_x; - lock_guard l(s_x); - - h256 secret; - bytes b = contents(s_file); - if (b.size() == 32) - memcpy(secret.data(), b.data(), 32); + RLP r(_b); + if (r.itemCount() == 3 && r[0].isInt() && r[0].toInt() == 1) + return move(KeyPair(move(Secret(r[1].toBytes())))); else - { - // todo: replace w/user entropy; abstract to devcrypto - std::mt19937_64 s_eng(time(0) + chrono::high_resolution_clock::now().time_since_epoch().count()); - std::uniform_int_distribution d(0, 255); - for (unsigned i = 0; i < 32; ++i) - secret[i] = (byte)d(s_eng); - } - - if (!secret) - BOOST_THROW_EXCEPTION(crypto::InvalidState()); - return move(KeyPair(move(secret))); + return move(KeyPair::create()); } diff --git a/libp2p/Host.h b/libp2p/Host.h index a4493f9bf..cd45f3268 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -66,7 +66,7 @@ class Host; * Specifically, peers can be utilized in a variety of * many-to-many relationships while also needing to modify shared instances of * those peers. Modifying these properties via a storage backend alleviates - * Host of the responsibility. (&& remove save/restoreNodes) + * Host of the responsibility. (&& remove save/restoreNetwork) * @todo reimplement recording of historical session information on per-transport basis * @todo rebuild nodetable when localNetworking is enabled/disabled * @todo move attributes into protected @@ -74,7 +74,7 @@ class Host; class Peer: public Node { friend class Session; /// Allows Session to update score and rating. - friend class Host; /// For Host: saveNodes(), restoreNodes() + friend class Host; /// For Host: saveNetwork(), restoreNetwork() public: bool isOffline() const { return !m_session.lock(); } @@ -90,6 +90,28 @@ public: unsigned failedAttempts = 0; DisconnectReason lastDisconnect = NoDisconnect; ///< Reason for disconnect that happened last. + virtual bool operator<(Peer const& _p) const + { + if (isOffline() != _p.isOffline()) + return isOffline(); + else if (isOffline()) + if (lastAttempted == _p.lastAttempted) + return failedAttempts < _p.failedAttempts; + else + return lastAttempted < _p.lastAttempted; + else + if (score == _p.score) + if (rating == _p.rating) + if (failedAttempts == _p.failedAttempts) + return id < _p.id; + else + return failedAttempts < _p.failedAttempts; + else + return rating < _p.rating; + else + return score < _p.score; + } + protected: /// Used by isOffline() and (todo) for peer to emit session information. std::weak_ptr m_session; @@ -109,6 +131,7 @@ class HostNodeTableHandler: public NodeTableEventHandler * @brief The Host class * Capabilities should be registered prior to startNetwork, since m_capabilities is not thread-safe. * + * @todo exceptions when nodeTable not set (prior to start) * @todo onNodeTableEvent: move peer-connection logic into ensurePeers * @todo handshake: gracefully disconnect peer if peer already connected * @todo abstract socket -> IPConnection @@ -116,6 +139,7 @@ class HostNodeTableHandler: public NodeTableEventHandler * @todo handle conflict if addNode/requireNode called and Node already exists w/conflicting tcp or udp port * @todo write host identifier to disk w/nodes * @todo per-session keepalive/ping instead of broadcast; set ping-timeout via median-latency + * @todo configuration-management (NetworkPrefs+Keys+Topology) */ class Host: public Worker { @@ -125,7 +149,7 @@ class Host: public Worker public: /// Start server, listening for connections on the given port. - Host(std::string const& _clientVersion, NetworkPreferences const& _n = NetworkPreferences()); + Host(std::string const& _clientVersion, NetworkPreferences const& _n = NetworkPreferences(), bytesConstRef _restoreNetwork = bytesConstRef()); /// Will block on network process events. virtual ~Host(); @@ -158,10 +182,10 @@ public: /// Get peer information. PeerSessionInfos peers() const; - - /// Get number of peers connected; equivalent to, but faster than, peers().size(). - size_t peerCount() const { RecursiveGuard l(x_sessions); return m_peers.size(); } - + + /// Get number of peers connected. + size_t peerCount() const; + /// Get the address we're listening on currently. std::string listenAddress() const { return m_tcpPublic.address().to_string(); } @@ -169,10 +193,7 @@ public: unsigned short listenPort() const { return m_tcpPublic.port(); } /// Serialise the set of known peers. - bytes saveNodes() const; - - /// Deserialise the data and populate the set of known peers. - void restoreNodes(bytesConstRef _b); + bytes saveNetwork() const; // TODO: P2P this should be combined with peers into a HostStat object of some kind; coalesce data, as it's only used for status information. Peers nodes() const { RecursiveGuard l(x_sessions); Peers ret; for (auto const& i: m_peers) ret.push_back(*i.second); return ret; } @@ -196,6 +217,9 @@ public: protected: void onNodeTableEvent(NodeId const& _n, NodeTableEventType const& _e); + /// Deserialise the data and populate the set of known peers. + void restoreNetwork(bytesConstRef _b); + private: /// Populate m_peerAddresses with available public addresses. void determinePublic(std::string const& _publicAddress, bool _upnp); @@ -226,13 +250,12 @@ private: /// Shutdown network. Not thread-safe; to be called only by worker. virtual void doneWorking(); - - /// Add node - void addNode(Node const& _node) { if (m_nodeTable) m_nodeTable->addNode(_node); } /// Get or create host identifier (KeyPair). - KeyPair getHostIdentifier(); + static KeyPair getNetworkAlias(bytesConstRef _b); + bytes m_restoreNetwork; ///< Set by constructor and used to set Host key and restore network peers & nodes. + bool m_run = false; ///< Whether network is running. std::mutex x_runTimer; ///< Start/stop mutex. diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 7f9e3877f..b0fafa6cf 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -161,6 +161,7 @@ public: NodeEntry root() const { return NodeEntry(m_node, m_node.publicKey(), m_node.endpoint.udp); } std::list nodes() const; + unsigned size() const { return m_nodes.size(); } std::list state() const; bool haveNode(NodeId _id) { Guard l(x_nodes); return m_nodes.count(_id); } @@ -347,7 +348,7 @@ struct Neighbours: RLPXDatagram void interpretRLP(RLP const& _r) { ipAddress = _r[0].toString(); port = _r[1].toInt(); node = h512(_r[2].toBytes()); } }; - Neighbours(bi::udp::endpoint _ep): RLPXDatagram(_ep) {} + Neighbours(bi::udp::endpoint _ep): RLPXDatagram(_ep), expiration(futureFromEpoch(std::chrono::seconds(30))) {} Neighbours(bi::udp::endpoint _to, std::vector> const& _nearest, unsigned _offset = 0, unsigned _limit = 0): RLPXDatagram(_to), expiration(futureFromEpoch(std::chrono::seconds(30))) { auto limit = _limit ? std::min(_nearest.size(), (size_t)(_offset + _limit)) : _nearest.size(); diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index 7e618fb3b..1e93f5b1d 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -102,7 +102,7 @@ template vector randomSelection(vector const& _t, unsigned _n) // TODO: P2P integration: replace w/asio post -> serviceNodesRequest() void Session::ensureNodesRequested() { - if (isOpen() && !m_weRequestedNodes) + if (isConnected() && !m_weRequestedNodes) { m_weRequestedNodes = true; RLPStream s; @@ -310,7 +310,7 @@ bool Session::interpret(RLP const& _r) // OK passed all our checks. Assume it's good. addRating(1000); - m_server->addNode(Node(id, NodeIPEndpoint(bi::udp::endpoint(ep.address(), ep.port()), ep))); + m_server->addNode(id, ep.address().to_string(), ep.port(), ep.port()); clogS(NetTriviaDetail) << "New peer: " << ep << "(" << id .abridged()<< ")"; CONTINUE:; LAMEPEER:; diff --git a/libp2p/Session.h b/libp2p/Session.h index 87620a604..4d55e7d9d 100644 --- a/libp2p/Session.h +++ b/libp2p/Session.h @@ -59,7 +59,7 @@ public: void ping(); - bool isOpen() const { return m_socket.is_open(); } + bool isConnected() const { return m_socket.is_open(); } NodeId id() const; unsigned socketId() const { return m_socket.native_handle(); } diff --git a/libwebthree/WebThree.cpp b/libwebthree/WebThree.cpp index 40e878858..7daec9693 100644 --- a/libwebthree/WebThree.cpp +++ b/libwebthree/WebThree.cpp @@ -35,9 +35,9 @@ using namespace dev::p2p; using namespace dev::eth; using namespace dev::shh; -WebThreeDirect::WebThreeDirect(std::string const& _clientVersion, std::string const& _dbPath, bool _forceClean, std::set const& _interfaces, NetworkPreferences const& _n): +WebThreeDirect::WebThreeDirect(std::string const& _clientVersion, std::string const& _dbPath, bool _forceClean, std::set const& _interfaces, NetworkPreferences const& _n, bytesConstRef _network): m_clientVersion(_clientVersion), - m_net(_clientVersion, _n) + m_net(_clientVersion, _n, _network) { if (_dbPath.size()) Defaults::setDBPath(_dbPath); @@ -90,14 +90,9 @@ void WebThreeDirect::setIdealPeerCount(size_t _n) return m_net.setIdealPeerCount(_n); } -bytes WebThreeDirect::saveNodes() +bytes WebThreeDirect::saveNetwork() { - return m_net.saveNodes(); -} - -void WebThreeDirect::restoreNodes(bytesConstRef _saved) -{ - return m_net.restoreNodes(_saved); + return m_net.saveNetwork(); } void WebThreeDirect::connect(std::string const& _seedHost, unsigned short _port) diff --git a/libwebthree/WebThree.h b/libwebthree/WebThree.h index 3b97a0763..f99eed5cf 100644 --- a/libwebthree/WebThree.h +++ b/libwebthree/WebThree.h @@ -63,10 +63,7 @@ public: virtual void connect(std::string const& _seedHost, unsigned short _port) = 0; /// Save peers - virtual dev::bytes saveNodes() = 0; - - /// Restore peers - virtual void restoreNodes(bytesConstRef _saved) = 0; + virtual dev::bytes saveNetwork() = 0; /// Sets the ideal number of peers. virtual void setIdealPeerCount(size_t _n) = 0; @@ -106,7 +103,7 @@ class WebThreeDirect : public WebThreeNetworkFace public: /// Constructor for private instance. If there is already another process on the machine using @a _dbPath, then this will throw an exception. /// ethereum() may be safely static_cast()ed to a eth::Client*. - WebThreeDirect(std::string const& _clientVersion, std::string const& _dbPath, bool _forceClean = false, std::set const& _interfaces = {"eth", "shh"}, p2p::NetworkPreferences const& _n = p2p::NetworkPreferences()); + WebThreeDirect(std::string const& _clientVersion, std::string const& _dbPath, bool _forceClean = false, std::set const& _interfaces = {"eth", "shh"}, p2p::NetworkPreferences const& _n = p2p::NetworkPreferences(), bytesConstRef _network = bytesConstRef()); /// Destructor. ~WebThreeDirect(); @@ -133,10 +130,10 @@ public: void connect(std::string const& _seedHost, unsigned short _port = 30303) override; /// Save peers - dev::bytes saveNodes() override; + dev::bytes saveNetwork() override; - /// Restore peers - void restoreNodes(bytesConstRef _saved) override; +// /// Restore peers +// void restoreNetwork(bytesConstRef _saved) override; /// Sets the ideal number of peers. void setIdealPeerCount(size_t _n) override; diff --git a/neth/main.cpp b/neth/main.cpp index e49e47c16..cb84e87c7 100644 --- a/neth/main.cpp +++ b/neth/main.cpp @@ -415,13 +415,14 @@ int main(int argc, char** argv) cout << credits(); NetworkPreferences netPrefs(listenPort, publicIP, upnp, useLocal); + auto nodesState = contents((dbPath.size() ? dbPath : getDataDir()) + "/network.rlp"); dev::WebThreeDirect web3( "NEthereum(++)/" + clientName + "v" + dev::Version + "/" DEV_QUOTED(ETH_BUILD_TYPE) "/" DEV_QUOTED(ETH_BUILD_PLATFORM), dbPath, false, mode == NodeMode::Full ? set{"eth", "shh"} : set(), - netPrefs - ); + netPrefs, + &nodesState); web3.setIdealPeerCount(peers); eth::Client* c = mode == NodeMode::Full ? web3.ethereum() : nullptr; @@ -431,9 +432,7 @@ int main(int argc, char** argv) c->setAddress(coinbase); } - auto nodesState = contents((dbPath.size() ? dbPath : getDataDir()) + "/nodeState.rlp"); - web3.restoreNodes(&nodesState); - + cout << "Address: " << endl << toHex(us.address().asArray()) << endl; web3.startNetwork(); if (bootstrap) diff --git a/test/net.cpp b/test/net.cpp index 67c50dae8..7f30ed03b 100644 --- a/test/net.cpp +++ b/test/net.cpp @@ -30,7 +30,7 @@ using namespace dev::p2p; namespace ba = boost::asio; namespace bi = ba::ip; -BOOST_AUTO_TEST_SUITE(p2p) +BOOST_AUTO_TEST_SUITE(net) /** * Only used for testing. Not useful beyond tests. @@ -190,6 +190,9 @@ BOOST_AUTO_TEST_CASE(kademlia) node.populateAll(); clog << "NodeTable:\n" << *node.nodeTable.get() << endl; + auto nodes = node.nodeTable->nodes(); + nodes.sort(); + node.nodeTable->reset(); clog << "NodeTable:\n" << *node.nodeTable.get() << endl; @@ -199,6 +202,12 @@ BOOST_AUTO_TEST_CASE(kademlia) node.nodeTable->join(); this_thread::sleep_for(chrono::milliseconds(2000)); clog << "NodeTable:\n" << *node.nodeTable.get() << endl; + + BOOST_REQUIRE_EQUAL(node.nodeTable->size(), 8); + + auto netNodes = node.nodeTable->nodes(); + netNodes.sort(); + } BOOST_AUTO_TEST_CASE(test_udp_once) diff --git a/test/peer.cpp b/test/peer.cpp index a4b07e0b3..7f3c19e1e 100644 --- a/test/peer.cpp +++ b/test/peer.cpp @@ -50,6 +50,43 @@ BOOST_AUTO_TEST_CASE(host) BOOST_REQUIRE_EQUAL(host2.peerCount(), host1.peerCount()); } +BOOST_AUTO_TEST_CASE(save_nodes) +{ + std::list hosts; + for (auto i:{0,1,2,3,4,5}) + { + Host* h = new Host("Test", NetworkPreferences(30300 + i, "127.0.0.1", true, true)); + // starting host is required so listenport is available + h->start(); + while (!h->isStarted()) + this_thread::sleep_for(chrono::milliseconds(2)); + hosts.push_back(h); + } + + Host& host = *hosts.front(); + for (auto const& h: hosts) + host.addNode(h->id(), "127.0.0.1", h->listenPort(), h->listenPort()); + + Host& host2 = *hosts.back(); + for (auto const& h: hosts) + host2.addNode(h->id(), "127.0.0.1", h->listenPort(), h->listenPort()); + + this_thread::sleep_for(chrono::milliseconds(1000)); + bytes firstHostNetwork(host.saveNetwork()); + bytes secondHostNetwork(host.saveNetwork()); + + BOOST_REQUIRE_EQUAL(sha3(firstHostNetwork), sha3(secondHostNetwork)); + + BOOST_CHECK_EQUAL(host.peerCount(), 5); + BOOST_CHECK_EQUAL(host2.peerCount(), 5); + + RLP r(firstHostNetwork); + BOOST_REQUIRE(r.itemCount() == 3); + BOOST_REQUIRE(r[0].toInt() == 1); + BOOST_REQUIRE_EQUAL(r[1].toBytes().size(), 32); // secret + BOOST_REQUIRE_EQUAL(r[2].itemCount(), 5); +} + BOOST_AUTO_TEST_SUITE_END() int peerTest(int argc, char** argv) diff --git a/third/MainWin.cpp b/third/MainWin.cpp index 35650f94b..71d89039d 100644 --- a/third/MainWin.cpp +++ b/third/MainWin.cpp @@ -114,7 +114,8 @@ Main::Main(QWidget *parent) : connect(ui->ourAccounts->model(), SIGNAL(rowsMoved(const QModelIndex &, int, int, const QModelIndex &, int)), SLOT(ourAccountsRowsMoved())); - m_web3.reset(new WebThreeDirect("Third", getDataDir() + "/Third", false, {"eth", "shh"})); + bytesConstRef networkConfig((byte*)m_networkConfig.data(), m_networkConfig.size()) + m_web3.reset(new WebThreeDirect("Third", getDataDir() + "/Third", false, {"eth", "shh"}, NetworkPreferences(), networkConfig)); m_web3->connect(Host::pocHost()); m_server = unique_ptr(new WebThreeStubServer(m_qwebConnector, *web3(), keysAsVector(m_myKeys))); @@ -377,10 +378,10 @@ void Main::writeSettings() s.setValue("address", b); s.setValue("url", ui->urlEdit->text()); - bytes d = m_web3->saveNodes(); + bytes d = m_web3->saveNetwork(); if (d.size()) - m_nodes = QByteArray((char*)d.data(), (int)d.size()); - s.setValue("peers", m_nodes); + m_networkConfig = QByteArray((char*)d.data(), (int)d.size()); + s.setValue("peers", m_networkConfig); s.setValue("geometry", saveGeometry()); s.setValue("windowState", saveState()); @@ -409,7 +410,7 @@ void Main::readSettings(bool _skipGeometry) } } ethereum()->setAddress(m_myKeys.back().address()); - m_nodes = s.value("peers").toByteArray(); + m_networkConfig = s.value("peers").toByteArray(); ui->urlEdit->setText(s.value("url", "about:blank").toString()); //http://gavwood.com/gavcoin.html on_urlEdit_returnPressed(); } @@ -581,11 +582,9 @@ void Main::ensureNetwork() web3()->startNetwork(); web3()->connect(defPeer); } - else - if (!m_web3->peerCount()) - m_web3->connect(defPeer); - if (m_nodes.size()) - m_web3->restoreNodes(bytesConstRef((byte*)m_nodes.data(), m_nodes.size())); +// else +// if (!m_web3->peerCount()) +// m_web3->connect(defPeer); } void Main::on_connect_triggered() diff --git a/third/MainWin.h b/third/MainWin.h index 0bd75d5de..924fc057e 100644 --- a/third/MainWin.h +++ b/third/MainWin.h @@ -131,7 +131,7 @@ private: unsigned m_currenciesFilter = (unsigned)-1; unsigned m_balancesFilter = (unsigned)-1; - QByteArray m_nodes; + QByteArray m_networkConfig; QStringList m_servers; QNetworkAccessManager m_webCtrl; From 6bbcaae5e29c8b81cc3e85e5d695e342016d37f6 Mon Sep 17 00:00:00 2001 From: subtly Date: Mon, 2 Feb 2015 09:46:41 -0800 Subject: [PATCH 31/71] alethzero, third: load network config --- alethzero/MainWin.cpp | 9 +++++---- third/MainWin.cpp | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index 6a8cf07fe..f20782a42 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -159,8 +159,9 @@ Main::Main(QWidget *parent) : statusBar()->addPermanentWidget(ui->blockCount); connect(ui->ourAccounts->model(), SIGNAL(rowsMoved(const QModelIndex &, int, int, const QModelIndex &, int)), SLOT(ourAccountsRowsMoved())); - - m_webThree.reset(new WebThreeDirect(string("AlethZero/v") + dev::Version + "/" DEV_QUOTED(ETH_BUILD_TYPE) "/" DEV_QUOTED(ETH_BUILD_PLATFORM), getDataDir() + "/AlethZero", false, {"eth", "shh"})); + + bytesConstRef network((byte*)m_networkConfig.data(), m_networkConfig.size()); + m_webThree.reset(new WebThreeDirect(string("AlethZero/v") + dev::Version + "/" DEV_QUOTED(ETH_BUILD_TYPE) "/" DEV_QUOTED(ETH_BUILD_PLATFORM), getDataDir() + "/AlethZero", false, {"eth", "shh"}, p2p::NetworkPreferences(), network)); m_qwebConnector.reset(new QWebThreeConnector()); m_server.reset(new OurWebThreeStubServer(*m_qwebConnector, *web3(), keysAsVector(m_myKeys), this)); @@ -1860,8 +1861,8 @@ void Main::on_net_triggered() web3()->setNetworkPreferences(netPrefs()); ethereum()->setNetworkId(m_privateChain.size() ? sha3(m_privateChain.toStdString()) : 0); // TODO: p2p - if (m_networkConfig.size()/* && ui->usePast->isChecked()*/) - web3()->restoreNetwork(bytesConstRef((byte*)m_networkConfig.data(), m_networkConfig.size())); +// if (m_networkConfig.size()/* && ui->usePast->isChecked()*/) +// web3()->restoreNetwork(bytesConstRef((byte*)m_networkConfig.data(), m_networkConfig.size())); web3()->startNetwork(); ui->downloadView->setDownloadMan(ethereum()->downloadMan()); } diff --git a/third/MainWin.cpp b/third/MainWin.cpp index b50722eff..e52cecc7e 100644 --- a/third/MainWin.cpp +++ b/third/MainWin.cpp @@ -114,7 +114,7 @@ Main::Main(QWidget *parent) : connect(ui->ourAccounts->model(), SIGNAL(rowsMoved(const QModelIndex &, int, int, const QModelIndex &, int)), SLOT(ourAccountsRowsMoved())); - bytesConstRef networkConfig((byte*)m_networkConfig.data(), m_networkConfig.size()) + bytesConstRef networkConfig((byte*)m_networkConfig.data(), m_networkConfig.size()); m_web3.reset(new WebThreeDirect("Third", getDataDir() + "/Third", false, {"eth", "shh"}, NetworkPreferences(), networkConfig)); m_web3->connect(Host::pocHost()); From 0ac7d86c87c28582a7a1c98c4cc1cf666540fdf9 Mon Sep 17 00:00:00 2001 From: subtly Date: Mon, 2 Feb 2015 09:47:16 -0800 Subject: [PATCH 32/71] update whisper test for node-network --- test/whisperTopic.cpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/whisperTopic.cpp b/test/whisperTopic.cpp index 6c08ae1ab..99032d99b 100644 --- a/test/whisperTopic.cpp +++ b/test/whisperTopic.cpp @@ -101,6 +101,7 @@ BOOST_AUTO_TEST_CASE(forwarding) bool done = false; bool startedListener = false; + Public phid; std::thread listener([&]() { setThreadName("listener"); @@ -110,6 +111,7 @@ BOOST_AUTO_TEST_CASE(forwarding) ph.setIdealPeerCount(0); auto wh = ph.registerCapability(new WhisperHost()); ph.start(); + phid = ph.id(); startedListener = true; @@ -130,6 +132,7 @@ BOOST_AUTO_TEST_CASE(forwarding) }); bool startedForwarder = false; + Public fwderid; std::thread forwarder([&]() { setThreadName("forwarder"); @@ -143,9 +146,10 @@ BOOST_AUTO_TEST_CASE(forwarding) auto wh = ph.registerCapability(new WhisperHost()); this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); + fwderid = ph.id(); this_thread::sleep_for(chrono::milliseconds(500)); - ph.connect("127.0.0.1", 50303); + ph.addNode(phid, "127.0.0.1", 50303, 50303); startedForwarder = true; @@ -172,7 +176,7 @@ BOOST_AUTO_TEST_CASE(forwarding) this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); this_thread::sleep_for(chrono::milliseconds(500)); - ph.connect("127.0.0.1", 50305); + ph.addNode(fwderid, "127.0.0.1", 50305, 50305); KeyPair us = KeyPair::create(); wh->post(us.sec(), RLPStream().append(1).out(), BuildTopic("test")); @@ -195,6 +199,7 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) unsigned result = 0; bool done = false; + Public listenerid; bool startedForwarder = false; std::thread forwarder([&]() { @@ -206,9 +211,9 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) auto wh = ph.registerCapability(new WhisperHost()); this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); - + this_thread::sleep_for(chrono::milliseconds(500)); - ph.connect("127.0.0.1", 50303); +// ph.addNode("127.0.0.1", 50303, 50303); startedForwarder = true; @@ -236,7 +241,7 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); this_thread::sleep_for(chrono::milliseconds(500)); - ph.connect("127.0.0.1", 50305); +// ph.addNode("127.0.0.1", 50305, 50305); KeyPair us = KeyPair::create(); wh->post(us.sec(), RLPStream().append(1).out(), BuildTopic("test")); @@ -250,7 +255,7 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); this_thread::sleep_for(chrono::milliseconds(500)); - ph.connect("127.0.0.1", 50305); +// ph.addNode("127.0.0.1", 50305, 50305); /// Only interested in odd packets auto w = wh->installWatch(BuildTopicMask("test")); From 27e2f98278280a7c613252d3de557c4cf4096357 Mon Sep 17 00:00:00 2001 From: subtly Date: Wed, 4 Feb 2015 17:50:03 -0800 Subject: [PATCH 33/71] fix whispertopic test --- test/whisperTopic.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/whisperTopic.cpp b/test/whisperTopic.cpp index 9c88d1b24..1dcc6b9d6 100644 --- a/test/whisperTopic.cpp +++ b/test/whisperTopic.cpp @@ -40,15 +40,6 @@ BOOST_AUTO_TEST_CASE(topic) auto whOther = phOther.registerCapability(new WhisperHost()); phOther.start(); - Host ph("Test", NetworkPreferences(30300, "127.0.0.1", false, true)); - auto wh = ph.registerCapability(new WhisperHost()); - ph.start(); - - this_thread::sleep_for(chrono::milliseconds(100)); - ph.addNode(phOther.id(), "127.0.0.1", 30303, 30303); - - this_thread::sleep_for(chrono::milliseconds(500)); - bool started = false; unsigned result = 0; std::thread listener([&]() @@ -66,7 +57,7 @@ BOOST_AUTO_TEST_CASE(topic) { for (auto i: whOther->checkWatch(w)) { - Message msg = wh->envelope(i).open(wh->fullTopic(w)); + Message msg = whOther->envelope(i).open(whOther->fullTopic(w)); last = RLP(msg.payload()).toInt(); if (received.count(last)) continue; @@ -78,6 +69,15 @@ BOOST_AUTO_TEST_CASE(topic) } }); + + Host ph("Test", NetworkPreferences(30300, "127.0.0.1", false, true)); + auto wh = ph.registerCapability(new WhisperHost()); + ph.start(); + + this_thread::sleep_for(chrono::milliseconds(100)); + ph.addNode(phOther.id(), "127.0.0.1", 30303, 30303); + + this_thread::sleep_for(chrono::milliseconds(500)); while (!started) this_thread::sleep_for(chrono::milliseconds(2)); From e9538b23c8d9bccb0905755d4150e59b58e8d90c Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 6 Feb 2015 11:54:00 -0800 Subject: [PATCH 34/71] updates for code-review --- libp2p/Common.h | 1 + libp2p/Host.cpp | 44 +++++------ libp2p/Host.h | 47 ++++++----- libp2p/NodeTable.cpp | 153 ++++++++++++++++++------------------ libp2p/NodeTable.h | 164 +++++++++++++++++++++++---------------- libp2p/Session.cpp | 20 ++--- libp2p/UDP.h | 6 +- libwebthree/WebThree.cpp | 2 +- libwebthree/WebThree.h | 2 +- test/net.cpp | 8 +- test/whisperTopic.cpp | 22 +++--- 11 files changed, 251 insertions(+), 218 deletions(-) diff --git a/libp2p/Common.h b/libp2p/Common.h index 19cf447ce..219ec804b 100644 --- a/libp2p/Common.h +++ b/libp2p/Common.h @@ -169,6 +169,7 @@ struct Node NodeIPEndpoint endpoint; /// If true, node will not be removed from Node list. + // TODO: p2p implement bool required = false; virtual operator bool() const { return (bool)id; } diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index fc86cf06b..25a095ae6 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -53,7 +53,7 @@ Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n, byte m_ifAddresses(Network::getInterfaceAddresses()), m_ioService(2), m_tcp4Acceptor(m_ioService), - m_alias(getNetworkAlias(_restoreNetwork)), + m_alias(networkAlias(_restoreNetwork)), m_lastPing(chrono::time_point::min()) { for (auto address: m_ifAddresses) @@ -182,7 +182,7 @@ void Host::onNodeTableEvent(NodeId const& _n, NodeTableEventType const& _e) { clog(NetNote) << "p2p.host.nodeTable.events.nodeEntryAdded " << _n; - auto n = (*m_nodeTable)[_n]; + auto n = m_nodeTable->node(_n); if (n) { RecursiveGuard l(x_sessions); @@ -195,7 +195,7 @@ void Host::onNodeTableEvent(NodeId const& _n, NodeTableEventType const& _e) } p->endpoint.tcp = n.endpoint.tcp; - // TODO: Implement similar to doFindNode. Attempt connecting to nodes + // TODO: Implement similar to discover. Attempt connecting to nodes // until ideal peer count is reached; if all nodes are tried, // repeat. Notably, this is an integrated process such that // when onNodeTableEvent occurs we should also update +/- @@ -442,9 +442,9 @@ void Host::connect(std::shared_ptr const& _p) Peer *nptr = _p.get(); { Guard l(x_pendingNodeConns); - if (m_pendingNodeConns.count(nptr)) + if (m_pendingPeerConns.count(nptr)) return; - m_pendingNodeConns.insert(nptr); + m_pendingPeerConns.insert(nptr); } clog(NetConnect) << "Attempting connection to node" << _p->id.abridged() << "@" << _p->peerEndpoint() << "from" << id().abridged(); @@ -454,25 +454,25 @@ void Host::connect(std::shared_ptr const& _p) if (ec) { clog(NetConnect) << "Connection refused to node" << _p->id.abridged() << "@" << _p->peerEndpoint() << "(" << ec.message() << ")"; - _p->lastDisconnect = TCPError; - _p->lastAttempted = std::chrono::system_clock::now(); + _p->m_lastDisconnect = TCPError; + _p->m_lastAttempted = std::chrono::system_clock::now(); } else { clog(NetConnect) << "Connected to" << _p->id.abridged() << "@" << _p->peerEndpoint(); - _p->lastConnected = std::chrono::system_clock::now(); + _p->m_lastConnected = std::chrono::system_clock::now(); auto ps = make_shared(this, std::move(*s), _p); ps->start(); } delete s; Guard l(x_pendingNodeConns); - m_pendingNodeConns.erase(nptr); + m_pendingPeerConns.erase(nptr); }); } -PeerSessionInfos Host::peers() const +PeerSessionInfos Host::peerSessionInfo() const { if (!m_run) return PeerSessionInfos(); @@ -622,7 +622,7 @@ bytes Host::saveNetwork() const // TODO: alpha: Figure out why it ever shares these ports.//p.address.port() >= 30300 && p.address.port() <= 30305 && // TODO: alpha: if/how to save private addresses // Only save peers which have connected within 2 days, with properly-advertised port and public IP address - if (chrono::system_clock::now() - p.lastConnected < chrono::seconds(3600 * 48) && p.peerEndpoint().port() > 0 && p.peerEndpoint().port() < /*49152*/32768 && p.id != id() && !isPrivateAddress(p.peerEndpoint().address())) + if (chrono::system_clock::now() - p.m_lastConnected < chrono::seconds(3600 * 48) && p.peerEndpoint().port() > 0 && p.peerEndpoint().port() < /*49152*/32768 && p.id != id() && !isPrivateAddress(p.peerEndpoint().address())) { network.appendList(10); if (p.peerEndpoint().address().is_v4()) @@ -631,15 +631,15 @@ bytes Host::saveNetwork() const network << p.peerEndpoint().address().to_v6().to_bytes(); // TODO: alpha: replace 0 with trust-state of node network << p.peerEndpoint().port() << p.id << 0 - << chrono::duration_cast(p.lastConnected.time_since_epoch()).count() - << chrono::duration_cast(p.lastAttempted.time_since_epoch()).count() - << p.failedAttempts << (unsigned)p.lastDisconnect << p.score << p.rating; + << chrono::duration_cast(p.m_lastConnected.time_since_epoch()).count() + << chrono::duration_cast(p.m_lastAttempted.time_since_epoch()).count() + << p.m_failedAttempts << (unsigned)p.m_lastDisconnect << p.m_score << p.m_rating; count++; } } } - auto state = m_nodeTable->state(); + auto state = m_nodeTable->snapshot(); state.sort(); for (auto const& s: state) { @@ -693,12 +693,12 @@ void Host::restoreNetwork(bytesConstRef _b) { shared_ptr p = make_shared(); p->id = id; - p->lastConnected = chrono::system_clock::time_point(chrono::seconds(i[4].toInt())); - p->lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[5].toInt())); - p->failedAttempts = i[6].toInt(); - p->lastDisconnect = (DisconnectReason)i[7].toInt(); - p->score = (int)i[8].toInt(); - p->rating = (int)i[9].toInt(); + p->m_lastConnected = chrono::system_clock::time_point(chrono::seconds(i[4].toInt())); + p->m_lastAttempted = chrono::system_clock::time_point(chrono::seconds(i[5].toInt())); + p->m_failedAttempts = i[6].toInt(); + p->m_lastDisconnect = (DisconnectReason)i[7].toInt(); + p->m_score = (int)i[8].toInt(); + p->m_rating = (int)i[9].toInt(); p->endpoint.tcp = tcp; p->endpoint.udp = udp; m_peers[p->id] = p; @@ -708,7 +708,7 @@ void Host::restoreNetwork(bytesConstRef _b) } } -KeyPair Host::getNetworkAlias(bytesConstRef _b) +KeyPair Host::networkAlias(bytesConstRef _b) { RLP r(_b); if (r.itemCount() == 3 && r[0].isInt() && r[0].toInt() == 1) diff --git a/libp2p/Host.h b/libp2p/Host.h index cd45f3268..b24f1343c 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -79,37 +79,37 @@ public: bool isOffline() const { return !m_session.lock(); } bi::tcp::endpoint const& peerEndpoint() const { return endpoint.tcp; } - - int score = 0; ///< All time cumulative. - int rating = 0; ///< Trending. + + int m_score = 0; ///< All time cumulative. + int m_rating = 0; ///< Trending. /// Network Availability - std::chrono::system_clock::time_point lastConnected; - std::chrono::system_clock::time_point lastAttempted; - unsigned failedAttempts = 0; - DisconnectReason lastDisconnect = NoDisconnect; ///< Reason for disconnect that happened last. + std::chrono::system_clock::time_point m_lastConnected; + std::chrono::system_clock::time_point m_lastAttempted; + unsigned m_failedAttempts = 0; + DisconnectReason m_lastDisconnect = NoDisconnect; ///< Reason for disconnect that happened last. virtual bool operator<(Peer const& _p) const { if (isOffline() != _p.isOffline()) return isOffline(); else if (isOffline()) - if (lastAttempted == _p.lastAttempted) - return failedAttempts < _p.failedAttempts; + if (m_lastAttempted == _p.m_lastAttempted) + return m_failedAttempts < _p.m_failedAttempts; else - return lastAttempted < _p.lastAttempted; + return m_lastAttempted < _p.m_lastAttempted; else - if (score == _p.score) - if (rating == _p.rating) - if (failedAttempts == _p.failedAttempts) + if (m_score == _p.m_score) + if (m_rating == _p.m_rating) + if (m_failedAttempts == _p.m_failedAttempts) return id < _p.id; else - return failedAttempts < _p.failedAttempts; + return m_failedAttempts < _p.m_failedAttempts; else - return rating < _p.rating; + return m_rating < _p.m_rating; else - return score < _p.score; + return m_score < _p.m_score; } protected: @@ -121,9 +121,14 @@ using Peers = std::vector; class HostNodeTableHandler: public NodeTableEventHandler { - friend class Host; +public: HostNodeTableHandler(Host& _host); + + Host const& host() const { return m_host; } + +private: virtual void processEvent(NodeId const& _n, NodeTableEventType const& _e); + Host& m_host; }; @@ -181,7 +186,7 @@ public: void setIdealPeerCount(unsigned _n) { m_idealPeerCount = _n; } /// Get peer information. - PeerSessionInfos peers() const; + PeerSessionInfos peerSessionInfo() const; /// Get number of peers connected. size_t peerCount() const; @@ -196,7 +201,7 @@ public: bytes saveNetwork() const; // TODO: P2P this should be combined with peers into a HostStat object of some kind; coalesce data, as it's only used for status information. - Peers nodes() const { RecursiveGuard l(x_sessions); Peers ret; for (auto const& i: m_peers) ret.push_back(*i.second); return ret; } + Peers getPeers() const { RecursiveGuard l(x_sessions); Peers ret; for (auto const& i: m_peers) ret.push_back(*i.second); return ret; } void setNetworkPreferences(NetworkPreferences const& _p) { auto had = isStarted(); if (had) stop(); m_netPrefs = _p; if (had) start(); } @@ -252,7 +257,7 @@ private: virtual void doneWorking(); /// Get or create host identifier (KeyPair). - static KeyPair getNetworkAlias(bytesConstRef _b); + static KeyPair networkAlias(bytesConstRef _b); bytes m_restoreNetwork; ///< Set by constructor and used to set Host key and restore network peers & nodes. @@ -274,7 +279,7 @@ private: std::unique_ptr m_timer; ///< Timer which, when network is running, calls scheduler() every c_timerInterval ms. static const unsigned c_timerInterval = 100; ///< Interval which m_timer is run when network is connected. - std::set m_pendingNodeConns; /// Used only by connect(Peer&) to limit concurrently connecting to same node. See connect(shared_ptrconst&). + std::set m_pendingPeerConns; /// Used only by connect(Peer&) to limit concurrently connecting to same node. See connect(shared_ptrconst&). Mutex x_pendingNodeConns; bi::tcp::endpoint m_tcpPublic; ///< Our public listening endpoint. diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index dffa0933b..186ece5e2 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -24,15 +24,15 @@ using namespace std; using namespace dev; using namespace dev::p2p; -NodeEntry::NodeEntry(Node _src, Public _pubk, NodeIPEndpoint _gw): Node(_pubk, _gw), distance(NodeTable::dist(_src.id,_pubk)) {} -NodeEntry::NodeEntry(Node _src, Public _pubk, bi::udp::endpoint _udp): Node(_pubk, NodeIPEndpoint(_udp)), distance(NodeTable::dist(_src.id,_pubk)) {} +NodeEntry::NodeEntry(Node _src, Public _pubk, NodeIPEndpoint _gw): Node(_pubk, _gw), distance(NodeTable::distance(_src.id,_pubk)) {} +NodeEntry::NodeEntry(Node _src, Public _pubk, bi::udp::endpoint _udp): Node(_pubk, NodeIPEndpoint(_udp)), distance(NodeTable::distance(_src.id,_pubk)) {} NodeTable::NodeTable(ba::io_service& _io, KeyPair _alias, uint16_t _udp): m_node(Node(_alias.pub(), bi::udp::endpoint())), m_secret(_alias.sec()), m_io(_io), m_socket(new NodeSocket(m_io, *this, _udp)), - m_socketPtr(m_socket.get()), + m_socketPointer(m_socket.get()), m_bucketRefreshTimer(m_io), m_evictionCheckTimer(m_io) { @@ -42,15 +42,18 @@ NodeTable::NodeTable(ba::io_service& _io, KeyPair _alias, uint16_t _udp): m_state[i].modified = chrono::steady_clock::now() - chrono::seconds(1); } - m_socketPtr->connect(); + m_socketPointer->connect(); doRefreshBuckets(boost::system::error_code()); } NodeTable::~NodeTable() { + // Cancel scheduled tasks to ensure. m_evictionCheckTimer.cancel(); m_bucketRefreshTimer.cancel(); - m_socketPtr->disconnect(); + + // Disconnect socket so that deallocation is safe. + m_socketPointer->disconnect(); } void NodeTable::processEvents() @@ -87,14 +90,14 @@ shared_ptr NodeTable::addNode(Node const& _node) m_nodes[_node.id] = ret; PingNode p(_node.endpoint.udp, m_node.endpoint.udp.address().to_string(), m_node.endpoint.udp.port()); p.sign(m_secret); - m_socketPtr->send(p); + m_socketPointer->send(p); } return move(ret); } -void NodeTable::join() +void NodeTable::discover() { - doFindNode(m_node.id); + discover(m_node.id); } list NodeTable::nodes() const @@ -106,7 +109,7 @@ list NodeTable::nodes() const return move(nodes); } -list NodeTable::state() const +list NodeTable::snapshot() const { list ret; Guard l(x_state); @@ -116,42 +119,35 @@ list NodeTable::state() const return move(ret); } -Node NodeTable::operator[](NodeId _id) +Node NodeTable::node(NodeId _id) { Guard l(x_nodes); auto n = m_nodes[_id]; return !!n ? *n : Node(); } -shared_ptr NodeTable::getNodeEntry(NodeId _id) +shared_ptr NodeTable::nodeEntry(NodeId _id) { Guard l(x_nodes); auto n = m_nodes[_id]; return !!n ? move(n) : move(shared_ptr()); } -void NodeTable::requestNeighbours(NodeEntry const& _node, NodeId _target) const +void NodeTable::discover(NodeId _node, unsigned _round, shared_ptr>> _tried) { - FindNode p(_node.endpoint.udp, _target); - p.sign(m_secret); - m_socketPtr->send(p); -} - -void NodeTable::doFindNode(NodeId _node, unsigned _round, shared_ptr>> _tried) -{ - if (!m_socketPtr->isOpen() || _round == s_maxSteps) + if (!m_socketPointer->isOpen() || _round == s_maxSteps) return; if (_round == s_maxSteps) { - clog(NodeTableNote) << "Terminating doFindNode after " << _round << " rounds."; + clog(NodeTableNote) << "Terminating discover after " << _round << " rounds."; return; } else if(!_round && !_tried) // initialized _tried on first round _tried.reset(new set>()); - auto nearest = findNearest(_node); + auto nearest = nearestNodeEntries(_node); list> tried; for (unsigned i = 0; i < nearest.size() && tried.size() < s_alpha; i++) if (!_tried->count(nearest[i])) @@ -160,12 +156,12 @@ void NodeTable::doFindNode(NodeId _node, unsigned _round, shared_ptrendpoint.udp, _node); p.sign(m_secret); - m_socketPtr->send(p); + m_socketPointer->send(p); } if (tried.empty()) { - clog(NodeTableNote) << "Terminating doFindNode after " << _round << " rounds."; + clog(NodeTableNote) << "Terminating discover after " << _round << " rounds."; return; } @@ -181,15 +177,15 @@ void NodeTable::doFindNode(NodeId _node, unsigned _round, shared_ptr> NodeTable::findNearest(NodeId _target) +vector> NodeTable::nearestNodeEntries(NodeId _target) { // send s_alpha FindNode packets to nodes we know, closest to target static unsigned lastBin = s_bins - 1; - unsigned head = dist(m_node.id, _target); + unsigned head = distance(m_node.id, _target); unsigned tail = head == 0 ? lastBin : (head - 1) % s_bins; map>> found; @@ -204,7 +200,7 @@ vector> NodeTable::findNearest(NodeId _target) if (auto p = n.lock()) { if (count < s_bucketSize) - found[dist(_target, p->id)].push_back(p); + found[distance(_target, p->id)].push_back(p); else break; } @@ -214,7 +210,7 @@ vector> NodeTable::findNearest(NodeId _target) if (auto p = n.lock()) { if (count < s_bucketSize) - found[dist(_target, p->id)].push_back(p); + found[distance(_target, p->id)].push_back(p); else break; } @@ -231,7 +227,7 @@ vector> NodeTable::findNearest(NodeId _target) if (auto p = n.lock()) { if (count < s_bucketSize) - found[dist(_target, p->id)].push_back(p); + found[distance(_target, p->id)].push_back(p); else break; } @@ -245,7 +241,7 @@ vector> NodeTable::findNearest(NodeId _target) if (auto p = n.lock()) { if (count < s_bucketSize) - found[dist(_target, p->id)].push_back(p); + found[distance(_target, p->id)].push_back(p); else break; } @@ -263,7 +259,7 @@ void NodeTable::ping(bi::udp::endpoint _to) const { PingNode p(_to, m_node.endpoint.udp.address().to_string(), m_node.endpoint.udp.port()); p.sign(m_secret); - m_socketPtr->send(p); + m_socketPointer->send(p); } void NodeTable::ping(NodeEntry* _n) const @@ -274,65 +270,64 @@ void NodeTable::ping(NodeEntry* _n) const void NodeTable::evict(shared_ptr _leastSeen, shared_ptr _new) { - if (!m_socketPtr->isOpen()) + if (!m_socketPointer->isOpen()) return; - Guard l(x_evictions); - m_evictions.push_back(EvictionTimeout(make_pair(_leastSeen->id,chrono::steady_clock::now()), _new->id)); - if (m_evictions.size() == 1) - doCheckEvictions(boost::system::error_code()); - - m_evictions.push_back(EvictionTimeout(make_pair(_leastSeen->id,chrono::steady_clock::now()), _new->id)); + { + Guard l(x_evictions); + m_evictions.push_back(EvictionTimeout(make_pair(_leastSeen->id,chrono::steady_clock::now()), _new->id)); + if (m_evictions.size() == 1) + doCheckEvictions(boost::system::error_code()); + + m_evictions.push_back(EvictionTimeout(make_pair(_leastSeen->id,chrono::steady_clock::now()), _new->id)); + } ping(_leastSeen.get()); } -void NodeTable::noteNode(Public const& _pubk, bi::udp::endpoint const& _endpoint) +void NodeTable::noteActiveNode(Public const& _pubk, bi::udp::endpoint const& _endpoint) { if (_pubk == m_node.address()) return; shared_ptr node(addNode(_pubk, _endpoint)); - // todo: sometimes node is nullptr here + // TODO p2p: old bug (maybe gone now) sometimes node is nullptr here if (!!node) - noteNode(node); -} - -void NodeTable::noteNode(shared_ptr _n) -{ - shared_ptr contested; { - NodeBucket& s = bucket(_n.get()); - Guard l(x_state); - s.nodes.remove_if([&_n](weak_ptr n) - { - if (n.lock() == _n) - return true; - return false; - }); - - if (s.nodes.size() >= s_bucketSize) + shared_ptr contested; { - contested = s.nodes.front().lock(); - if (!contested) + Guard l(x_state); + NodeBucket& s = bucket_UNSAFE(node.get()); + s.nodes.remove_if([&node](weak_ptr n) + { + if (n.lock() == node) + return true; + return false; + }); + + if (s.nodes.size() >= s_bucketSize) { - s.nodes.pop_front(); - s.nodes.push_back(_n); + contested = s.nodes.front().lock(); + if (!contested) + { + s.nodes.pop_front(); + s.nodes.push_back(node); + } } + else + s.nodes.push_back(node); } - else - s.nodes.push_back(_n); + + if (contested) + evict(contested, node); } - - if (contested) - evict(contested, _n); } void NodeTable::dropNode(shared_ptr _n) { - NodeBucket &s = bucket(_n.get()); { Guard l(x_state); + NodeBucket& s = bucket_UNSAFE(_n.get()); s.nodes.remove_if([&_n](weak_ptr n) { return n.lock() == _n; }); } { @@ -345,7 +340,7 @@ void NodeTable::dropNode(shared_ptr _n) m_nodeEventHandler->appendEvent(_n->id, NodeEntryRemoved); } -NodeTable::NodeBucket& NodeTable::bucket(NodeEntry const* _n) +NodeTable::NodeBucket& NodeTable::bucket_UNSAFE(NodeEntry const* _n) { return m_state[_n->distance - 1]; } @@ -381,7 +376,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes unsigned packetType = signedBytes[0]; if (packetType && packetType < 4) - noteNode(nodeid, _from); + noteActiveNode(nodeid, _from); bytesConstRef rlpBytes(_packet.cropped(h256::size + Signature::size + 1)); RLP rlp(rlpBytes); @@ -398,10 +393,10 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes for (auto it = m_evictions.begin(); it != m_evictions.end(); it++) if (it->first.first == nodeid && it->first.second > std::chrono::steady_clock::now()) { - if (auto n = getNodeEntry(it->second)) + if (auto n = nodeEntry(it->second)) dropNode(n); - if (auto n = (*this)[it->first.first]) + if (auto n = node(it->first.first)) addNode(n); it = m_evictions.erase(it); @@ -414,7 +409,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes Neighbours in = Neighbours::fromBytesConstRef(_from, rlpBytes); // clog(NodeTableMessageSummary) << "Received " << in.nodes.size() << " Neighbours from " << _from.address().to_string() << ":" << _from.port(); for (auto n: in.nodes) - noteNode(n.node, bi::udp::endpoint(bi::address::from_string(n.ipAddress), n.port)); + noteActiveNode(n.node, bi::udp::endpoint(bi::address::from_string(n.ipAddress), n.port)); break; } @@ -423,13 +418,13 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes // clog(NodeTableMessageSummary) << "Received FindNode from " << _from.address().to_string() << ":" << _from.port(); FindNode in = FindNode::fromBytesConstRef(_from, rlpBytes); - vector> nearest = findNearest(in.target); - static unsigned const nlimit = (m_socketPtr->maxDatagramSize - 11) / 86; + vector> nearest = nearestNodeEntries(in.target); + static unsigned const nlimit = (m_socketPointer->maxDatagramSize - 11) / 86; for (unsigned offset = 0; offset < nearest.size(); offset += nlimit) { Neighbours out(_from, nearest, offset, nlimit); out.sign(m_secret); - m_socketPtr->send(out); + m_socketPointer->send(out); } break; } @@ -442,7 +437,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes Pong p(_from); p.echo = sha3(rlpBytes); p.sign(m_secret); - m_socketPtr->send(p); + m_socketPointer->send(p); break; } @@ -459,7 +454,7 @@ void NodeTable::onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytes void NodeTable::doCheckEvictions(boost::system::error_code const& _ec) { - if (_ec || !m_socketPtr->isOpen()) + if (_ec || !m_socketPointer->isOpen()) return; auto self(shared_from_this()); @@ -472,8 +467,8 @@ void NodeTable::doCheckEvictions(boost::system::error_code const& _ec) bool evictionsRemain = false; list> drop; { - Guard le(x_evictions); Guard ln(x_nodes); + Guard le(x_evictions); for (auto& e: m_evictions) if (chrono::steady_clock::now() - e.first.second > c_reqTimeout) if (auto n = m_nodes[e.second]) @@ -496,7 +491,7 @@ void NodeTable::doRefreshBuckets(boost::system::error_code const& _ec) return; clog(NodeTableNote) << "refreshing buckets"; - bool connected = m_socketPtr->isOpen(); + bool connected = m_socketPointer->isOpen(); bool refreshed = false; if (connected) { diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index e2ba06f4b..72f7800f5 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -53,7 +53,7 @@ class NodeTableEventHandler { friend class NodeTable; public: - virtual void processEvent(NodeId const& _n, NodeTableEventType const& _e) =0; + virtual void processEvent(NodeId const& _n, NodeTableEventType const& _e) = 0; protected: /// Called by NodeTable on behalf of an implementation (Host) to process new events without blocking nodetable. @@ -65,7 +65,8 @@ protected: if (!m_nodeEventHandler.size()) return; m_nodeEventHandler.unique(); - for (auto const& n: m_nodeEventHandler) events.push_back(std::make_pair(n,m_events[n])); + for (auto const& n: m_nodeEventHandler) + events.push_back(std::make_pair(n,m_events[n])); m_nodeEventHandler.clear(); m_events.clear(); } @@ -80,10 +81,17 @@ protected: std::list m_nodeEventHandler; std::map m_events; }; + +class NodeTable; +inline std::ostream& operator<<(std::ostream& _out, NodeTable const& _nodeTable); /** * NodeTable using modified kademlia for node discovery and preference. - * untouched buckets are refreshed if they have not been touched within an hour + * Node table requires an IO service, creates a socket for incoming + * UDP messages and implements a kademlia-like protocol. Node requests and + * responses are used to build a node table which can be queried to + * obtain a list of potential nodes to connect to, and, passes events to + * Host whenever a node is added or removed to/from the table. * * Thread-safety is ensured by modifying NodeEntry details via * shared_ptr replacement instead of mutating values. @@ -101,7 +109,7 @@ protected: * @todo serialize evictions per-bucket * @todo store evictions in map, unit-test eviction logic * @todo store root node in table - * @todo encapsulate doFindNode into NetworkAlgorithm (task) + * @todo encapsulate discover into NetworkAlgorithm (task) * @todo Pong to include ip:port where ping was received * @todo expiration and sha3(id) 'to' for messages which are replies (prevents replay) * @todo cache Ping and FindSelf @@ -117,33 +125,17 @@ protected: */ class NodeTable: UDPSocketEvents, public std::enable_shared_from_this { + friend std::ostream& operator<<(std::ostream& _out, NodeTable const& _nodeTable); using NodeSocket = UDPSocket; using TimePoint = std::chrono::steady_clock::time_point; - using EvictionTimeout = std::pair,NodeId>; ///< First NodeId may be evicted and replaced with second NodeId. + using EvictionTimeout = std::pair, NodeId>; ///< First NodeId may be evicted and replaced with second NodeId. public: NodeTable(ba::io_service& _io, KeyPair _alias, uint16_t _udpPort = 30303); ~NodeTable(); - /// Constants for Kademlia, derived from address space. - - static unsigned const s_addressByteSize = sizeof(NodeId); ///< Size of address type in bytes. - static unsigned const s_bits = 8 * s_addressByteSize; ///< Denoted by n in [Kademlia]. - static unsigned const s_bins = s_bits - 1; ///< Size of m_state (excludes root, which is us). - static unsigned const s_maxSteps = boost::static_log2::value; ///< Max iterations of discovery. (doFindNode) - - /// Chosen constants - - static unsigned const s_bucketSize = 16; ///< Denoted by k in [Kademlia]. Number of nodes stored in each bucket. - static unsigned const s_alpha = 3; ///< Denoted by \alpha in [Kademlia]. Number of concurrent FindNode requests. - - /// Intervals - - boost::posix_time::milliseconds const c_evictionCheckInterval = boost::posix_time::milliseconds(75); ///< Interval at which eviction timeouts are checked. - std::chrono::milliseconds const c_reqTimeout = std::chrono::milliseconds(300); ///< How long to wait for requests (evict, find iterations). - std::chrono::seconds const c_bucketRefresh = std::chrono::seconds(3600); ///< Refresh interval prevents bucket from becoming stale. [Kademlia] - - static unsigned dist(NodeId const& _a, NodeId const& _b) { u512 d = _a ^ _b; unsigned ret; for (ret = 0; d >>= 1; ++ret) {}; return ret; } + /// Returns distance based on xor metric two node ids. Used by NodeEntry and NodeTable. + static unsigned distance(NodeId const& _a, NodeId const& _b) { u512 d = _a ^ _b; unsigned ret; for (ret = 0; d >>= 1; ++ret) {}; return ret; } /// Set event handler for NodeEntryAdded and NodeEntryRemoved events. void setEventHandler(NodeTableEventHandler* _handler) { m_nodeEventHandler.reset(_handler); } @@ -157,18 +149,49 @@ public: /// Add node. Node will be pinged if it's not already known. std::shared_ptr addNode(Node const& _node); - void join(); + /// To be called when node table is empty. Runs node discovery with m_node.id as the target in order to populate node-table. + void discover(); - NodeEntry root() const { return NodeEntry(m_node, m_node.publicKey(), m_node.endpoint.udp); } + /// Returns list of node ids active in node table. std::list nodes() const; - unsigned size() const { return m_nodes.size(); } - std::list state() const; + /// Returns node count. + unsigned count() const { return m_nodes.size(); } + + /// Returns snapshot of table. + std::list snapshot() const; + + /// Returns true if node id is in node table. bool haveNode(NodeId _id) { Guard l(x_nodes); return m_nodes.count(_id); } - Node operator[](NodeId _id); - std::shared_ptr getNodeEntry(NodeId _id); + /// Returns the Node to the corresponding node id or the empty Node if that id is not found. + Node node(NodeId _id); + +#ifndef BOOST_AUTO_TEST_SUITE +private: +#else protected: +#endif + + /// Constants for Kademlia, derived from address space. + + static unsigned const s_addressByteSize = sizeof(NodeId); ///< Size of address type in bytes. + static unsigned const s_bits = 8 * s_addressByteSize; ///< Denoted by n in [Kademlia]. + static unsigned const s_bins = s_bits - 1; ///< Size of m_state (excludes root, which is us). + static unsigned const s_maxSteps = boost::static_log2::value; ///< Max iterations of discovery. (discover) + + /// Chosen constants + + static unsigned const s_bucketSize = 16; ///< Denoted by k in [Kademlia]. Number of nodes stored in each bucket. + static unsigned const s_alpha = 3; ///< Denoted by \alpha in [Kademlia]. Number of concurrent FindNode requests. + + /// Intervals + + /* todo: replace boost::posix_time; change constants to upper camelcase */ + boost::posix_time::milliseconds const c_evictionCheckInterval = boost::posix_time::milliseconds(75); ///< Interval at which eviction timeouts are checked. + std::chrono::milliseconds const c_reqTimeout = std::chrono::milliseconds(300); ///< How long to wait for requests (evict, find iterations). + std::chrono::seconds const c_bucketRefresh = std::chrono::seconds(3600); ///< Refresh interval prevents bucket from becoming stale. [Kademlia] + struct NodeBucket { unsigned distance; @@ -176,81 +199,95 @@ protected: std::list> nodes; }; - /// Repeatedly sends s_alpha concurrent requests to nodes nearest to target, for nodes nearest to target, up to s_maxSteps rounds. - void doFindNode(NodeId _node, unsigned _round = 0, std::shared_ptr>> _tried = std::shared_ptr>>()); - - /// Returns nodes nearest to target. - std::vector> findNearest(NodeId _target); - + /// Used to ping endpoint. void ping(bi::udp::endpoint _to) const; + /// Used ping known node. Used by node table when refreshing buckets and as part of eviction process (see evict). void ping(NodeEntry* _n) const; - void evict(std::shared_ptr _leastSeen, std::shared_ptr _new); + /// Returns center node entry which describes this node and used with dist() to calculate xor metric for node table nodes. + NodeEntry center() const { return NodeEntry(m_node, m_node.publicKey(), m_node.endpoint.udp); } + + /// Used by asynchronous operations to return NodeEntry which is active and managed by node table. + std::shared_ptr nodeEntry(NodeId _id); - void noteNode(Public const& _pubk, bi::udp::endpoint const& _endpoint); + /// Used to discovery nodes on network which are close to the given target. + /// Sends s_alpha concurrent requests to nodes nearest to target, for nodes nearest to target, up to s_maxSteps rounds. + void discover(NodeId _target, unsigned _round = 0, std::shared_ptr>> _tried = std::shared_ptr>>()); + + /// Returns nodes from node table which are closest to target. + std::vector> nearestNodeEntries(NodeId _target); - void noteNode(std::shared_ptr _n); + /// Asynchronously drops _leastSeen node if it doesn't reply and adds _new node, otherwise _new node is thrown away. + void evict(std::shared_ptr _leastSeen, std::shared_ptr _new); + /// Called whenever activity is received from an unknown node in order to maintain node table. + void noteActiveNode(Public const& _pubk, bi::udp::endpoint const& _endpoint); + + /// Used to drop node when timeout occurs or when evict() result is to keep previous node. void dropNode(std::shared_ptr _n); - NodeBucket& bucket(NodeEntry const* _n); + /// Returns references to bucket which corresponds to distance of node id. + /// @warning Only use the return reference locked x_state mutex. + // TODO p2p: Remove this method after removing offset-by-one functionality. + NodeBucket& bucket_UNSAFE(NodeEntry const* _n); /// General Network Events + /// Called by m_socket when packet is received. void onReceived(UDPSocketFace*, bi::udp::endpoint const& _from, bytesConstRef _packet); - void onDisconnected(UDPSocketFace*) {}; + /// Called by m_socket when socket is disconnected. + void onDisconnected(UDPSocketFace*) {} /// Tasks + /// Called by evict() to ensure eviction check is scheduled to run and terminates when no evictions remain. Asynchronous. void doCheckEvictions(boost::system::error_code const& _ec); - - void doRefreshBuckets(boost::system::error_code const& _ec); -#ifndef BOOST_AUTO_TEST_SUITE -private: -#else -protected: -#endif - /// Sends FindNeighbor packet. See doFindNode. - void requestNeighbours(NodeEntry const& _node, NodeId _target) const; + /// Purges and pings nodes for any buckets which haven't been touched for c_bucketRefresh seconds. + void doRefreshBuckets(boost::system::error_code const& _ec); - std::unique_ptr m_nodeEventHandler; ///< Event handler for node events. + std::unique_ptr m_nodeEventHandler; ///< Event handler for node events. Node m_node; ///< This node. Secret m_secret; ///< This nodes secret key. - mutable Mutex x_nodes; ///< Mutable for thread-safe copy in nodes() const. + mutable Mutex x_nodes; ///< LOCK x_state first if both locks are required. Mutable for thread-safe copy in nodes() const. std::map> m_nodes; ///< Nodes - mutable Mutex x_state; + mutable Mutex x_state; ///< LOCK x_state first if both x_nodes and x_state locks are required. std::array m_state; ///< State of p2p node network. - Mutex x_evictions; + Mutex x_evictions; ///< LOCK x_nodes first if both x_nodes and x_evictions locks are required. std::deque m_evictions; ///< Eviction timeouts. ba::io_service& m_io; ///< Used by bucket refresh timer. std::shared_ptr m_socket; ///< Shared pointer for our UDPSocket; ASIO requires shared_ptr. - NodeSocket* m_socketPtr; ///< Set to m_socket.get(). + NodeSocket* m_socketPointer; ///< Set to m_socket.get(). Socket is created in constructor and disconnected in destructor to ensure access to pointer is safe. boost::asio::deadline_timer m_bucketRefreshTimer; ///< Timer which schedules and enacts bucket refresh. boost::asio::deadline_timer m_evictionCheckTimer; ///< Timer for handling node evictions. }; - + inline std::ostream& operator<<(std::ostream& _out, NodeTable const& _nodeTable) { - _out << _nodeTable.root().address() << "\t" << "0\t" << _nodeTable.root().endpoint.udp.address() << ":" << _nodeTable.root().endpoint.udp.port() << std::endl; - auto s = _nodeTable.state(); + _out << _nodeTable.center().address() << "\t" << "0\t" << _nodeTable.center().endpoint.udp.address() << ":" << _nodeTable.center().endpoint.udp.port() << std::endl; + auto s = _nodeTable.snapshot(); for (auto n: s) _out << n.address() << "\t" << n.distance << "\t" << n.endpoint.udp.address() << ":" << n.endpoint.udp.port() << std::endl; return _out; } /** - * Ping packet: Check if node is alive. + * Ping packet: Sent to check if node is alive. * PingNode is cached and regenerated after expiration - t, where t is timeout. * + * Ping is used to implement evict. When a new node is seen for + * a given bucket which is full, the least-responsive node is pinged. + * If the pinged node doesn't respond, then it is removed and the new + * node is inserted. + * * RLP Encoded Items: 3 * Minimum Encoded Size: 18 bytes * Maximum Encoded Size: bytes // todo after u128 addresses @@ -260,11 +297,6 @@ inline std::ostream& operator<<(std::ostream& _out, NodeTable const& _nodeTable) * port: Our port. * expiration: Triggers regeneration of packet. May also provide control over synchronization. * - * Ping is used to implement evict. When a new node is seen for - * a given bucket which is full, the least-responsive node is pinged. - * If the pinged node doesn't respond then it is removed and the new - * node is inserted. - * * @todo uint128_t for ip address (<->integer ipv4/6, asio-address, asio-endpoint) * */ @@ -285,7 +317,7 @@ struct PingNode: RLPXDatagram }; /** - * Pong packet: response to ping + * Pong packet: Sent in response to ping * * RLP Encoded Items: 2 * Minimum Encoded Size: 33 bytes @@ -365,7 +397,7 @@ struct Neighbours: RLPXDatagram } static const uint8_t type = 4; - std::list nodes; + std::vector nodes; unsigned expiration = 1; void streamRLP(RLPStream& _s) const { _s.appendList(2); _s.appendList(nodes.size()); for (auto& n: nodes) n.streamRLP(_s); _s << expiration; } diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index 4a6621d09..1d1d69b7d 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -48,7 +48,7 @@ Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr const& Session::~Session() { - m_peer->lastConnected = m_peer->lastAttempted - chrono::seconds(1); + m_peer->m_lastConnected = m_peer->m_lastAttempted - chrono::seconds(1); // Read-chain finished for one reason or another. for (auto& i: m_capabilities) @@ -75,14 +75,14 @@ void Session::addRating(unsigned _r) { if (m_peer) { - m_peer->rating += _r; - m_peer->score += _r; + m_peer->m_rating += _r; + m_peer->m_score += _r; } } int Session::rating() const { - return m_peer->rating; + return m_peer->m_rating; } template vector randomSelection(vector const& _t, unsigned _n) @@ -206,7 +206,7 @@ bool Session::interpret(RLP const& _r) } if (m_peer->isOffline()) - m_peer->lastConnected = chrono::system_clock::now(); + m_peer->m_lastConnected = chrono::system_clock::now(); if (m_protocolVersion != m_server->protocolVersion()) { @@ -446,13 +446,13 @@ void Session::drop(DisconnectReason _reason) if (m_peer) { - if (_reason != m_peer->lastDisconnect || _reason == NoDisconnect || _reason == ClientQuit || _reason == DisconnectRequested) - m_peer->failedAttempts = 0; - m_peer->lastDisconnect = _reason; + if (_reason != m_peer->m_lastDisconnect || _reason == NoDisconnect || _reason == ClientQuit || _reason == DisconnectRequested) + m_peer->m_failedAttempts = 0; + m_peer->m_lastDisconnect = _reason; if (_reason == BadProtocol) { - m_peer->rating /= 2; - m_peer->score /= 2; + m_peer->m_rating /= 2; + m_peer->m_score /= 2; } } m_dropped = true; diff --git a/libp2p/UDP.h b/libp2p/UDP.h index bf9a6a372..5c3b9362f 100644 --- a/libp2p/UDP.h +++ b/libp2p/UDP.h @@ -64,12 +64,12 @@ struct RLPXDatagramFace: public UDPDatagram static uint64_t futureFromEpoch(std::chrono::seconds _sec) { return std::chrono::duration_cast((std::chrono::system_clock::now() + _sec).time_since_epoch()).count(); } static Public authenticate(bytesConstRef _sig, bytesConstRef _rlp); - virtual uint8_t packetType() =0; + virtual uint8_t packetType() = 0; RLPXDatagramFace(bi::udp::endpoint const& _ep): UDPDatagram(_ep) {} virtual h256 sign(Secret const& _from); - virtual void streamRLP(RLPStream&) const =0; - virtual void interpretRLP(bytesConstRef _bytes) =0; + virtual void streamRLP(RLPStream&) const = 0; + virtual void interpretRLP(bytesConstRef _bytes) = 0; }; template diff --git a/libwebthree/WebThree.cpp b/libwebthree/WebThree.cpp index 7daec9693..3075108d1 100644 --- a/libwebthree/WebThree.cpp +++ b/libwebthree/WebThree.cpp @@ -77,7 +77,7 @@ void WebThreeDirect::setNetworkPreferences(p2p::NetworkPreferences const& _n) std::vector WebThreeDirect::peers() { - return m_net.peers(); + return m_net.peerSessionInfo(); } size_t WebThreeDirect::peerCount() const diff --git a/libwebthree/WebThree.h b/libwebthree/WebThree.h index f99eed5cf..1bce0820c 100644 --- a/libwebthree/WebThree.h +++ b/libwebthree/WebThree.h @@ -145,7 +145,7 @@ public: p2p::NodeId id() const override { return m_net.id(); } /// Gets the nodes. - p2p::Peers nodes() const override { return m_net.nodes(); } + p2p::Peers nodes() const override { return m_net.getPeers(); } /// Start the network subsystem. void startNetwork() override { m_net.start(); } diff --git a/test/net.cpp b/test/net.cpp index 7f30ed03b..5a7e56d23 100644 --- a/test/net.cpp +++ b/test/net.cpp @@ -87,7 +87,7 @@ struct TestNodeTable: public NodeTable bi::address ourIp = bi::address::from_string("127.0.0.1"); for (auto& n: _testNodes) if (_count--) - noteNode(n.first.pub(), bi::udp::endpoint(ourIp, n.second)); + noteActiveNode(n.first.pub(), bi::udp::endpoint(ourIp, n.second)); else break; } @@ -182,7 +182,7 @@ BOOST_AUTO_TEST_CASE(kademlia) // Not yet a 'real' test. TestNodeTableHost node(8); node.start(); - node.nodeTable->join(); // ideally, joining with empty node table logs warning we can check for + node.nodeTable->discover(); // ideally, joining with empty node table logs warning we can check for node.setup(); node.populate(); clog << "NodeTable:\n" << *node.nodeTable.get() << endl; @@ -199,11 +199,11 @@ BOOST_AUTO_TEST_CASE(kademlia) node.populate(1); clog << "NodeTable:\n" << *node.nodeTable.get() << endl; - node.nodeTable->join(); + node.nodeTable->discover(); this_thread::sleep_for(chrono::milliseconds(2000)); clog << "NodeTable:\n" << *node.nodeTable.get() << endl; - BOOST_REQUIRE_EQUAL(node.nodeTable->size(), 8); + BOOST_REQUIRE_EQUAL(node.nodeTable->count(), 8); auto netNodes = node.nodeTable->nodes(); netNodes.sort(); diff --git a/test/whisperTopic.cpp b/test/whisperTopic.cpp index 1dcc6b9d6..e59a3f289 100644 --- a/test/whisperTopic.cpp +++ b/test/whisperTopic.cpp @@ -111,7 +111,7 @@ BOOST_AUTO_TEST_CASE(forwarding) setThreadName("listener"); // Host must be configured not to share peers. - Host ph("Listner", NetworkPreferences(50303, "", false, true)); + Host ph("Listner", NetworkPreferences(30303, "", false, true)); ph.setIdealPeerCount(0); auto wh = ph.registerCapability(new WhisperHost()); ph.start(); @@ -145,7 +145,7 @@ BOOST_AUTO_TEST_CASE(forwarding) this_thread::sleep_for(chrono::milliseconds(50)); // Host must be configured not to share peers. - Host ph("Forwarder", NetworkPreferences(50305, "", false, true)); + Host ph("Forwarder", NetworkPreferences(30305, "", false, true)); ph.setIdealPeerCount(0); auto wh = ph.registerCapability(new WhisperHost()); this_thread::sleep_for(chrono::milliseconds(500)); @@ -153,7 +153,7 @@ BOOST_AUTO_TEST_CASE(forwarding) fwderid = ph.id(); this_thread::sleep_for(chrono::milliseconds(500)); - ph.addNode(phid, "127.0.0.1", 50303, 50303); + ph.addNode(phid, "127.0.0.1", 30303, 30303); startedForwarder = true; @@ -174,13 +174,13 @@ BOOST_AUTO_TEST_CASE(forwarding) while (!startedForwarder) this_thread::sleep_for(chrono::milliseconds(50)); - Host ph("Sender", NetworkPreferences(50300, "", false, true)); + Host ph("Sender", NetworkPreferences(30300, "", false, true)); ph.setIdealPeerCount(0); shared_ptr wh = ph.registerCapability(new WhisperHost()); this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); this_thread::sleep_for(chrono::milliseconds(500)); - ph.addNode(fwderid, "127.0.0.1", 50305, 50305); + ph.addNode(fwderid, "127.0.0.1", 30305, 30305); KeyPair us = KeyPair::create(); wh->post(us.sec(), RLPStream().append(1).out(), BuildTopic("test")); @@ -210,14 +210,14 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) setThreadName("forwarder"); // Host must be configured not to share peers. - Host ph("Forwarder", NetworkPreferences(50305, "", false, true)); + Host ph("Forwarder", NetworkPreferences(30305, "", false, true)); ph.setIdealPeerCount(0); auto wh = ph.registerCapability(new WhisperHost()); this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); this_thread::sleep_for(chrono::milliseconds(500)); -// ph.addNode("127.0.0.1", 50303, 50303); +// ph.addNode("127.0.0.1", 30303, 30303); startedForwarder = true; @@ -239,13 +239,13 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) this_thread::sleep_for(chrono::milliseconds(50)); { - Host ph("Sender", NetworkPreferences(50300, "", false, true)); + Host ph("Sender", NetworkPreferences(30300, "", false, true)); ph.setIdealPeerCount(0); shared_ptr wh = ph.registerCapability(new WhisperHost()); this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); this_thread::sleep_for(chrono::milliseconds(500)); -// ph.addNode("127.0.0.1", 50305, 50305); +// ph.addNode("127.0.0.1", 30305, 30305); KeyPair us = KeyPair::create(); wh->post(us.sec(), RLPStream().append(1).out(), BuildTopic("test")); @@ -253,13 +253,13 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) } { - Host ph("Listener", NetworkPreferences(50300, "", false, true)); + Host ph("Listener", NetworkPreferences(30300, "", false, true)); ph.setIdealPeerCount(0); shared_ptr wh = ph.registerCapability(new WhisperHost()); this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); this_thread::sleep_for(chrono::milliseconds(500)); -// ph.addNode("127.0.0.1", 50305, 50305); +// ph.addNode("127.0.0.1", 30305, 30305); /// Only interested in odd packets auto w = wh->installWatch(BuildTopicMask("test")); From 8c2daa4195c0a9acc4f10396b457628edbc60640 Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 6 Feb 2015 13:55:01 -0800 Subject: [PATCH 35/71] update whisper topic and forwarding tests for node-discovery --- test/whisperTopic.cpp | 87 +++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/test/whisperTopic.cpp b/test/whisperTopic.cpp index e59a3f289..d2f2d9d89 100644 --- a/test/whisperTopic.cpp +++ b/test/whisperTopic.cpp @@ -36,9 +36,12 @@ BOOST_AUTO_TEST_CASE(topic) auto oldLogVerbosity = g_logVerbosity; g_logVerbosity = 0; - Host phOther("Test", NetworkPreferences(30303, "127.0.0.1", false, true)); - auto whOther = phOther.registerCapability(new WhisperHost()); - phOther.start(); + Host host1("Test", NetworkPreferences(30303, "127.0.0.1", false, true)); + auto whost1 = host1.registerCapability(new WhisperHost()); + host1.start(); + + while (!host1.isStarted()) + this_thread::sleep_for(chrono::milliseconds(2)); bool started = false; unsigned result = 0; @@ -48,16 +51,16 @@ BOOST_AUTO_TEST_CASE(topic) started = true; /// Only interested in odd packets - auto w = whOther->installWatch(BuildTopicMask("odd")); + auto w = whost1->installWatch(BuildTopicMask("odd")); started = true; set received; for (int iterout = 0, last = 0; iterout < 200 && last < 81; ++iterout) { - for (auto i: whOther->checkWatch(w)) + for (auto i: whost1->checkWatch(w)) { - Message msg = whOther->envelope(i).open(whOther->fullTopic(w)); + Message msg = whost1->envelope(i).open(whost1->fullTopic(w)); last = RLP(msg.payload()).toInt(); if (received.count(last)) continue; @@ -70,12 +73,15 @@ BOOST_AUTO_TEST_CASE(topic) }); - Host ph("Test", NetworkPreferences(30300, "127.0.0.1", false, true)); - auto wh = ph.registerCapability(new WhisperHost()); - ph.start(); + Host host2("Test", NetworkPreferences(30300, "127.0.0.1", false, true)); + auto whost2 = host2.registerCapability(new WhisperHost()); + host2.start(); + + while (!host2.isStarted()) + this_thread::sleep_for(chrono::milliseconds(2)); this_thread::sleep_for(chrono::milliseconds(100)); - ph.addNode(phOther.id(), "127.0.0.1", 30303, 30303); + host2.addNode(host1.id(), "127.0.0.1", 30303, 30303); this_thread::sleep_for(chrono::milliseconds(500)); @@ -85,7 +91,7 @@ BOOST_AUTO_TEST_CASE(topic) KeyPair us = KeyPair::create(); for (int i = 0; i < 10; ++i) { - wh->post(us.sec(), RLPStream().append(i * i).out(), BuildTopic(i)(i % 2 ? "odd" : "even")); + whost2->post(us.sec(), RLPStream().append(i * i).out(), BuildTopic(i)(i % 2 ? "odd" : "even")); this_thread::sleep_for(chrono::milliseconds(250)); } @@ -100,33 +106,33 @@ BOOST_AUTO_TEST_CASE(forwarding) cnote << "Testing Whisper forwarding..."; auto oldLogVerbosity = g_logVerbosity; g_logVerbosity = 0; - + + // Host must be configured not to share peers. + Host host1("Listner", NetworkPreferences(30303, "", false, true)); + host1.setIdealPeerCount(0); + auto whost1 = host1.registerCapability(new WhisperHost()); + host1.start(); + while (!host1.isStarted()) + this_thread::sleep_for(chrono::milliseconds(2)); + unsigned result = 0; bool done = false; bool startedListener = false; - Public phid; std::thread listener([&]() { setThreadName("listener"); - // Host must be configured not to share peers. - Host ph("Listner", NetworkPreferences(30303, "", false, true)); - ph.setIdealPeerCount(0); - auto wh = ph.registerCapability(new WhisperHost()); - ph.start(); - phid = ph.id(); - startedListener = true; /// Only interested in odd packets - auto w = wh->installWatch(BuildTopicMask("test")); + auto w = whost1->installWatch(BuildTopicMask("test")); for (int i = 0; i < 200 && !result; ++i) { - for (auto i: wh->checkWatch(w)) + for (auto i: whost1->checkWatch(w)) { - Message msg = wh->envelope(i).open(wh->fullTopic(w)); + Message msg = whost1->envelope(i).open(whost1->fullTopic(w)); unsigned last = RLP(msg.payload()).toInt(); cnote << "New message from:" << msg.from().abridged() << RLP(msg.payload()).toInt(); result = last; @@ -135,8 +141,17 @@ BOOST_AUTO_TEST_CASE(forwarding) } }); - bool startedForwarder = false; + + // Host must be configured not to share peers. + Host host2("Forwarder", NetworkPreferences(30305, "", false, true)); + host2.setIdealPeerCount(1); + auto whost2 = host2.registerCapability(new WhisperHost()); + host2.start(); + while (!host2.isStarted()) + this_thread::sleep_for(chrono::milliseconds(2)); + Public fwderid; + bool startedForwarder = false; std::thread forwarder([&]() { setThreadName("forwarder"); @@ -144,27 +159,19 @@ BOOST_AUTO_TEST_CASE(forwarding) while (!startedListener) this_thread::sleep_for(chrono::milliseconds(50)); - // Host must be configured not to share peers. - Host ph("Forwarder", NetworkPreferences(30305, "", false, true)); - ph.setIdealPeerCount(0); - auto wh = ph.registerCapability(new WhisperHost()); this_thread::sleep_for(chrono::milliseconds(500)); - ph.start(); - fwderid = ph.id(); - - this_thread::sleep_for(chrono::milliseconds(500)); - ph.addNode(phid, "127.0.0.1", 30303, 30303); + host2.addNode(host1.id(), "127.0.0.1", 30303, 30303); startedForwarder = true; /// Only interested in odd packets - auto w = wh->installWatch(BuildTopicMask("test")); + auto w = whost2->installWatch(BuildTopicMask("test")); while (!done) { - for (auto i: wh->checkWatch(w)) + for (auto i: whost2->checkWatch(w)) { - Message msg = wh->envelope(i).open(wh->fullTopic(w)); + Message msg = whost2->envelope(i).open(whost2->fullTopic(w)); cnote << "New message from:" << msg.from().abridged() << RLP(msg.payload()).toInt(); } this_thread::sleep_for(chrono::milliseconds(50)); @@ -175,12 +182,12 @@ BOOST_AUTO_TEST_CASE(forwarding) this_thread::sleep_for(chrono::milliseconds(50)); Host ph("Sender", NetworkPreferences(30300, "", false, true)); - ph.setIdealPeerCount(0); + ph.setIdealPeerCount(1); shared_ptr wh = ph.registerCapability(new WhisperHost()); - this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); - this_thread::sleep_for(chrono::milliseconds(500)); - ph.addNode(fwderid, "127.0.0.1", 30305, 30305); + ph.addNode(host2.id(), "127.0.0.1", 30305, 30305); + while (!ph.isStarted()) + this_thread::sleep_for(chrono::milliseconds(10)); KeyPair us = KeyPair::create(); wh->post(us.sec(), RLPStream().append(1).out(), BuildTopic("test")); From fc15ac8071ccce6724b1ec33ff1bf2907f72dcfd Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 6 Feb 2015 14:58:23 -0800 Subject: [PATCH 36/71] update remaining whisper tests --- test/whisperTopic.cpp | 48 +++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/test/whisperTopic.cpp b/test/whisperTopic.cpp index d2f2d9d89..ab0cdc115 100644 --- a/test/whisperTopic.cpp +++ b/test/whisperTopic.cpp @@ -209,19 +209,19 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) unsigned result = 0; bool done = false; + + // Host must be configured not to share peers. + Host host1("Forwarder", NetworkPreferences(30305, "", false, true)); + host1.setIdealPeerCount(1); + auto whost1 = host1.registerCapability(new WhisperHost()); + host1.start(); + while (!host1.isStarted()) + this_thread::sleep_for(chrono::milliseconds(2)); - Public listenerid; bool startedForwarder = false; std::thread forwarder([&]() { setThreadName("forwarder"); - - // Host must be configured not to share peers. - Host ph("Forwarder", NetworkPreferences(30305, "", false, true)); - ph.setIdealPeerCount(0); - auto wh = ph.registerCapability(new WhisperHost()); - this_thread::sleep_for(chrono::milliseconds(500)); - ph.start(); this_thread::sleep_for(chrono::milliseconds(500)); // ph.addNode("127.0.0.1", 30303, 30303); @@ -229,13 +229,13 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) startedForwarder = true; /// Only interested in odd packets - auto w = wh->installWatch(BuildTopicMask("test")); + auto w = whost1->installWatch(BuildTopicMask("test")); while (!done) { - for (auto i: wh->checkWatch(w)) + for (auto i: whost1->checkWatch(w)) { - Message msg = wh->envelope(i).open(wh->fullTopic(w)); + Message msg = whost1->envelope(i).open(whost1->fullTopic(w)); cnote << "New message from:" << msg.from().abridged() << RLP(msg.payload()).toInt(); } this_thread::sleep_for(chrono::milliseconds(50)); @@ -245,28 +245,28 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) while (!startedForwarder) this_thread::sleep_for(chrono::milliseconds(50)); + Host host2("Sender", NetworkPreferences(30300, "", false, true)); + host2.setIdealPeerCount(1); + shared_ptr whost2 = host2.registerCapability(new WhisperHost()); + host2.start(); + while (!host2.isStarted()) + this_thread::sleep_for(chrono::milliseconds(2)); + host2.addNode(host1.id(), "127.0.0.1", 30305, 30305); + { - Host ph("Sender", NetworkPreferences(30300, "", false, true)); - ph.setIdealPeerCount(0); - shared_ptr wh = ph.registerCapability(new WhisperHost()); - this_thread::sleep_for(chrono::milliseconds(500)); - ph.start(); - this_thread::sleep_for(chrono::milliseconds(500)); -// ph.addNode("127.0.0.1", 30305, 30305); - KeyPair us = KeyPair::create(); - wh->post(us.sec(), RLPStream().append(1).out(), BuildTopic("test")); + whost2->post(us.sec(), RLPStream().append(1).out(), BuildTopic("test")); this_thread::sleep_for(chrono::milliseconds(250)); } { Host ph("Listener", NetworkPreferences(30300, "", false, true)); - ph.setIdealPeerCount(0); + ph.setIdealPeerCount(1); shared_ptr wh = ph.registerCapability(new WhisperHost()); - this_thread::sleep_for(chrono::milliseconds(500)); ph.start(); - this_thread::sleep_for(chrono::milliseconds(500)); -// ph.addNode("127.0.0.1", 30305, 30305); + while (!ph.isStarted()) + this_thread::sleep_for(chrono::milliseconds(2)); + ph.addNode(host1.id(), "127.0.0.1", 30305, 30305); /// Only interested in odd packets auto w = wh->installWatch(BuildTopicMask("test")); From eae64afe21c97cf4e76e7d52875e8538f1e675a2 Mon Sep 17 00:00:00 2001 From: subtly Date: Fri, 6 Feb 2015 16:49:35 -0800 Subject: [PATCH 37/71] update packet test --- test/net.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/net.cpp b/test/net.cpp index 5a7e56d23..5039c5436 100644 --- a/test/net.cpp +++ b/test/net.cpp @@ -152,7 +152,7 @@ BOOST_AUTO_TEST_CASE(test_neighbours_packet) out.sign(k.sec()); bytesConstRef packet(out.data.data(), out.data.size()); - bytesConstRef rlpBytes(packet.cropped(97, packet.size() - 97)); + bytesConstRef rlpBytes(packet.cropped(h256::size + Signature::size + 1)); Neighbours in = Neighbours::fromBytesConstRef(to, rlpBytes); int count = 0; for (auto n: in.nodes) From eb2a4e375a8b8612342f6d03011a14d3ec3e5db6 Mon Sep 17 00:00:00 2001 From: subtly Date: Mon, 9 Feb 2015 00:35:25 -0500 Subject: [PATCH 38/71] update for cr. update whisper test. --- libp2p/Host.h | 73 ++------------------------------------ libp2p/Peer.cpp | 57 ++++++++++++++++++++++++++++++ libp2p/Peer.h | 82 +++++++++++++++++++++++++++++++++++++++++++ test/whisperTopic.cpp | 19 +++++----- 4 files changed, 152 insertions(+), 79 deletions(-) create mode 100644 libp2p/Peer.cpp create mode 100644 libp2p/Peer.h diff --git a/libp2p/Host.h b/libp2p/Host.h index b24f1343c..baf8f0585 100644 --- a/libp2p/Host.h +++ b/libp2p/Host.h @@ -37,6 +37,7 @@ #include "NodeTable.h" #include "HostCapability.h" #include "Network.h" +#include "Peer.h" #include "Common.h" namespace ba = boost::asio; namespace bi = ba::ip; @@ -44,81 +45,11 @@ namespace bi = ba::ip; namespace dev { -class RLPStream; - namespace p2p { class Host; -/** - * @brief Representation of connectivity state and all other pertinent Peer metadata. - * A Peer represents connectivity between two nodes, which in this case, are the host - * and remote nodes. - * - * State information necessary for loading network topology is maintained by NodeTable. - * - * @todo Implement 'bool required' - * @todo reputation: Move score, rating to capability-specific map (&& remove friend class) - * @todo reputation: implement via origin-tagged events - * @todo Populate metadata upon construction; save when destroyed. - * @todo Metadata for peers needs to be handled via a storage backend. - * Specifically, peers can be utilized in a variety of - * many-to-many relationships while also needing to modify shared instances of - * those peers. Modifying these properties via a storage backend alleviates - * Host of the responsibility. (&& remove save/restoreNetwork) - * @todo reimplement recording of historical session information on per-transport basis - * @todo rebuild nodetable when localNetworking is enabled/disabled - * @todo move attributes into protected - */ -class Peer: public Node -{ - friend class Session; /// Allows Session to update score and rating. - friend class Host; /// For Host: saveNetwork(), restoreNetwork() -public: - bool isOffline() const { return !m_session.lock(); } - - bi::tcp::endpoint const& peerEndpoint() const { return endpoint.tcp; } - - int m_score = 0; ///< All time cumulative. - int m_rating = 0; ///< Trending. - - /// Network Availability - - std::chrono::system_clock::time_point m_lastConnected; - std::chrono::system_clock::time_point m_lastAttempted; - unsigned m_failedAttempts = 0; - DisconnectReason m_lastDisconnect = NoDisconnect; ///< Reason for disconnect that happened last. - - virtual bool operator<(Peer const& _p) const - { - if (isOffline() != _p.isOffline()) - return isOffline(); - else if (isOffline()) - if (m_lastAttempted == _p.m_lastAttempted) - return m_failedAttempts < _p.m_failedAttempts; - else - return m_lastAttempted < _p.m_lastAttempted; - else - if (m_score == _p.m_score) - if (m_rating == _p.m_rating) - if (m_failedAttempts == _p.m_failedAttempts) - return id < _p.id; - else - return m_failedAttempts < _p.m_failedAttempts; - else - return m_rating < _p.m_rating; - else - return m_score < _p.m_score; - } - -protected: - /// Used by isOffline() and (todo) for peer to emit session information. - std::weak_ptr m_session; -}; -using Peers = std::vector; - - class HostNodeTableHandler: public NodeTableEventHandler { public: @@ -131,7 +62,7 @@ private: Host& m_host; }; - + /** * @brief The Host class * Capabilities should be registered prior to startNetwork, since m_capabilities is not thread-safe. diff --git a/libp2p/Peer.cpp b/libp2p/Peer.cpp new file mode 100644 index 000000000..1811da930 --- /dev/null +++ b/libp2p/Peer.cpp @@ -0,0 +1,57 @@ +/* + This file is part of cpp-ethereum. + + cpp-ethereum is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + cpp-ethereum is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with cpp-ethereum. If not, see . +*/ +/** @file Peer.cpp + * @author Alex Leverington + * @author Gav Wood + * @date 2014 + */ + +#include "Peer.h" +using namespace std; +using namespace dev; +using namespace dev::p2p; + +namespace dev +{ + +namespace p2p +{ + +bool Peer::operator<(Peer const& _p) const +{ + if (isOffline() != _p.isOffline()) + return isOffline(); + else if (isOffline()) + if (m_lastAttempted == _p.m_lastAttempted) + return m_failedAttempts < _p.m_failedAttempts; + else + return m_lastAttempted < _p.m_lastAttempted; + else + if (m_score == _p.m_score) + if (m_rating == _p.m_rating) + if (m_failedAttempts == _p.m_failedAttempts) + return id < _p.id; + else + return m_failedAttempts < _p.m_failedAttempts; + else + return m_rating < _p.m_rating; + else + return m_score < _p.m_score; +} + +} +} diff --git a/libp2p/Peer.h b/libp2p/Peer.h new file mode 100644 index 000000000..f3db9d7e1 --- /dev/null +++ b/libp2p/Peer.h @@ -0,0 +1,82 @@ +/* + This file is part of cpp-ethereum. + + cpp-ethereum is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + cpp-ethereum is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with cpp-ethereum. If not, see . +*/ +/** @file Peer.h + * @author Alex Leverington + * @author Gav Wood + * @date 2014 + */ + +#pragma once + +#include "Common.h" + +namespace dev +{ + +namespace p2p +{ + +/** + * @brief Representation of connectivity state and all other pertinent Peer metadata. + * A Peer represents connectivity between two nodes, which in this case, are the host + * and remote nodes. + * + * State information necessary for loading network topology is maintained by NodeTable. + * + * @todo Implement 'bool required' + * @todo reputation: Move score, rating to capability-specific map (&& remove friend class) + * @todo reputation: implement via origin-tagged events + * @todo Populate metadata upon construction; save when destroyed. + * @todo Metadata for peers needs to be handled via a storage backend. + * Specifically, peers can be utilized in a variety of + * many-to-many relationships while also needing to modify shared instances of + * those peers. Modifying these properties via a storage backend alleviates + * Host of the responsibility. (&& remove save/restoreNetwork) + * @todo reimplement recording of historical session information on per-transport basis + * @todo rebuild nodetable when localNetworking is enabled/disabled + * @todo move attributes into protected + */ +class Peer: public Node +{ + friend class Session; /// Allows Session to update score and rating. + friend class Host; /// For Host: saveNetwork(), restoreNetwork() + +public: + bool isOffline() const { return !m_session.lock(); } + + bi::tcp::endpoint const& peerEndpoint() const { return endpoint.tcp; } + + virtual bool operator<(Peer const& _p) const; + +protected: + int m_score = 0; ///< All time cumulative. + int m_rating = 0; ///< Trending. + + /// Network Availability + + std::chrono::system_clock::time_point m_lastConnected; + std::chrono::system_clock::time_point m_lastAttempted; + unsigned m_failedAttempts = 0; + DisconnectReason m_lastDisconnect = NoDisconnect; ///< Reason for disconnect that happened last. + + /// Used by isOffline() and (todo) for peer to emit session information. + std::weak_ptr m_session; +}; +using Peers = std::vector; + +} +} diff --git a/test/whisperTopic.cpp b/test/whisperTopic.cpp index ab0cdc115..be93174ec 100644 --- a/test/whisperTopic.cpp +++ b/test/whisperTopic.cpp @@ -243,17 +243,20 @@ BOOST_AUTO_TEST_CASE(asyncforwarding) }); while (!startedForwarder) - this_thread::sleep_for(chrono::milliseconds(50)); - - Host host2("Sender", NetworkPreferences(30300, "", false, true)); - host2.setIdealPeerCount(1); - shared_ptr whost2 = host2.registerCapability(new WhisperHost()); - host2.start(); - while (!host2.isStarted()) this_thread::sleep_for(chrono::milliseconds(2)); - host2.addNode(host1.id(), "127.0.0.1", 30305, 30305); { + Host host2("Sender", NetworkPreferences(30300, "", false, true)); + host2.setIdealPeerCount(1); + shared_ptr whost2 = host2.registerCapability(new WhisperHost()); + host2.start(); + while (!host2.isStarted()) + this_thread::sleep_for(chrono::milliseconds(2)); + host2.addNode(host1.id(), "127.0.0.1", 30305, 30305); + + while (!host2.peerCount()) + this_thread::sleep_for(chrono::milliseconds(5)); + KeyPair us = KeyPair::create(); whost2->post(us.sec(), RLPStream().append(1).out(), BuildTopic("test")); this_thread::sleep_for(chrono::milliseconds(250)); From ff2682710887489b5a5b2a508ff67cfcd8d77080 Mon Sep 17 00:00:00 2001 From: subtly Date: Mon, 9 Feb 2015 02:00:21 -0500 Subject: [PATCH 39/71] update alethzero mainwin --- alethzero/MainWin.cpp | 4 ++-- libp2p/Peer.h | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index e9824ffce..465ae6877 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -965,8 +965,8 @@ void Main::refreshNetwork() .arg(QString::fromStdString(i.id.abridged())) .arg(QString::fromStdString(i.peerEndpoint().address().to_string())) .arg(i.id == web3()->id() ? "self" : sessions.count(i.id) ? sessions[i.id] : "disconnected") - .arg(i.isOffline() ? " | " + QString::fromStdString(reasonOf(i.lastDisconnect)) + " | " + QString::number(i.failedAttempts) + "x" : "") - .arg(i.rating) + .arg(i.isOffline() ? " | " + QString::fromStdString(reasonOf(i.lastDisconnect())) + " | " + QString::number(i.failedAttempts()) + "x" : "") + .arg(i.rating()) ); } } diff --git a/libp2p/Peer.h b/libp2p/Peer.h index f3db9d7e1..415573c0c 100644 --- a/libp2p/Peer.h +++ b/libp2p/Peer.h @@ -62,6 +62,15 @@ public: virtual bool operator<(Peer const& _p) const; + /// This peers rating. + int rating() const { return m_rating; } + + /// Number of times connection has been attempted to peer. + int failedAttempts() const { return m_failedAttempts; } + + /// Reason peer was previously disconnected. + DisconnectReason lastDisconnect() const { return m_lastDisconnect; } + protected: int m_score = 0; ///< All time cumulative. int m_rating = 0; ///< Trending. From a406402a4c9409b9c19421597f81dee8cb0158b6 Mon Sep 17 00:00:00 2001 From: subtly Date: Wed, 11 Feb 2015 01:03:13 -0500 Subject: [PATCH 40/71] Fixes for uninit. shared pointers and add const&. --- libp2p/Host.cpp | 29 ++++++++++++-------- libp2p/NodeTable.cpp | 64 ++++++++++++++++++++++++++++---------------- libp2p/NodeTable.h | 8 +++--- 3 files changed, 64 insertions(+), 37 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 25a095ae6..5ffb12c8f 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -185,15 +185,24 @@ void Host::onNodeTableEvent(NodeId const& _n, NodeTableEventType const& _e) auto n = m_nodeTable->node(_n); if (n) { - RecursiveGuard l(x_sessions); - auto p = m_peers[_n]; - if (!p) + shared_ptr p; { - m_peers[_n] = make_shared(); - p = m_peers[_n]; - p->id = _n; + RecursiveGuard l(x_sessions); + if (m_peers.count(_n)) + p = m_peers[_n]; + else + { + // TODO p2p: construct peer from node + p.reset(new Peer()); + p->id = _n; + p->endpoint = NodeIPEndpoint(n.endpoint.udp, n.endpoint.tcp); + p->required = n.required; + m_peers[_n] = p; + + clog(NetNote) << "p2p.host.peers.events.peersAdded " << _n << p->endpoint.tcp.address() << p->endpoint.udp.address(); + } + p->endpoint.tcp = n.endpoint.tcp; } - p->endpoint.tcp = n.endpoint.tcp; // TODO: Implement similar to discover. Attempt connecting to nodes // until ideal peer count is reached; if all nodes are tried, @@ -369,10 +378,8 @@ void Host::doHandshake(bi::tcp::socket* _socket, NodeId _nodeId) p = m_peers[_nodeId]; if (!p) - { - p = make_shared(); - p->endpoint.tcp.address(_socket->remote_endpoint().address()); - } + p.reset(new Peer()); + p->endpoint.tcp.address(_socket->remote_endpoint().address()); auto ps = std::make_shared(this, std::move(*_socket), p); ps->start(); diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 186ece5e2..3ccbca62d 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -70,28 +70,37 @@ shared_ptr NodeTable::addNode(Public const& _pubk, bi::udp::endpoint shared_ptr NodeTable::addNode(Node const& _node) { + // ping address if nodeid is empty + if (!_node.id) + { + PingNode p(_node.endpoint.udp, m_node.endpoint.udp.address().to_string(), m_node.endpoint.udp.port()); + p.sign(m_secret); + m_socketPointer->send(p); + shared_ptr n; + return move(n); + } + Guard l(x_nodes); - shared_ptr ret = m_nodes[_node.id]; - if (ret) + if (m_nodes.count(_node.id)) { - // TODO: p2p robust percolation of node-endpoint changes // // SECURITY: remove this in beta - it's only for lazy connections and presents an easy attack vector. // if (m_server->m_peers.count(id) && isPrivateAddress(m_server->m_peers.at(id)->address.address()) && ep.port() != 0) // // Update address if the node if we now have a public IP for it. // m_server->m_peers[id]->address = ep; + return m_nodes[_node.id]; } - else - { - clog(NodeTableNote) << "p2p.nodes.add " << _node.id.abridged(); - if (m_nodeEventHandler) - m_nodeEventHandler->appendEvent(_node.id, NodeEntryAdded); - - ret.reset(new NodeEntry(m_node, _node.id, NodeIPEndpoint(_node.endpoint.udp, _node.endpoint.tcp))); - m_nodes[_node.id] = ret; - PingNode p(_node.endpoint.udp, m_node.endpoint.udp.address().to_string(), m_node.endpoint.udp.port()); - p.sign(m_secret); - m_socketPointer->send(p); - } + + shared_ptr ret(new NodeEntry(m_node, _node.id, NodeIPEndpoint(_node.endpoint.udp, _node.endpoint.tcp))); + m_nodes[_node.id] = ret; + PingNode p(_node.endpoint.udp, m_node.endpoint.udp.address().to_string(), m_node.endpoint.udp.port()); + p.sign(m_secret); + m_socketPointer->send(p); + + // TODO p2p: rename to p2p.nodes.pending, add p2p.nodes.add event (when pong is received) + clog(NodeTableNote) << "p2p.nodes.add " << _node.id.abridged(); + if (m_nodeEventHandler) + m_nodeEventHandler->appendEvent(_node.id, NodeEntryAdded); + return move(ret); } @@ -119,18 +128,23 @@ list NodeTable::snapshot() const return move(ret); } -Node NodeTable::node(NodeId _id) +Node NodeTable::node(NodeId const& _id) { + // TODO p2p: eloquent copy operator Guard l(x_nodes); - auto n = m_nodes[_id]; - return !!n ? *n : Node(); + if (m_nodes.count(_id)) + { + auto entry = m_nodes[_id]; + Node n(_id, NodeIPEndpoint(entry->endpoint.udp, entry->endpoint.tcp), entry->required); + return move(n); + } + return move(Node()); } shared_ptr NodeTable::nodeEntry(NodeId _id) { Guard l(x_nodes); - auto n = m_nodes[_id]; - return !!n ? move(n) : move(shared_ptr()); + return m_nodes.count(_id) ? move(m_nodes[_id]) : move(shared_ptr()); } void NodeTable::discover(NodeId _node, unsigned _round, shared_ptr>> _tried) @@ -289,7 +303,9 @@ void NodeTable::noteActiveNode(Public const& _pubk, bi::udp::endpoint const& _en if (_pubk == m_node.address()) return; - shared_ptr node(addNode(_pubk, _endpoint)); + clog(NodeTableNote) << "Noting active node:" << _pubk.abridged() << _endpoint.address().to_string() << ":" << _endpoint.port(); + + shared_ptr node(addNode(_pubk, _endpoint, bi::tcp::endpoint(_endpoint.address(), _endpoint.port()))); // TODO p2p: old bug (maybe gone now) sometimes node is nullptr here if (!!node) @@ -471,8 +487,8 @@ void NodeTable::doCheckEvictions(boost::system::error_code const& _ec) Guard le(x_evictions); for (auto& e: m_evictions) if (chrono::steady_clock::now() - e.first.second > c_reqTimeout) - if (auto n = m_nodes[e.second]) - drop.push_back(n); + if (m_nodes.count(e.second)) + drop.push_back(m_nodes[e.second]); evictionsRemain = m_evictions.size() - drop.size() > 0; } @@ -498,6 +514,7 @@ void NodeTable::doRefreshBuckets(boost::system::error_code const& _ec) Guard l(x_state); for (auto& d: m_state) if (chrono::steady_clock::now() - d.modified > c_bucketRefresh) + { while (!d.nodes.empty()) { auto n = d.nodes.front(); @@ -509,6 +526,7 @@ void NodeTable::doRefreshBuckets(boost::system::error_code const& _ec) } d.nodes.pop_front(); } + } } unsigned nextRefresh = connected ? (refreshed ? 200 : c_bucketRefresh.count()*1000) : 10000; diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 72f7800f5..04e8d009c 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -115,6 +115,7 @@ inline std::ostream& operator<<(std::ostream& _out, NodeTable const& _nodeTable) * @todo cache Ping and FindSelf * * [Networking] + * @todo node-endpoint updates * @todo TCP endpoints * @todo eth/upnp/natpmp/stun/ice/etc for public-discovery * @todo firewall @@ -144,7 +145,7 @@ public: void processEvents(); /// Add node. Node will be pinged if it's not already known. - std::shared_ptr addNode(Public const& _pubk, bi::udp::endpoint const& _udp, bi::tcp::endpoint const& _tcp = bi::tcp::endpoint()); + std::shared_ptr addNode(Public const& _pubk, bi::udp::endpoint const& _udp, bi::tcp::endpoint const& _tcp); /// Add node. Node will be pinged if it's not already known. std::shared_ptr addNode(Node const& _node); @@ -162,10 +163,10 @@ public: std::list snapshot() const; /// Returns true if node id is in node table. - bool haveNode(NodeId _id) { Guard l(x_nodes); return m_nodes.count(_id); } + bool haveNode(NodeId const& _id) { Guard l(x_nodes); return m_nodes.count(_id); } /// Returns the Node to the corresponding node id or the empty Node if that id is not found. - Node node(NodeId _id); + Node node(NodeId const& _id); #ifndef BOOST_AUTO_TEST_SUITE private: @@ -197,6 +198,7 @@ protected: unsigned distance; TimePoint modified; std::list> nodes; + void touch() { modified = std::chrono::steady_clock::now(); } }; /// Used to ping endpoint. From 49d21e9ada1fa140fb66137c7b3bc2453df7df6c Mon Sep 17 00:00:00 2001 From: subtly Date: Wed, 11 Feb 2015 01:43:36 -0500 Subject: [PATCH 41/71] add back fallback for retrying when connect peers is 0 and idealpeercount > 0 --- libp2p/Host.cpp | 10 ++++++++++ libp2p/Peer.cpp | 26 ++++++++++++++++++++++++++ libp2p/Peer.h | 10 ++++++++-- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 5ffb12c8f..e35f8100f 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -528,6 +528,16 @@ void Host::run(boost::system::error_code const&) keepAlivePeers(); disconnectLatePeers(); + + if (m_idealPeerCount && !peerCount()) + for (auto p: m_peers) + if (p.second->shouldReconnect()) + { + // TODO p2p: fixme + p.second->m_lastAttempted = std::chrono::system_clock::now(); + connect(p.second); + break; + } auto runcb = [this](boost::system::error_code const& error) { run(error); }; m_timer->expires_from_now(boost::posix_time::milliseconds(c_timerInterval)); diff --git a/libp2p/Peer.cpp b/libp2p/Peer.cpp index 1811da930..4be0fd799 100644 --- a/libp2p/Peer.cpp +++ b/libp2p/Peer.cpp @@ -31,6 +31,32 @@ namespace dev namespace p2p { +bool Peer::shouldReconnect() const +{ + return chrono::system_clock::now() > m_lastAttempted + chrono::seconds(fallbackSeconds()); +} + +unsigned Peer::fallbackSeconds() const +{ + switch (m_lastDisconnect) + { + case BadProtocol: + return 30 * (m_failedAttempts + 1); + case UselessPeer: + case TooManyPeers: + case ClientQuit: + return 15 * (m_failedAttempts + 1); + case NoDisconnect: + default: + if (m_failedAttempts < 5) + return m_failedAttempts ? m_failedAttempts * 5 : 5; + else if (m_failedAttempts < 15) + return 25 + (m_failedAttempts - 5) * 10; + else + return 25 + 100 + (m_failedAttempts - 15) * 20; + } +} + bool Peer::operator<(Peer const& _p) const { if (isOffline() != _p.isOffline()) diff --git a/libp2p/Peer.h b/libp2p/Peer.h index 415573c0c..704e5c2b4 100644 --- a/libp2p/Peer.h +++ b/libp2p/Peer.h @@ -62,16 +62,22 @@ public: virtual bool operator<(Peer const& _p) const; - /// This peers rating. + /// WIP: Returns current peer rating. int rating() const { return m_rating; } + /// Return true if connection attempt should be made to this peer or false if + bool shouldReconnect() const; + /// Number of times connection has been attempted to peer. int failedAttempts() const { return m_failedAttempts; } - + /// Reason peer was previously disconnected. DisconnectReason lastDisconnect() const { return m_lastDisconnect; } protected: + /// Returns number of seconds to wait until attempting connection, based on attempted connection history. + unsigned fallbackSeconds() const; + int m_score = 0; ///< All time cumulative. int m_rating = 0; ///< Trending. From c60b0099c797ef950e1645dcebc0854f33c7101d Mon Sep 17 00:00:00 2001 From: subtly Date: Wed, 11 Feb 2015 02:00:39 -0500 Subject: [PATCH 42/71] update last disconnect for fallback timer after successful connect. --- libp2p/Host.cpp | 3 ++- libp2p/Session.cpp | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index e35f8100f..e9c172ac1 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -467,8 +467,9 @@ void Host::connect(std::shared_ptr const& _p) else { clog(NetConnect) << "Connected to" << _p->id.abridged() << "@" << _p->peerEndpoint(); - + _p->m_lastDisconnect = NoDisconnect; _p->m_lastConnected = std::chrono::system_clock::now(); + _p->m_failedAttempts = 0; auto ps = make_shared(this, std::move(*s), _p); ps->start(); diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index 1d1d69b7d..83270b6cf 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -193,8 +193,12 @@ bool Session::interpret(RLP const& _r) } else if (m_peer->id != id) { - disconnect(UnexpectedIdentity); - return true; + // TODO p2p: FIXME. Host should catch this and reattempt adding node to table. + m_peer->id = id; + m_peer->m_score = 0; + m_peer->m_rating = 0; +// disconnect(UnexpectedIdentity); +// return true; } if (m_server->havePeerSession(id)) From 572e451bab9f259843018ae9522fe4f351e46c8e Mon Sep 17 00:00:00 2001 From: subtly Date: Wed, 11 Feb 2015 03:27:04 -0500 Subject: [PATCH 43/71] Added session to hostcapabilityface::peers to ensure it isn't deallocated when shared-ptr to cap is returned. Previously hosts depended on using Session however this could result in an infinite session when hostcapface::peers is processed concurrently (mutexes can make this a likely event). This will be cleaner with better integration of Session and Peer. --- libethereum/EthereumHost.cpp | 25 ++++++++++++++----------- libp2p/HostCapability.cpp | 10 +++++----- libp2p/HostCapability.h | 3 ++- libp2p/Session.cpp | 2 ++ libwhisper/WhisperHost.cpp | 16 ++++++++++------ 5 files changed, 33 insertions(+), 23 deletions(-) diff --git a/libethereum/EthereumHost.cpp b/libethereum/EthereumHost.cpp index 94ef9d35c..7dfc51b47 100644 --- a/libethereum/EthereumHost.cpp +++ b/libethereum/EthereumHost.cpp @@ -51,8 +51,8 @@ EthereumHost::EthereumHost(BlockChain const& _ch, TransactionQueue& _tq, BlockQu EthereumHost::~EthereumHost() { - for (auto const& i: peers()) - i->cap()->abortSync(); + for (auto i: peerSessions()) + i.first->cap().get()->abortSync(); } bool EthereumHost::ensureInitialised() @@ -95,16 +95,19 @@ void EthereumHost::changeSyncer(EthereumPeer* _syncer) if (isSyncing()) { if (_syncer->m_asking == Asking::Blocks) - for (auto j: peers()) - if (j->cap().get() != _syncer && j->cap()->m_asking == Asking::Nothing) - j->cap()->transition(Asking::Blocks); + for (auto j: peerSessions()) + { + auto e = j.first->cap().get(); + if (e != _syncer && e->m_asking == Asking::Nothing) + e->transition(Asking::Blocks); + } } else { // start grabbing next hash chain if there is one. - for (auto j: peers()) + for (auto j: peerSessions()) { - j->cap()->attemptSync(); + j.first->cap()->attemptSync(); if (isSyncing()) return; } @@ -167,8 +170,8 @@ void EthereumHost::doWork() void EthereumHost::maintainTransactions() { // Send any new transactions. - for (auto const& p: peers()) - if (auto ep = p->cap()) + for (auto p: peerSessions()) + if (auto ep = p.first->cap().get()) { bytes b; unsigned n = 0; @@ -198,9 +201,9 @@ void EthereumHost::maintainBlocks(h256 _currentHash) { clog(NetMessageSummary) << "Sending a new block (current is" << _currentHash << ", was" << m_latestBlockSent << ")"; - for (auto j: peers()) + for (auto j: peerSessions()) { - auto p = j->cap(); + auto p = j.first->cap().get(); RLPStream ts; p->prep(ts, NewBlockPacket, 2).appendRaw(m_chain.block(), 1).append(m_chain.details().totalDifficulty); diff --git a/libp2p/HostCapability.cpp b/libp2p/HostCapability.cpp index 8ff74d3b6..9437cd45c 100644 --- a/libp2p/HostCapability.cpp +++ b/libp2p/HostCapability.cpp @@ -32,13 +32,13 @@ void HostCapabilityFace::seal(bytes& _b) m_host->seal(_b); } -std::vector > HostCapabilityFace::peers() const +std::vector,std::shared_ptr>> HostCapabilityFace::peerSessions() const { RecursiveGuard l(m_host->x_sessions); - std::vector > ret; + std::vector,std::shared_ptr>> ret; for (auto const& i: m_host->m_sessions) - if (std::shared_ptr p = i.second.lock()) - if (p->m_capabilities.count(capDesc())) - ret.push_back(p); + if (std::shared_ptr s = i.second.lock()) + if (s->m_capabilities.count(capDesc())) + ret.push_back(make_pair(s,s->m_peer)); return ret; } diff --git a/libp2p/HostCapability.h b/libp2p/HostCapability.h index 9666ef65a..9122ca1fa 100644 --- a/libp2p/HostCapability.h +++ b/libp2p/HostCapability.h @@ -23,6 +23,7 @@ #pragma once +#include "Peer.h" #include "Common.h" namespace dev @@ -44,7 +45,7 @@ public: Host* host() const { return m_host; } - std::vector > peers() const; + std::vector,std::shared_ptr>> peerSessions() const; protected: virtual std::string name() const = 0; diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index 83270b6cf..cb7225f0f 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -205,6 +205,8 @@ bool Session::interpret(RLP const& _r) { // Already connected. clogS(NetWarn) << "Already connected to a peer with id" << id.abridged(); + // Possible that two nodes continually connect to each other with exact same timing. + this_thread::sleep_for(chrono::milliseconds(rand() % 100)); disconnect(DuplicatePeer); return true; } diff --git a/libwhisper/WhisperHost.cpp b/libwhisper/WhisperHost.cpp index 213134db9..22a6a56fe 100644 --- a/libwhisper/WhisperHost.cpp +++ b/libwhisper/WhisperHost.cpp @@ -79,11 +79,15 @@ void WhisperHost::inject(Envelope const& _m, WhisperPeer* _p) noteChanged(h, f.first); } - for (auto& i: peers()) - if (i->cap().get() == _p) - i->addRating(1); + // TODO p2p: capability-based rating + for (auto i: peerSessions()) + { + auto w = i.first->cap().get(); + if (w == _p) + w->addRating(1); else - i->cap()->noteNewMessage(h, _m); + w->noteNewMessage(h, _m); + } } void WhisperHost::noteChanged(h256 _messageHash, h256 _filter) @@ -158,8 +162,8 @@ void WhisperHost::uninstallWatch(unsigned _i) void WhisperHost::doWork() { - for (auto& i: peers()) - i->cap()->sendMessages(); + for (auto& i: peerSessions()) + i.first->cap().get()->sendMessages(); cleanup(); } From a153d078bce9e8c497afc02fa89259264faea685 Mon Sep 17 00:00:00 2001 From: subtly Date: Wed, 11 Feb 2015 03:58:46 -0500 Subject: [PATCH 44/71] Prevent infinite sleep on connect. Write settings before shutdown of web3. --- alethzero/MainWin.cpp | 2 +- libp2p/Host.cpp | 6 ++++-- libp2p/NodeTable.cpp | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index 05daada69..f3714e76e 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -211,11 +211,11 @@ Main::Main(QWidget *parent) : Main::~Main() { + writeSettings(); // Must do this here since otherwise m_ethereum'll be deleted (and therefore clearWatches() called by the destructor) // *after* the client is dead. m_qweb->clientDieing(); g_logPost = simpleDebugOut; - writeSettings(); } void Main::on_newIdentity_triggered() diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index e9c172ac1..7010f9b04 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -428,8 +428,9 @@ void Host::addNode(NodeId const& _node, std::string const& _addr, unsigned short void Host::connect(std::shared_ptr const& _p) { - while (isWorking() && !m_run) - this_thread::sleep_for(chrono::milliseconds(50)); + for (unsigned i = 0; i < 40; i++) + if (isWorking() && !m_run) + this_thread::sleep_for(chrono::milliseconds(50)); if (!m_run) return; @@ -442,6 +443,7 @@ void Host::connect(std::shared_ptr const& _p) if (!m_nodeTable->haveNode(_p->id)) { clog(NetWarn) << "Aborted connect. Node not in node table."; + m_nodeTable->addNode(*_p.get()); return; } diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 3ccbca62d..e6f8a171b 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -65,7 +65,7 @@ void NodeTable::processEvents() shared_ptr NodeTable::addNode(Public const& _pubk, bi::udp::endpoint const& _udp, bi::tcp::endpoint const& _tcp) { auto node = Node(_pubk, NodeIPEndpoint(_udp, _tcp)); - return move(addNode(node)); + return addNode(node); } shared_ptr NodeTable::addNode(Node const& _node) @@ -101,7 +101,7 @@ shared_ptr NodeTable::addNode(Node const& _node) if (m_nodeEventHandler) m_nodeEventHandler->appendEvent(_node.id, NodeEntryAdded); - return move(ret); + return ret; } void NodeTable::discover() From 7a126cf22378deeea59994339616675fad2dce76 Mon Sep 17 00:00:00 2001 From: subtly Date: Wed, 11 Feb 2015 14:10:08 -0500 Subject: [PATCH 45/71] update nodebucket modification time when node is inserted or bucket is refreshed so bucket-refresh doesn't loop --- libp2p/NodeTable.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index e6f8a171b..a65ebd53c 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -323,15 +323,20 @@ void NodeTable::noteActiveNode(Public const& _pubk, bi::udp::endpoint const& _en if (s.nodes.size() >= s_bucketSize) { + // It's only contested iff nodeentry exists contested = s.nodes.front().lock(); if (!contested) { s.nodes.pop_front(); s.nodes.push_back(node); + s.touch(); } } else + { s.nodes.push_back(node); + s.touch(); + } } if (contested) @@ -515,6 +520,7 @@ void NodeTable::doRefreshBuckets(boost::system::error_code const& _ec) for (auto& d: m_state) if (chrono::steady_clock::now() - d.modified > c_bucketRefresh) { + d.touch(); while (!d.nodes.empty()) { auto n = d.nodes.front(); From 38230cb1e4ac81d75c4587f133a28412a5e77a0a Mon Sep 17 00:00:00 2001 From: subtly Date: Wed, 11 Feb 2015 15:54:31 -0500 Subject: [PATCH 46/71] Increase connect timeout for upnp initialization and fix resolver. --- libp2p/Host.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 7010f9b04..5892121ed 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -413,13 +413,15 @@ void Host::addNode(NodeId const& _node, std::string const& _addr, unsigned short bi::address addr = bi::address::from_string(_addr, ec); if (ec) { - bi::tcp::resolver r(m_ioService); - r.async_resolve({_addr, toString(_tcpPeerPort)}, [=](boost::system::error_code const& _ec, bi::tcp::resolver::iterator _epIt) + bi::tcp::resolver *r = new bi::tcp::resolver(m_ioService); + r->async_resolve({_addr, toString(_tcpPeerPort)}, [=](boost::system::error_code const& _ec, bi::tcp::resolver::iterator _epIt) { - if (_ec) - return; - bi::tcp::endpoint tcp = *_epIt; - if (m_nodeTable) m_nodeTable->addNode(Node(_node, NodeIPEndpoint(bi::udp::endpoint(tcp.address(), _udpNodePort), tcp))); + if (!_ec) + { + bi::tcp::endpoint tcp = *_epIt; + if (m_nodeTable) m_nodeTable->addNode(Node(_node, NodeIPEndpoint(bi::udp::endpoint(tcp.address(), _udpNodePort), tcp))); + } + delete r; }); } else @@ -428,7 +430,7 @@ void Host::addNode(NodeId const& _node, std::string const& _addr, unsigned short void Host::connect(std::shared_ptr const& _p) { - for (unsigned i = 0; i < 40; i++) + for (unsigned i = 0; i < 200; i++) if (isWorking() && !m_run) this_thread::sleep_for(chrono::milliseconds(50)); if (!m_run) From f070de2047843f31d8bbedaa9b992a0667276c2c Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 12 Feb 2015 00:00:05 -0500 Subject: [PATCH 47/71] load network before constructing webthreedirect --- alethzero/MainWin.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index f3714e76e..1d4427789 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -162,6 +162,8 @@ Main::Main(QWidget *parent) : connect(ui->ourAccounts->model(), SIGNAL(rowsMoved(const QModelIndex &, int, int, const QModelIndex &, int)), SLOT(ourAccountsRowsMoved())); + QSettings s("ethereum", "alethzero"); + m_networkConfig = s.value("peers").toByteArray(); bytesConstRef network((byte*)m_networkConfig.data(), m_networkConfig.size()); m_webThree.reset(new WebThreeDirect(string("AlethZero/v") + dev::Version + "/" DEV_QUOTED(ETH_BUILD_TYPE) "/" DEV_QUOTED(ETH_BUILD_PLATFORM), getDataDir() + "/AlethZero", false, {"eth", "shh"}, p2p::NetworkPreferences(), network)); @@ -715,7 +717,6 @@ void Main::readSettings(bool _skipGeometry) } } - m_networkConfig = s.value("peers").toByteArray(); ui->upnp->setChecked(s.value("upnp", true).toBool()); ui->forceAddress->setText(s.value("forceAddress", "").toString()); ui->usePast->setChecked(s.value("usePast", true).toBool()); From 3291bc1d9aabc703349263006592620f4408d646 Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 12 Feb 2015 00:17:31 -0500 Subject: [PATCH 48/71] perform node discovery when there aren't enough nodes --- libp2p/Host.cpp | 6 +++++- libp2p/NodeTable.cpp | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 5892121ed..f09468e0d 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -534,7 +534,8 @@ void Host::run(boost::system::error_code const&) keepAlivePeers(); disconnectLatePeers(); - if (m_idealPeerCount && !peerCount()) + auto c = peerCount(); + if (m_idealPeerCount && !c) for (auto p: m_peers) if (p.second->shouldReconnect()) { @@ -544,6 +545,9 @@ void Host::run(boost::system::error_code const&) break; } + if (c < m_idealPeerCount) + m_nodeTable->discover(); + auto runcb = [this](boost::system::error_code const& error) { run(error); }; m_timer->expires_from_now(boost::posix_time::milliseconds(c_timerInterval)); m_timer->async_wait(runcb); diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index a65ebd53c..15eca291c 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -106,7 +106,9 @@ shared_ptr NodeTable::addNode(Node const& _node) void NodeTable::discover() { - discover(m_node.id); + static chrono::steady_clock::time_point s_lastDiscover = chrono::steady_clock::now() - std::chrono::seconds(30); + if (chrono::steady_clock::now() > s_lastDiscover + std::chrono::seconds(30)) + discover(m_node.id); } list NodeTable::nodes() const From 57f6aacf9ed981811873f8012586259ddfbc3ac3 Mon Sep 17 00:00:00 2001 From: subtly Date: Thu, 12 Feb 2015 00:31:48 -0500 Subject: [PATCH 49/71] update last discovery timepoint --- libp2p/NodeTable.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libp2p/NodeTable.cpp b/libp2p/NodeTable.cpp index 15eca291c..adeb43c2e 100644 --- a/libp2p/NodeTable.cpp +++ b/libp2p/NodeTable.cpp @@ -108,7 +108,10 @@ void NodeTable::discover() { static chrono::steady_clock::time_point s_lastDiscover = chrono::steady_clock::now() - std::chrono::seconds(30); if (chrono::steady_clock::now() > s_lastDiscover + std::chrono::seconds(30)) + { + s_lastDiscover = chrono::steady_clock::now(); discover(m_node.id); + } } list NodeTable::nodes() const From 11283fef7e328887176e9bcc00b7206861a9dfa1 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 12 Feb 2015 15:01:38 +0100 Subject: [PATCH 50/71] Bytes type cleanup. --- libsolidity/CompilerUtils.cpp | 9 +++------ libsolidity/Types.cpp | 12 +++--------- libsolidity/Types.h | 10 +--------- 3 files changed, 7 insertions(+), 24 deletions(-) diff --git a/libsolidity/CompilerUtils.cpp b/libsolidity/CompilerUtils.cpp index 192d66d5c..73be38176 100644 --- a/libsolidity/CompilerUtils.cpp +++ b/libsolidity/CompilerUtils.cpp @@ -184,17 +184,14 @@ void CompilerUtils::copyByteArrayToStorage(ByteArrayType const& _targetType, << eth::Instruction::SWAP1; // stack here: target_ref target_data_end target_data_ref // store length (in bytes) - if (_sourceType.getOffset() == 0) - m_context << eth::Instruction::CALLDATASIZE; - else - m_context << _sourceType.getOffset() << eth::Instruction::CALLDATASIZE << eth::Instruction::SUB; + m_context << eth::Instruction::CALLDATASIZE; m_context << eth::Instruction::DUP1 << eth::Instruction::DUP5 << eth::Instruction::SSTORE; // jump to end if length is zero m_context << eth::Instruction::ISZERO; eth::AssemblyItem copyLoopEnd = m_context.newTag(); m_context.appendConditionalJumpTo(copyLoopEnd); - - m_context << _sourceType.getOffset(); + // store start offset + m_context << u256(0); // stack now: target_ref target_data_end target_data_ref calldata_offset eth::AssemblyItem copyLoopStart = m_context.newTag(); m_context << copyLoopStart diff --git a/libsolidity/Types.cpp b/libsolidity/Types.cpp index f43a3ffe3..b1029ec6f 100644 --- a/libsolidity/Types.cpp +++ b/libsolidity/Types.cpp @@ -58,7 +58,7 @@ TypePointer Type::fromElementaryTypeName(Token::Value _typeToken) else if (Token::String0 <= _typeToken && _typeToken <= Token::String32) return make_shared(int(_typeToken) - int(Token::String0)); else if (_typeToken == Token::Bytes) - return make_shared(ByteArrayType::Location::Storage, 0, 0, true); + return make_shared(ByteArrayType::Location::Storage); else BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_comment("Unable to convert elementary typename " + std::string(Token::toString(_typeToken)) + " to type.")); @@ -515,12 +515,7 @@ TypePointer ContractType::unaryOperatorResult(Token::Value _operator) const bool ByteArrayType::isImplicitlyConvertibleTo(const Type& _convertTo) const { - if (*this == _convertTo) - return true; - if (_convertTo.getCategory() != Category::ByteArray) - return false; - auto const& other = dynamic_cast(_convertTo); - return (m_dynamicLength == other.m_dynamicLength || m_length == other.m_length); + return _convertTo.getCategory() == getCategory(); } TypePointer ByteArrayType::unaryOperatorResult(Token::Value _operator) const @@ -535,8 +530,7 @@ bool ByteArrayType::operator==(Type const& _other) const if (_other.getCategory() != getCategory()) return false; ByteArrayType const& other = dynamic_cast(_other); - return other.m_location == m_location && other.m_dynamicLength == m_dynamicLength - && other.m_length == m_length && other.m_offset == m_offset; + return other.m_location == m_location; } unsigned ByteArrayType::getSizeOnStack() const diff --git a/libsolidity/Types.h b/libsolidity/Types.h index 0270773fb..627bc76f0 100644 --- a/libsolidity/Types.h +++ b/libsolidity/Types.h @@ -286,9 +286,7 @@ public: enum class Location { Storage, CallData, Memory }; virtual Category getCategory() const override { return Category::ByteArray; } - explicit ByteArrayType(Location _location, u256 const& _offset = 0, u256 const& _length = 0, - bool _dynamicLength = false): - m_location(_location), m_offset(_offset), m_length(_length), m_dynamicLength(_dynamicLength) {} + explicit ByteArrayType(Location _location): m_location(_location) {} virtual bool isImplicitlyConvertibleTo(Type const& _convertTo) const override; virtual TypePointer unaryOperatorResult(Token::Value _operator) const override; virtual bool operator==(const Type& _other) const override; @@ -296,15 +294,9 @@ public: virtual std::string toString() const override { return "bytes"; } Location getLocation() const { return m_location; } - u256 const& getOffset() const { return m_offset; } - u256 const& getLength() const { return m_length; } - bool hasDynamicLength() const { return m_dynamicLength; } private: Location m_location; - u256 m_offset; - u256 m_length; - bool m_dynamicLength; }; /** From c57775bdd971401cdc12112efa31e650cbdd62a3 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 12 Feb 2015 15:18:50 +0100 Subject: [PATCH 51/71] Test for bytes in mapping. --- test/SolidityEndToEndTest.cpp | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/SolidityEndToEndTest.cpp b/test/SolidityEndToEndTest.cpp index 67d05d6e3..a57eb2700 100644 --- a/test/SolidityEndToEndTest.cpp +++ b/test/SolidityEndToEndTest.cpp @@ -2385,6 +2385,30 @@ BOOST_AUTO_TEST_CASE(copy_removes_bytes_data) BOOST_CHECK(m_state.storage(m_contractAddress).empty()); } +BOOST_AUTO_TEST_CASE(bytes_inside_mappings) +{ + char const* sourceCode = R"( + contract c { + function set(uint key) returns (bool) { data[key] = msg.data; return true; } + function copy(uint from, uint to) returns (bool) { data[to] = data[from]; return true; } + mapping(uint => bytes) data; + } + )"; + compileAndRun(sourceCode); + // store a short byte array at 1 and a longer one at 2 + BOOST_CHECK(callContractFunction("set(uint256)", 1, 2) == encodeArgs(true)); + BOOST_CHECK(callContractFunction("set(uint256)", 2, 2, 3, 4, 5) == encodeArgs(true)); + BOOST_CHECK(!m_state.storage(m_contractAddress).empty()); + // copy shorter to longer + BOOST_CHECK(callContractFunction("copy(uint256,uint256)", 1, 2) == encodeArgs(true)); + BOOST_CHECK(!m_state.storage(m_contractAddress).empty()); + // copy empty to both + BOOST_CHECK(callContractFunction("copy(uint256,uint256)", 99, 1) == encodeArgs(true)); + BOOST_CHECK(!m_state.storage(m_contractAddress).empty()); + BOOST_CHECK(callContractFunction("copy(uint256,uint256)", 99, 2) == encodeArgs(true)); + BOOST_CHECK(m_state.storage(m_contractAddress).empty()); +} + BOOST_AUTO_TEST_SUITE_END() } From a0db309212f7b64f599bed3ff4a83f6102da7354 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 12 Feb 2015 15:44:35 +0100 Subject: [PATCH 52/71] length member for byte arrays. --- libsolidity/ExpressionCompiler.cpp | 6 ++++++ libsolidity/Types.cpp | 2 ++ libsolidity/Types.h | 2 ++ test/SolidityEndToEndTest.cpp | 15 +++++++++++++++ 4 files changed, 25 insertions(+) diff --git a/libsolidity/ExpressionCompiler.cpp b/libsolidity/ExpressionCompiler.cpp index 63132a124..3dbb4012d 100644 --- a/libsolidity/ExpressionCompiler.cpp +++ b/libsolidity/ExpressionCompiler.cpp @@ -504,6 +504,12 @@ void ExpressionCompiler::endVisit(MemberAccess const& _memberAccess) } BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_comment("Invalid member access to " + type.toString())); } + case Type::Category::ByteArray: + { + solAssert(member == "length", "Illegal bytearray member."); + m_context << eth::Instruction::SLOAD; + break; + } default: BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_comment("Member access to unknown type.")); } diff --git a/libsolidity/Types.cpp b/libsolidity/Types.cpp index b1029ec6f..33cc8a1ec 100644 --- a/libsolidity/Types.cpp +++ b/libsolidity/Types.cpp @@ -541,6 +541,8 @@ unsigned ByteArrayType::getSizeOnStack() const return 1; } +const MemberList ByteArrayType::s_byteArrayMemberList = MemberList({{"length", make_shared(256)}}); + bool ContractType::operator==(Type const& _other) const { if (_other.getCategory() != getCategory()) diff --git a/libsolidity/Types.h b/libsolidity/Types.h index 627bc76f0..5a0e2c429 100644 --- a/libsolidity/Types.h +++ b/libsolidity/Types.h @@ -292,11 +292,13 @@ public: virtual bool operator==(const Type& _other) const override; virtual unsigned getSizeOnStack() const override; virtual std::string toString() const override { return "bytes"; } + virtual MemberList const& getMembers() const override { return s_byteArrayMemberList; } Location getLocation() const { return m_location; } private: Location m_location; + static const MemberList s_byteArrayMemberList; }; /** diff --git a/test/SolidityEndToEndTest.cpp b/test/SolidityEndToEndTest.cpp index a57eb2700..71cb4a6f1 100644 --- a/test/SolidityEndToEndTest.cpp +++ b/test/SolidityEndToEndTest.cpp @@ -2409,6 +2409,21 @@ BOOST_AUTO_TEST_CASE(bytes_inside_mappings) BOOST_CHECK(m_state.storage(m_contractAddress).empty()); } +BOOST_AUTO_TEST_CASE(bytes_length_member) +{ + char const* sourceCode = R"( + contract c { + function set() returns (bool) { data = msg.data; return true; } + function getLength() returns (uint) { return data.length; } + bytes data; + } + )"; + compileAndRun(sourceCode); + BOOST_CHECK(callContractFunction("getLength()") == encodeArgs(0)); + BOOST_CHECK(callContractFunction("set()", 1, 2) == encodeArgs(true)); + BOOST_CHECK(callContractFunction("getLength()") == encodeArgs(4+32+32)); +} + BOOST_AUTO_TEST_SUITE_END() } From 07ced4cab31092efb78dc63e7bf794e0d08048ad Mon Sep 17 00:00:00 2001 From: yann300 Date: Thu, 12 Feb 2015 16:27:45 +0100 Subject: [PATCH 53/71] - Add Reusable common controls (Text, Label, ...). - Add Generic Style.qml. - TransactionDialog/StateDialog ui improvement. - Other Small changes. --- mix/ClientModel.cpp | 3 +- mix/qml/CommonSeparator.qml | 9 ++ mix/qml/DefaultLabel.qml | 16 +++ mix/qml/DefaultTextField.qml | 13 ++ mix/qml/Ether.qml | 11 +- mix/qml/MainContent.qml | 2 +- mix/qml/NewProjectDialog.qml | 2 +- mix/qml/QHashTypeView.qml | 1 + mix/qml/QIntTypeView.qml | 1 + mix/qml/QStringTypeView.qml | 1 + mix/qml/StateDialog.qml | 230 ++++++++++++++++---------------- mix/qml/StateListModel.qml | 9 +- mix/qml/Style.qml | 16 +++ mix/qml/TransactionDialog.qml | 124 +++++++++-------- mix/qml/TransactionLog.qml | 10 ++ mix/qml/WebPreview.qml | 4 +- mix/qml/img/delete_sign.png | Bin 0 -> 465 bytes mix/qml/img/edit.png | Bin 0 -> 511 bytes mix/qml/img/plus.png | Bin 0 -> 254 bytes mix/qml/js/QEtherHelper.js | 9 ++ mix/qml/js/TransactionHelper.js | 2 +- mix/qml/qmldir | 1 + mix/res.qrc | 7 + 23 files changed, 284 insertions(+), 187 deletions(-) create mode 100644 mix/qml/CommonSeparator.qml create mode 100644 mix/qml/DefaultLabel.qml create mode 100644 mix/qml/DefaultTextField.qml create mode 100644 mix/qml/Style.qml create mode 100644 mix/qml/img/delete_sign.png create mode 100644 mix/qml/img/edit.png create mode 100644 mix/qml/img/plus.png diff --git a/mix/ClientModel.cpp b/mix/ClientModel.cpp index b7be8988a..57caf573c 100644 --- a/mix/ClientModel.cpp +++ b/mix/ClientModel.cpp @@ -156,7 +156,8 @@ void ClientModel::setupState(QVariantMap _state) { QVariantMap transaction = t.toMap(); QString functionId = transaction.value("functionId").toString(); - u256 gas = (qvariant_cast(transaction.value("gas")))->toU256Wei(); + + u256 gas = boost::get(qvariant_cast(transaction.value("gas"))->internalValue()); u256 value = (qvariant_cast(transaction.value("value")))->toU256Wei(); u256 gasPrice = (qvariant_cast(transaction.value("gasPrice")))->toU256Wei(); diff --git a/mix/qml/CommonSeparator.qml b/mix/qml/CommonSeparator.qml new file mode 100644 index 000000000..4d081e05f --- /dev/null +++ b/mix/qml/CommonSeparator.qml @@ -0,0 +1,9 @@ +import QtQuick 2.0 +import "." + +Rectangle +{ + height: 3 + color: Style.generic.layout.separatorColor +} + diff --git a/mix/qml/DefaultLabel.qml b/mix/qml/DefaultLabel.qml new file mode 100644 index 000000000..da5495863 --- /dev/null +++ b/mix/qml/DefaultLabel.qml @@ -0,0 +1,16 @@ +import QtQuick 2.0 +import QtQuick.Controls 1.1 + +Label { + text: text + font.family: regularFont.name + + SourceSansProRegular + { + id: regularFont + } +} + + + + diff --git a/mix/qml/DefaultTextField.qml b/mix/qml/DefaultTextField.qml new file mode 100644 index 000000000..6705273db --- /dev/null +++ b/mix/qml/DefaultTextField.qml @@ -0,0 +1,13 @@ +import QtQuick 2.0 +import QtQuick.Controls 1.1 + +TextField { + id: titleField + focus: true + font.family: regularFont.name + + SourceSansProRegular + { + id: regularFont; + } +} diff --git a/mix/qml/Ether.qml b/mix/qml/Ether.qml index f737b12b6..be41fced9 100644 --- a/mix/qml/Ether.qml +++ b/mix/qml/Ether.qml @@ -32,12 +32,8 @@ RowLayout { units.currentIndex = unit; } - SourceSansProRegular - { - id: regularFont - } - TextField + DefaultTextField { implicitWidth: 200 onTextChanged: @@ -51,7 +47,10 @@ RowLayout { readOnly: !edit visible: edit id: etherValueEdit; - font.family: regularFont.name + } + + SourceSansProBold { + id: regularFont; } ComboBox diff --git a/mix/qml/MainContent.qml b/mix/qml/MainContent.qml index d984e2753..6c6781878 100644 --- a/mix/qml/MainContent.qml +++ b/mix/qml/MainContent.qml @@ -85,7 +85,7 @@ Rectangle { property alias webWidth: webPreview.width property alias webHeight: webPreview.height property alias showProjectView: projectList.visible - property bool runOnProjectLoad: false + property bool runOnProjectLoad: true } ColumnLayout diff --git a/mix/qml/NewProjectDialog.qml b/mix/qml/NewProjectDialog.qml index 770775df2..4fcb524b2 100644 --- a/mix/qml/NewProjectDialog.qml +++ b/mix/qml/NewProjectDialog.qml @@ -6,7 +6,7 @@ import QtQuick.Dialogs 1.1 Window { id: newProjectWin - modality: Qt.WindowModal + modality: Qt.ApplicationModal width: 640 height: 120 diff --git a/mix/qml/QHashTypeView.qml b/mix/qml/QHashTypeView.qml index 6f3c1910e..73678f953 100644 --- a/mix/qml/QHashTypeView.qml +++ b/mix/qml/QHashTypeView.qml @@ -19,6 +19,7 @@ Item text: text anchors.fill: parent wrapMode: Text.WrapAnywhere + clip: true font.family: boldFont.name MouseArea { id: mouseArea diff --git a/mix/qml/QIntTypeView.qml b/mix/qml/QIntTypeView.qml index 00a08d819..98344dd8b 100644 --- a/mix/qml/QIntTypeView.qml +++ b/mix/qml/QIntTypeView.qml @@ -19,6 +19,7 @@ Item text: text anchors.fill: parent font.family: boldFont.name + clip: true MouseArea { id: mouseArea anchors.fill: parent diff --git a/mix/qml/QStringTypeView.qml b/mix/qml/QStringTypeView.qml index 7e7d2ec76..016206e6d 100644 --- a/mix/qml/QStringTypeView.qml +++ b/mix/qml/QStringTypeView.qml @@ -17,6 +17,7 @@ Item TextInput { id: textinput text: text + clip: true anchors.fill: parent wrapMode: Text.WrapAnywhere font.family: boldFont.name diff --git a/mix/qml/StateDialog.qml b/mix/qml/StateDialog.qml index a8ac27ecf..ebcf5fd1d 100644 --- a/mix/qml/StateDialog.qml +++ b/mix/qml/StateDialog.qml @@ -10,11 +10,11 @@ import "." Window { id: modalStateDialog - modality: Qt.WindowModal + modality: Qt.ApplicationModal width: 450 height: 480 - title: qsTr("State Edition") + title: qsTr("Edit State") visible: false color: StateDialogStyle.generic.backgroundColor @@ -44,6 +44,7 @@ Window { isDefault = setDefault; titleField.focus = true; defaultCheckBox.enabled = !isDefault; + forceActiveFocus(); } function close() { @@ -60,53 +61,67 @@ Window { return item; } - SourceSansProRegular - { - id: regularFont - } - - Rectangle { + ColumnLayout { anchors.fill: parent anchors.margins: 10 - color: StateDialogStyle.generic.backgroundColor - GridLayout { + ColumnLayout { id: dialogContent - columns: 2 anchors.top: parent.top - rowSpacing: 10 - columnSpacing: 10 - Label { - text: qsTr("Title") - font.family: regularFont.name - color: "#808080" + RowLayout + { + Layout.fillWidth: true + DefaultLabel { + Layout.preferredWidth: 75 + text: qsTr("Title") + } + DefaultTextField + { + id: titleField + Layout.fillWidth: true + } } - TextField { - id: titleField - focus: true + + CommonSeparator + { Layout.fillWidth: true - font.family: regularFont.name } - Label { - text: qsTr("Balance") - font.family: regularFont.name - color: "#808080" + RowLayout + { + Layout.fillWidth: true + DefaultLabel { + Layout.preferredWidth: 75 + text: qsTr("Balance") + } + Ether { + id: balanceField + edit: true + displayFormattedValue: true + Layout.fillWidth: true + } } - Ether { - id: balanceField - edit: true - displayFormattedValue: true + + CommonSeparator + { Layout.fillWidth: true } - Label { - text: qsTr("Default") - font.family: regularFont.name - color: "#808080" + RowLayout + { + Layout.fillWidth: true + DefaultLabel { + Layout.preferredWidth: 75 + text: qsTr("Default") + } + CheckBox { + id: defaultCheckBox + Layout.fillWidth: true + } } - CheckBox { - id: defaultCheckBox + + CommonSeparator + { Layout.fillWidth: true } } @@ -114,52 +129,51 @@ Window { ColumnLayout { anchors.top: dialogContent.bottom anchors.topMargin: 5 - spacing: 5 + spacing: 0 RowLayout { - Label { - text: qsTr("Transactions") - font.family: regularFont.name - color: "#808080" + DefaultLabel { + text: qsTr("Transactions: ") } - Button { - tooltip: qsTr("Create a new transaction") - onClicked: transactionsModel.addTransaction() - height: 5 - width: 5 - style: ButtonStyle { - label: Text { - font.family: regularFont.name - text: qsTr("+") - font.pointSize: 15 - color: "#808080" - height: 5 - width: 5 - } - background: Rectangle { - radius: 4 - border.width: 1 - color: "#f7f7f7" - height: 5 - implicitHeight: 5 - } - } + Button + { + iconSource: "qrc:/qml/img/plus.png" + action: newTrAction + width: 10 + height: 10 + } + Action { + id: newTrAction + tooltip: qsTr("Create a new transaction") + onTriggered: transactionsModel.addTransaction() } } - ListView { - id: trList - Layout.preferredWidth: 200 + ScrollView + { Layout.fillHeight: true - Layout.minimumHeight: 20 * transactionsModel.count - model: transactionsModel - delegate: transactionRenderDelegate - visible: transactionsModel.count > 0 + Layout.preferredWidth: 300 + Column + { + Layout.fillHeight: true + Repeater + { + id: trRepeater + model: transactionsModel + delegate: transactionRenderDelegate + visible: transactionsModel.count > 0 + height: 20 * transactionsModel.count + } + } } - } + CommonSeparator + { + Layout.fillWidth: true + } + } RowLayout { @@ -204,53 +218,37 @@ Window { Component { id: transactionRenderDelegate - Item { - id: wrapperItem - height: 20 - width: parent.width - RowLayout { - anchors.fill: parent - Text { - Layout.fillWidth: true - Layout.fillHeight: true - text: functionId - font.pointSize: StateStyle.general.basicFontSize //12 - verticalAlignment: Text.AlignBottom - font.family: regularFont.name - } - ToolButton { - text: qsTr("Edit"); - visible: !stdContract - Layout.fillHeight: true - onClicked: transactionsModel.editTransaction(index) - style: ButtonStyle { - label: Text { - font.family: regularFont.name - text: qsTr("Edit") - font.italic: true - font.pointSize: 9 - } - background: Rectangle { - color: "transparent" - } - } + RowLayout { + DefaultLabel { + Layout.preferredWidth: 150 + text: functionId + } + + Button + { + id: deleteBtn + iconSource: "qrc:/qml/img/delete_sign.png" + action: deleteAction + width: 10 + height: 10 + Action { + id: deleteAction + tooltip: qsTr("Delete") + onTriggered: transactionsModel.deleteTransaction(index) } - ToolButton { - visible: index >= 0 ? !transactionsModel.get(index).executeConstructor : false - text: qsTr("Delete"); - Layout.fillHeight: true - onClicked: transactionsModel.deleteTransaction(index) - style: ButtonStyle { - label: Text { - font.family: regularFont.name - text: qsTr("Delete") - font.italic: true - font.pointSize: 9 - } - background: Rectangle { - color: "transparent" - } - } + } + + Button + { + iconSource: "qrc:/qml/img/edit.png" + action: editAction + visible: !stdContract + width: 10 + height: 10 + Action { + id: editAction + tooltip: qsTr("Edit") + onTriggered: transactionsModel.editTransaction(index) } } } diff --git a/mix/qml/StateListModel.qml b/mix/qml/StateListModel.qml index fd7959477..67372421c 100644 --- a/mix/qml/StateListModel.qml +++ b/mix/qml/StateListModel.qml @@ -25,7 +25,7 @@ Item { functionId: t.functionId, url: t.url, value: QEtherHelper.createEther(t.value.value, t.value.unit), - gas: QEtherHelper.createEther(t.gas.value, t.gas.unit), + gas: QEtherHelper.createBigInt(t.gas.value), //t.gas,//QEtherHelper.createEther(t.gas.value, t.gas.unit), gasPrice: QEtherHelper.createEther(t.gasPrice.value, t.gasPrice.unit), executeConstructor: t.executeConstructor, stdContract: t.stdContract, @@ -81,7 +81,7 @@ Item { functionId: t.functionId, url: t.url, value: { value: t.value.value, unit: t.value.unit }, - gas: { value: t.gas.value, unit: t.gas.unit }, + gas: { value: t.gas.value() }, gasPrice: { value: t.gasPrice.value, unit: t.gasPrice.unit }, executeConstructor: t.executeConstructor, stdContract: t.stdContract, @@ -157,7 +157,7 @@ Item { function defaultTransactionItem() { return { value: QEtherHelper.createEther("100", QEther.Wei), - gas: QEtherHelper.createEther("125000", QEther.Wei), + gas: QEtherHelper.createBigInt("125000"), gasPrice: QEtherHelper.createEther("10000000000000", QEther.Wei), executeConstructor: false, stdContract: false @@ -165,7 +165,8 @@ Item { } function createDefaultState() { - var ether = QEtherHelper.createEther("100000000000000000000000000", QEther.Wei); + //var ether = QEtherHelper.createEther("100000000000000000000000000", QEther.Wei); + var ether = QEtherHelper.createEther("1000000", QEther.Ether); var item = { title: "", balance: ether, diff --git a/mix/qml/Style.qml b/mix/qml/Style.qml new file mode 100644 index 000000000..4dd5d978c --- /dev/null +++ b/mix/qml/Style.qml @@ -0,0 +1,16 @@ +pragma Singleton +import QtQuick 2.0 + +QtObject { + + function absoluteSize(rel) + { + return systemPointSize + rel; + } + + property QtObject generic: QtObject { + property QtObject layout : QtObject { + property string separatorColor: "#f7f7f7" + } + } +} diff --git a/mix/qml/TransactionDialog.qml b/mix/qml/TransactionDialog.qml index 4b2b46cd2..cef82ecc9 100644 --- a/mix/qml/TransactionDialog.qml +++ b/mix/qml/TransactionDialog.qml @@ -9,15 +9,15 @@ import "." Window { id: modalTransactionDialog - modality: Qt.WindowModal + modality: Qt.ApplicationModal width: 450 - height: (paramsModel.count > 0 ? 550 : 300) + height: (paramsModel.count > 0 ? 500 : 300) visible: false color: StateDialogStyle.generic.backgroundColor - title: qsTr("Transaction Edition") + title: qsTr("Edit Transaction") property int transactionIndex property alias transactionParams: paramsModel; - property alias gas: gasField.value; + property alias gas: gasValueEdit.gasValue; property alias gasPrice: gasPriceField.value; property alias transactionValue: valueField.value; property alias functionId: functionComboBox.currentText; @@ -36,7 +36,7 @@ Window { rowGasPrice.visible = !useTransactionDefaultValue; transactionIndex = index; - gasField.value = item.gas; + gasValueEdit.gasValue = item.gas; gasPriceField.value = item.gasPrice; valueField.value = item.value; var functionId = item.functionId; @@ -170,32 +170,27 @@ Window { return item; } - SourceSansProRegular - { - id: regularFont - } - - Rectangle { + ColumnLayout { anchors.fill: parent - anchors.left: parent.left - anchors.right: parent.right - anchors.top: parent.top anchors.margins: 10 - color: StateDialogStyle.generic.backgroundColor + + SourceSansProLight + { + id: lightFont + } ColumnLayout { id: dialogContent - spacing: 30 + anchors.top: parent.top + spacing: 10 RowLayout { id: rowFunction Layout.fillWidth: true height: 150 - Label { + DefaultLabel { Layout.preferredWidth: 75 text: qsTr("Function") - font.family: regularFont.name - color: "#808080" } ComboBox { id: functionComboBox @@ -210,22 +205,24 @@ Window { loadParameters(); } style: ComboBoxStyle { - font: regularFont.name + font: lightFont.name } } } + CommonSeparator + { + Layout.fillWidth: true + } RowLayout { id: rowValue Layout.fillWidth: true height: 150 - Label { + DefaultLabel { Layout.preferredWidth: 75 text: qsTr("Value") - font.family: regularFont.name - color: "#808080" } Ether { id: valueField @@ -234,35 +231,44 @@ Window { } } + CommonSeparator + { + Layout.fillWidth: true + } RowLayout { id: rowGas Layout.fillWidth: true height: 150 - Label { + DefaultLabel { Layout.preferredWidth: 75 text: qsTr("Gas") - font.family: regularFont.name - color: "#808080" } - Ether { - id: gasField - edit: true - displayFormattedValue: true + + DefaultTextField + { + property variant gasValue + onGasValueChanged: text = gasValue.value(); + onTextChanged: gasValue.setValue(text); + implicitWidth: 200 + id: gasValueEdit; } } + CommonSeparator + { + Layout.fillWidth: true + } + RowLayout { id: rowGasPrice Layout.fillWidth: true height: 150 - Label { + DefaultLabel { Layout.preferredWidth: 75 text: qsTr("Gas Price") - font.family: regularFont.name - color: "#808080" } Ether { id: gasPriceField @@ -271,59 +277,62 @@ Window { } } - Label { - text: qsTr("Parameters") + CommonSeparator + { + Layout.fillWidth: true + } + + DefaultLabel { + id: paramLabel + text: qsTr("Parameters:") Layout.preferredWidth: 75 - font.family: regularFont.name - color: "#808080" visible: paramsModel.count > 0 } ScrollView { - Layout.fillWidth: true + anchors.top: paramLabel.bottom + anchors.topMargin: 10 + Layout.preferredWidth: 350 + Layout.fillHeight: true visible: paramsModel.count > 0 - ColumnLayout + Column { id: paramRepeater Layout.fillWidth: true - spacing: 10 + Layout.fillHeight: true + spacing: 3 Repeater { - anchors.fill: parent + height: 20 * paramsModel.count model: paramsModel visible: paramsModel.count > 0 RowLayout { id: row Layout.fillWidth: true - height: 150 - - Label { + height: 20 + DefaultLabel { id: typeLabel text: type - font.family: regularFont.name Layout.preferredWidth: 50 } - Label { + DefaultLabel { id: nameLabel text: name - font.family: regularFont.name - Layout.preferredWidth: 50 + Layout.preferredWidth: 80 } - Label { + DefaultLabel { id: equalLabel text: "=" - font.family: regularFont.name Layout.preferredWidth: 15 } Loader { id: typeLoader - Layout.preferredHeight: 50 Layout.preferredWidth: 150 function getCurrent() { @@ -356,7 +365,7 @@ Window { id: intViewComp QIntTypeView { - height: 50 + height: 20 width: 150 id: intView text: typeLoader.getCurrent().value @@ -368,7 +377,7 @@ Window { id: boolViewComp QBoolTypeView { - height: 50 + height: 20 width: 150 id: boolView defaultValue: "1" @@ -385,7 +394,7 @@ Window { id: stringViewComp QStringTypeView { - height: 50 + height: 20 width: 150 id: stringView text: @@ -400,7 +409,7 @@ Window { id: hashViewComp QHashTypeView { - height: 50 + height: 20 width: 150 id: hashView text: typeLoader.getCurrent().value @@ -411,6 +420,11 @@ Window { } } } + + CommonSeparator + { + Layout.fillWidth: true + } } RowLayout diff --git a/mix/qml/TransactionLog.qml b/mix/qml/TransactionLog.qml index b31956898..61d2e0920 100644 --- a/mix/qml/TransactionLog.qml +++ b/mix/qml/TransactionLog.qml @@ -32,6 +32,16 @@ Item { anchors.fill: parent RowLayout { + Connections + { + target: projectModel + onProjectSaved: + { + if (codeModel.hasContract && !clientModel.running) + projectModel.stateListModel.debugDefaultState(); + } + } + ComboBox { id: statesCombo model: projectModel.stateListModel diff --git a/mix/qml/WebPreview.qml b/mix/qml/WebPreview.qml index f4ddca84e..f258c2e21 100644 --- a/mix/qml/WebPreview.qml +++ b/mix/qml/WebPreview.qml @@ -72,8 +72,8 @@ Item { Connections { target: projectModel - onProjectSaved : reloadOnSave(); - onDocumentSaved: reloadOnSave(); + //onProjectSaved : reloadOnSave(); + //onDocumentSaved: reloadOnSave(); onDocumentAdded: { var document = projectModel.getDocument(documentId) if (document.isHtml) diff --git a/mix/qml/img/delete_sign.png b/mix/qml/img/delete_sign.png new file mode 100644 index 0000000000000000000000000000000000000000..5c00a20bac86f1ef0f68bd3f1fe9ea5626f49cec GIT binary patch literal 465 zcmV;?0WSWDP)B|tu`-VVe7@ox3z8-jWL zDkTUyI9t6Qh?yfkZ62q!1+9JTBQb6S!c&8w%?L~r!VBq*X-2d_dSOC{R)`uCMnoVg zObQW&urX;w2ExKn5LrkTLqjkiQ4AHqg0y1j2qwgfks#QR8b*dtK-MsLK3&C&w^%fc z6wjqG((liUCi*}=Awj4jGQsBp9ON{~An=vAHBHDd0q{oB$zwXFpV z@`#ILKxGp|=Cp>uz%__+a@4+e2z5CLo$V1JEF!gw%|zr2CTY+!=nFUS00000NkvXX Hu0mjfYje6+ literal 0 HcmV?d00001 diff --git a/mix/qml/img/edit.png b/mix/qml/img/edit.png new file mode 100644 index 0000000000000000000000000000000000000000..0530fd192ff2354ce5975c2ee306003d6df288e7 GIT binary patch literal 511 zcmVQKoj{#HH&G|BPM~fOCr~G_-9RU(|LZ}H;~)ts@cx85E*GFF zA3V}K=yEZWw48}!F$KsPgMEj5X$JiZ*GJ?RcM{kkH|XqhBx;^eesAm8Au{@B9pQh0 zx<=~RAt$DzG0X?*CG;pE5>5}uR|e6A`9+&kMu@v6+OR=KW0)Q4R%{R#af{AvytNU+ zyra$GZN{=j`ViLSV*XHgj=W`o3`O`D`7LQ@O?yUC5E4SNxEn9z4!4?+6ohTtE^Z&A z1sQ{ozK31{YBQ1@p%Gd|5Vax25H%r15w#%25j7xH5Tzm25G5g15v3s25hWlhh};l0 zL{5k*A{Rs*kpn`3I2J;KI0iz67z?38jDb)hQX#a66bLn9D1;s{1TqOkA7nC!F36-1 zG05Z)G00vjkfOVKralF1K2q-DZ<)$Q@#_Ex6(TevvQ;lf!8V0vg$@yde4Sz;!WX)sC4ZN&r>3^^+k|JSHq|Fm=YAd&F_nueR+21zt@+1{OAx^;sQHmQ*CtvQgIh?lfb5gTCvqYh_ zo05{@D%GZ6sUhAfZd+XfLQb=~ZjDL06Q#8A@hVe2m0vg1v}Xr%snu`1Z6?A#wR##; zPsa+^NX|VX9qeaQj@Q0fU1#uU(&8wvDUZzW!E}QReRQqJF*aED`j1U^wb{D*&-Qf) rPLGK4SIRu}u12V^bLrRXDxa7Ru2C?~D8DZU^bmumtDnm{r-UW|B%opw literal 0 HcmV?d00001 diff --git a/mix/qml/js/QEtherHelper.js b/mix/qml/js/QEtherHelper.js index 9761b2f45..7563941d2 100644 --- a/mix/qml/js/QEtherHelper.js +++ b/mix/qml/js/QEtherHelper.js @@ -6,3 +6,12 @@ function createEther(_value, _unit, _parent) ether.setUnit(_unit); return ether; } + +function createBigInt(_value) +{ + var bigintComponent = Qt.createComponent("qrc:/qml/BigIntValue.qml"); + var bigint = bigintComponent.createObject(); + bigint.setValue(_value); + return bigint; +} + diff --git a/mix/qml/js/TransactionHelper.js b/mix/qml/js/TransactionHelper.js index f404685cf..87dd74beb 100644 --- a/mix/qml/js/TransactionHelper.js +++ b/mix/qml/js/TransactionHelper.js @@ -5,7 +5,7 @@ function defaultTransaction() return { value: createEther("0", QEther.Wei), functionId: "", - gas: createEther("125000", QEther.Wei), + gas: createBigInt("125000"), gasPrice: createEther("100000", QEther.Wei), executeConstructor: false, parameters: {} diff --git a/mix/qml/qmldir b/mix/qml/qmldir index a90fd7135..ca8e494fe 100644 --- a/mix/qml/qmldir +++ b/mix/qml/qmldir @@ -1,3 +1,4 @@ +singleton Style 1.0 Style.qml singleton StateDialogStyle 1.0 StateDialogStyle.qml singleton ProjectFilesStyle 1.0 ProjectFilesStyle.qml singleton DebuggerPaneStyle 1.0 DebuggerPaneStyle.qml diff --git a/mix/res.qrc b/mix/res.qrc index 5d44e1299..6d9ce3d79 100644 --- a/mix/res.qrc +++ b/mix/res.qrc @@ -90,5 +90,12 @@ qml/CodeEditorStyle.qml qml/StatusPaneStyle.qml qml/StateStyle.qml + qml/img/plus.png + qml/img/delete_sign.png + qml/img/edit.png + qml/DefaultLabel.qml + qml/DefaultTextField.qml + qml/CommonSeparator.qml + qml/Style.qml From 6810ad3d86f1568e98af9c38d89457a9b615b1b7 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 11 Feb 2015 18:49:48 +0100 Subject: [PATCH 54/71] deploy to network --- mix/CodeModel.cpp | 9 +++- mix/CodeModel.h | 6 +-- mix/qml/ProjectModel.qml | 7 ++- mix/qml/StatusPane.qml | 8 +++- mix/qml/js/ProjectModel.js | 93 +++++++++++++++++++++++++++++++++++++- mix/qml/main.qml | 13 +++++- 6 files changed, 127 insertions(+), 9 deletions(-) diff --git a/mix/CodeModel.cpp b/mix/CodeModel.cpp index a9cfcc336..d2e06f6b3 100644 --- a/mix/CodeModel.cpp +++ b/mix/CodeModel.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include "QContractDefinition.h" #include "QFunctionDefinition.h" #include "QVariableDeclaration.h" @@ -61,7 +62,6 @@ CompilationResult::CompilationResult(const dev::solidity::CompilerStack& _compil auto const& contractDefinition = _compiler.getContractDefinition(std::string()); m_contract.reset(new QContractDefinition(&contractDefinition)); m_bytes = _compiler.getBytecode(); - m_assemblyCode = QString::fromStdString(dev::eth::disassemble(m_bytes)); dev::solidity::InterfaceHandler interfaceHandler; m_contractInterface = QString::fromStdString(*interfaceHandler.getABIInterface(contractDefinition)); if (m_contractInterface.isEmpty()) @@ -78,11 +78,16 @@ CompilationResult::CompilationResult(CompilationResult const& _prev, QString con m_contract(_prev.m_contract), m_compilerMessage(_compilerMessage), m_bytes(_prev.m_bytes), - m_assemblyCode(_prev.m_assemblyCode), m_contractInterface(_prev.m_contractInterface), m_codeHighlighter(_prev.m_codeHighlighter) {} + +QString CompilationResult::codeHex() const +{ + return QString::fromStdString(toJS(m_bytes)); +} + CodeModel::CodeModel(QObject* _parent): QObject(_parent), m_compiling(false), diff --git a/mix/CodeModel.h b/mix/CodeModel.h index 5f2add874..0262aa094 100644 --- a/mix/CodeModel.h +++ b/mix/CodeModel.h @@ -69,6 +69,7 @@ class CompilationResult: public QObject Q_PROPERTY(QString compilerMessage READ compilerMessage CONSTANT) Q_PROPERTY(bool successful READ successful CONSTANT) Q_PROPERTY(QString contractInterface READ contractInterface CONSTANT) + Q_PROPERTY(QString codeHex READ codeHex CONSTANT) public: /// Empty compilation result constructor @@ -88,8 +89,8 @@ public: QString compilerMessage() const { return m_compilerMessage; } /// @returns contract bytecode dev::bytes const& bytes() const { return m_bytes; } - /// @returns contract bytecode in human-readable form - QString assemblyCode() const { return m_assemblyCode; } + /// @returns contract bytecode as hex string + QString codeHex() const; /// @returns contract definition in JSON format QString contractInterface() const { return m_contractInterface; } /// Get code highlighter @@ -101,7 +102,6 @@ private: std::shared_ptr m_contract; QString m_compilerMessage; ///< @todo: use some structure here dev::bytes m_bytes; - QString m_assemblyCode; QString m_contractInterface; std::shared_ptr m_codeHighlighter; diff --git a/mix/qml/ProjectModel.qml b/mix/qml/ProjectModel.qml index 10dde5b41..98ed0b02e 100644 --- a/mix/qml/ProjectModel.qml +++ b/mix/qml/ProjectModel.qml @@ -20,6 +20,9 @@ Item { signal projectSaved() signal newProject(var projectData) signal documentSaved(var documentId) + signal deploymentStarted() + signal deploymentComplete() + signal deploymentError(string error) property bool isEmpty: (projectPath === "") readonly property string projectFileName: ".mix" @@ -28,6 +31,7 @@ Item { property string projectPath: "" property string projectTitle: "" property string currentDocumentId: "" + property string deploymentAddress: "" property var listModel: projectListModel property var stateListModel: projectStateListModel.model @@ -48,7 +52,8 @@ Item { function removeDocument(documentId) { ProjectModelCode.removeDocument(documentId); } function getDocument(documentId) { return ProjectModelCode.getDocument(documentId); } function getDocumentIndex(documentId) { return ProjectModelCode.getDocumentIndex(documentId); } - function doAddExistingFiles(paths) { ProjectModelCode.doAddExistingFiles(paths); } + function addExistingFiles(paths) { ProjectModelCode.doAddExistingFiles(paths); } + function deployProject() { ProjectModelCode.deployProject(); } Connections { target: appContext diff --git a/mix/qml/StatusPane.qml b/mix/qml/StatusPane.qml index 956d3f2ec..ddadb9953 100644 --- a/mix/qml/StatusPane.qml +++ b/mix/qml/StatusPane.qml @@ -38,11 +38,17 @@ Rectangle { Connections { target:clientModel - onRunStarted: infoMessage(qsTr("Running transactions..")); + onRunStarted: infoMessage(qsTr("Running transactions...")); onRunFailed: infoMessage(qsTr("Error running transactions")); onRunComplete: infoMessage(qsTr("Run complete")); onNewBlock: infoMessage(qsTr("New block created")); } + Connections { + target:projectModel + onDeploymentStarted: infoMessage(qsTr("Running deployment...")); + onDeploymentError: infoMessage(error); + onDeploymentComplete: infoMessage(qsTr("Deployment complete")); + } color: "transparent" anchors.fill: parent diff --git a/mix/qml/js/ProjectModel.js b/mix/qml/js/ProjectModel.js index 7c7a8ae4c..dcea66b13 100644 --- a/mix/qml/js/ProjectModel.js +++ b/mix/qml/js/ProjectModel.js @@ -39,7 +39,11 @@ function closeProject() { function saveProject() { if (!isEmpty) { - var projectData = { files: [] }; + var projectData = { + files: [], + title: projectTitle, + deploymentAddress: deploymentAddress + }; for (var i = 0; i < projectListModel.count; i++) projectData.files.push(projectListModel.get(i).fileName) projectSaving(projectData); @@ -60,6 +64,7 @@ function loadProject(path) { var parts = path.split("/"); projectData.title = parts[parts.length - 2]; } + deploymentAddress = projectData.deploymentAddress ? projectData.deploymentAddress : ""; projectTitle = projectData.title; projectPath = path; if (!projectData.files) @@ -246,3 +251,89 @@ function generateFileName(name, extension) { return fileName } + +var jsonRpcRequestId = 1; +function deployProject() { + + saveAll(); //TODO: ask user + + var deploymentId = Date.now().toLocaleString("ddMMyyyHHmmsszzz"); + var jsonRpcUrl = "http://localhost:8080"; + console.log("Deploying " + deploymentId + " to " + jsonRpcUrl); + deploymentStarted(); + var code = codeModel.codeHex + var rpcRequest = JSON.stringify({ + jsonrpc: "2.0", + method: "eth_transact", + params: [ { + "code": code + } ], + id: jsonRpcRequestId++ + }); + var httpRequest = new XMLHttpRequest() + httpRequest.open("POST", jsonRpcUrl, true); + httpRequest.setRequestHeader("Content-type", "application/json"); + httpRequest.setRequestHeader("Content-length", rpcRequest.length); + httpRequest.setRequestHeader("Connection", "close"); + httpRequest.onreadystatechange = function() { + if (httpRequest.readyState === XMLHttpRequest.DONE) { + if (httpRequest.status === 200) { + var rpcResponse = JSON.parse(httpRequest.responseText); + var address = rpcResponse.result; + console.log("Created contract, address: " + address); + finalizeDeployment(deploymentId, address); + } else { + var errorText = qsTr("Deployment error: RPC server HTTP status ") + http.status; + console.log(errorText); + deploymentError(errorText); + } + } + } + httpRequest.send(rpcRequest); +} + +function finalizeDeployment(deploymentId, address) { + //create a dir for frontend files and copy them + var deploymentDir = projectPath + deploymentId + "/"; + fileIo.makeDir(deploymentDir); + for (var i = 0; i < projectListModel.count; i++) { + var doc = projectListModel.get(i); + if (doc.isContract) + continue; + if (doc.isHtml) { + //inject the script to access contract API + //TODO: use a template + var html = fileIo.readFile(doc.path); + var insertAt = html.indexOf("") + if (insertAt < 0) + insertAt = 0; + else + insertAt += 6; + html = html.substr(0, insertAt) + + "" + + "" + + "" + + html.substr(insertAt); + fileIo.writeFile(deploymentDir + doc.fileName, html); + } + else + fileIo.copyFile(doc.path, deploymentDir + doc.fileName); + } + //write deployment js + var deploymentJs = + "// Autogenerated by Mix\n" + + "var web3 = require(\"web3\");\n" + + "var contractInterface = " + codeModel.code.contractInterface + ";\n" + + "deployment = {\n" + + "\tweb3: web3,\n" + + "\tcontractAddress: \"" + address + "\",\n" + + "\tcontractInterface: contractInterface,\n" + + "};\n" + + "deplyment.contract = web3.eth.contract(deplyment.contractAddress, deployment.contractInterface);\n"; + fileIo.writeFile(deploymentDir + "deployment.js", deploymentJs); + //copy scripts + fileIo.copyFile("qrc:///js/bignumber.min.js", deploymentDir + "bignumber.min.js"); + fileIo.copyFile("qrc:///js/webthree.js", deploymentDir + "ethereum.js"); + saveProject(); + deploymentComplete(); +} diff --git a/mix/qml/main.qml b/mix/qml/main.qml index 79430eb59..38f58c852 100644 --- a/mix/qml/main.qml +++ b/mix/qml/main.qml @@ -42,6 +42,8 @@ ApplicationWindow { MenuSeparator {} MenuItem { action: editStatesAction } MenuSeparator {} + MenuItem { action: deployViaRpcAction } + MenuSeparator {} MenuItem { action: toggleRunOnLoadAction } } Menu { @@ -265,7 +267,7 @@ ApplicationWindow { selectFolder: false onAccepted: { var paths = addExistingFileDialog.fileUrls; - projectModel.doAddExistingFiles(paths); + projectModel.addExistingFiles(paths); } } @@ -301,4 +303,13 @@ ApplicationWindow { onTriggered: projectModel.openPrevDocument(); } + Action { + id: deployViaRpcAction + text: qsTr("Deploy to Network") + shortcut: "Ctrl+Shift+D" + enabled: !projectModel.isEmpty && codeModel.hasContract + onTriggered: projectModel.deployProject(); + } + + } From d2b72415e878779e2568c11003fc9c52ad96947e Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 12 Feb 2015 16:32:29 +0100 Subject: [PATCH 55/71] web preview via http server --- mix/FileIo.cpp | 6 +++++ mix/HttpServer.cpp | 12 +++++---- mix/HttpServer.h | 4 +++ mix/qml/CodeEditorView.qml | 23 ++++++++++++----- mix/qml/ProjectModel.qml | 15 ++++++++++- mix/qml/WebPreview.qml | 47 ++++++++++++++++++++++++---------- mix/qml/html/WebContainer.html | 7 ++++- mix/qml/js/ProjectModel.js | 17 ++++++++---- 8 files changed, 99 insertions(+), 32 deletions(-) diff --git a/mix/FileIo.cpp b/mix/FileIo.cpp index 52fd57902..818d8c887 100644 --- a/mix/FileIo.cpp +++ b/mix/FileIo.cpp @@ -70,6 +70,12 @@ void FileIo::writeFile(QString const& _url, QString const& _data) void FileIo::copyFile(QString const& _sourceUrl, QString const& _destUrl) { + if (QUrl(_sourceUrl).scheme() == "qrc") + { + writeFile(_destUrl, readFile(_sourceUrl)); + return; + } + QUrl sourceUrl(_sourceUrl); QUrl destUrl(_destUrl); if (!QFile::copy(sourceUrl.path(), destUrl.path())) diff --git a/mix/HttpServer.cpp b/mix/HttpServer.cpp index cfe5c37f4..bf210444b 100644 --- a/mix/HttpServer.cpp +++ b/mix/HttpServer.cpp @@ -132,23 +132,25 @@ void HttpServer::readClient() if (socket->canReadLine()) { QString hdr = QString(socket->readLine()); - if (hdr.startsWith("POST")) + if (hdr.startsWith("POST") || hdr.startsWith("GET")) { + QUrl url(hdr.split(' ')[1]); QString l; do l = socket->readLine(); while (!(l.isEmpty() || l == "\r" || l == "\r\n")); QString content = socket->readAll(); - QUrl url; std::unique_ptr request(new HttpRequest(this, url, content)); clientConnected(request.get()); QTextStream os(socket); os.setAutoDetectUnicode(true); + QString q; ///@todo: allow setting response content-type, charset, etc - os << "HTTP/1.0 200 Ok\r\n" - "Content-Type: text/plain; charset=\"utf-8\"\r\n" - "\r\n"; + os << "HTTP/1.0 200 Ok\r\n"; + if (!request->m_responseContentType.isEmpty()) + os << "Content-Type: " << request->m_responseContentType << "; "; + os << "charset=\"utf-8\"\r\n\r\n"; os << request->m_response; } } diff --git a/mix/HttpServer.h b/mix/HttpServer.h index 00d63a073..add83238b 100644 --- a/mix/HttpServer.h +++ b/mix/HttpServer.h @@ -51,11 +51,15 @@ public: /// Set response for a request /// @param _response Response body. If no response is set, server returns status 200 with empty body Q_INVOKABLE void setResponse(QString const& _response) { m_response = _response; } + /// Set response content type + /// @param _contentType Response content type string. text/plain by default + Q_INVOKABLE void setResponseContentType(QString const& _contentType) { m_responseContentType = _contentType ; } private: QUrl m_url; QString m_content; QString m_response; + QString m_responseContentType; friend class HttpServer; }; diff --git a/mix/qml/CodeEditorView.qml b/mix/qml/CodeEditorView.qml index 36fc586b3..4d54994fe 100644 --- a/mix/qml/CodeEditorView.qml +++ b/mix/qml/CodeEditorView.qml @@ -4,18 +4,26 @@ import QtQuick.Layouts 1.0 import QtQuick.Controls 1.0 Item { - + id: codeEditorView property string currentDocumentId: "" + signal documentEdit(string documentId) function getDocumentText(documentId) { - for (i = 0; i < editorListModel.count; i++) { + for (var i = 0; i < editorListModel.count; i++) { if (editorListModel.get(i).documentId === documentId) { - return editors.itemAt(i).getText(); + return editors.itemAt(i).item.getText(); } } return ""; } + function isDocumentOpen(documentId) { + for (var i = 0; i < editorListModel.count; i++) + if (editorListModel.get(i).documentId === documentId) + return true; + return false; + } + function openDocument(document) { loadDocument(document); currentDocumentId = document.documentId; @@ -31,13 +39,16 @@ Item { function doLoadDocument(editor, document) { var data = fileIo.readFile(document.path); - if (document.isContract) - editor.onEditorTextChanged.connect(function() { + editor.onEditorTextChanged.connect(function() { + documentEdit(document.documentId); + if (document.isContract) codeModel.registerCodeChange(editor.getText()); - }); + }); editor.setText(data, document.syntaxMode); } + Component.onCompleted: projectModel.codeEditor = codeEditorView; + Connections { target: projectModel onDocumentOpened: { diff --git a/mix/qml/ProjectModel.qml b/mix/qml/ProjectModel.qml index 98ed0b02e..7e65f3174 100644 --- a/mix/qml/ProjectModel.qml +++ b/mix/qml/ProjectModel.qml @@ -34,6 +34,7 @@ Item { property string deploymentAddress: "" property var listModel: projectListModel property var stateListModel: projectStateListModel.model + property CodeEditorView codeEditor: null //interface function saveAll() { ProjectModelCode.saveAll(); } @@ -53,7 +54,7 @@ Item { function getDocument(documentId) { return ProjectModelCode.getDocument(documentId); } function getDocumentIndex(documentId) { return ProjectModelCode.getDocumentIndex(documentId); } function addExistingFiles(paths) { ProjectModelCode.doAddExistingFiles(paths); } - function deployProject() { ProjectModelCode.deployProject(); } + function deployProject() { ProjectModelCode.deployProject(false); } Connections { target: appContext @@ -88,6 +89,18 @@ Item { } } + MessageDialog { + id: deployWarningDialog + title: qsTr("Project") + text: qsTr("This project has been already deployed to the network. Do you want to re-deploy it?") + standardButtons: StandardButton.Ok | StandardButton.Cancel + icon: StandardIcon.Question + onAccepted: { + ProjectModelCode.deployProject(true); + } + } + + ListModel { id: projectListModel } diff --git a/mix/qml/WebPreview.qml b/mix/qml/WebPreview.qml index f4ddca84e..a952d6345 100644 --- a/mix/qml/WebPreview.qml +++ b/mix/qml/WebPreview.qml @@ -43,8 +43,8 @@ Item { } function changePage() { - if (pageCombo.currentIndex >=0 && pageCombo.currentIndex < pageListModel.count) { - setPreviewUrl(pageListModel.get(pageCombo.currentIndex).path); + if (pageCombo.currentIndex >= 0 && pageCombo.currentIndex < pageListModel.count) { + setPreviewUrl(httpServer.url + "/" + pageListModel.get(pageCombo.currentIndex).documentId); } else { setPreviewUrl(""); } @@ -54,7 +54,7 @@ Item { onAppLoaded: { //We need to load the container using file scheme so that web security would allow loading local files in iframe var containerPage = fileIo.readFile("qrc:///qml/html/WebContainer.html"); - webView.loadHtml(containerPage, "file:///WebContainer.html") + webView.loadHtml(containerPage, httpServer.url + "/WebContainer.html") } } @@ -112,16 +112,35 @@ Item { accept: true port: 8893 onClientConnected: { - //filter polling spam - //TODO: do it properly - //var log = _request.content.indexOf("eth_changed") < 0; - var log = true; - if (log) - console.log(_request.content); - var response = clientModel.apiCall(_request.content); - if (log) - console.log(response); - _request.setResponse(response); + var urlPath = _request.url.toString(); + if (urlPath.indexOf("/rpc/") === 0) + { + //jsonrpc request + //filter polling requests //TODO: do it properly + var log = _request.content.indexOf("eth_changed") < 0; + if (log) + console.log(_request.content); + var response = clientModel.apiCall(_request.content); + if (log) + console.log(response); + _request.setResponse(response); + } + else + { + //document request + var documentId = urlPath.substr(urlPath.lastIndexOf("/") + 1); + var content = ""; + if (projectModel.codeEditor.isDocumentOpen(documentId)) + content = projectModel.codeEditor.getDocumentText(documentId); + else + content = fileIo.readFile(projectModel.getDocument(documentId).path); + if (documentId === pageListModel.get(pageCombo.currentIndex).documentId) { + //root page, inject deployment script + content = "\n" + content; + _request.setResponseContentType("text/html"); + } + _request.setResponse(content); + } } } @@ -163,7 +182,7 @@ Item { onLoadingChanged: { if (!loading) { initialized = true; - webView.runJavaScript("init(\"" + httpServer.url + "\")"); + webView.runJavaScript("init(\"" + httpServer.url + "/rpc/\")"); if (pendingPageUrl) setPreviewUrl(pendingPageUrl); } diff --git a/mix/qml/html/WebContainer.html b/mix/qml/html/WebContainer.html index 04ba8ab73..e48668041 100644 --- a/mix/qml/html/WebContainer.html +++ b/mix/qml/html/WebContainer.html @@ -21,13 +21,18 @@ updateContract = function(address, contractFace) { window.contractAddress = address; window.contractInterface = contractFace; window.contract = window.web3.eth.contract(address, contractFace); + window.deploy = { + contractAddress: address, + contractInterface: contractFace, + contract: window.contract + }; } }; init = function(url) { web3 = require('web3'); - web3.setProvider(new web3.providers.HttpSyncProvider(url)); window.web3 = web3; + web3.setProvider(new web3.providers.HttpSyncProvider(url)); }; diff --git a/mix/qml/js/ProjectModel.js b/mix/qml/js/ProjectModel.js index dcea66b13..fb6872bd4 100644 --- a/mix/qml/js/ProjectModel.js +++ b/mix/qml/js/ProjectModel.js @@ -253,11 +253,17 @@ function generateFileName(name, extension) { var jsonRpcRequestId = 1; -function deployProject() { +function deployProject(force) { saveAll(); //TODO: ask user - var deploymentId = Date.now().toLocaleString("ddMMyyyHHmmsszzz"); + if (!force && deploymentAddress !== "") { + deployWarningDialog.visible = true; + return; + } + + var date = new Date(); + var deploymentId = date.toLocaleString(Qt.locale(), "ddMMyyHHmmsszzz"); var jsonRpcUrl = "http://localhost:8080"; console.log("Deploying " + deploymentId + " to " + jsonRpcUrl); deploymentStarted(); @@ -283,7 +289,7 @@ function deployProject() { console.log("Created contract, address: " + address); finalizeDeployment(deploymentId, address); } else { - var errorText = qsTr("Deployment error: RPC server HTTP status ") + http.status; + var errorText = qsTr("Deployment error: RPC server HTTP status ") + httpRequest.status; console.log(errorText); deploymentError(errorText); } @@ -324,16 +330,17 @@ function finalizeDeployment(deploymentId, address) { "// Autogenerated by Mix\n" + "var web3 = require(\"web3\");\n" + "var contractInterface = " + codeModel.code.contractInterface + ";\n" + - "deployment = {\n" + + "deploy = {\n" + "\tweb3: web3,\n" + "\tcontractAddress: \"" + address + "\",\n" + "\tcontractInterface: contractInterface,\n" + "};\n" + - "deplyment.contract = web3.eth.contract(deplyment.contractAddress, deployment.contractInterface);\n"; + "deploy.contract = web3.eth.contract(deploy.contractAddress, deploy.contractInterface);\n"; fileIo.writeFile(deploymentDir + "deployment.js", deploymentJs); //copy scripts fileIo.copyFile("qrc:///js/bignumber.min.js", deploymentDir + "bignumber.min.js"); fileIo.copyFile("qrc:///js/webthree.js", deploymentDir + "ethereum.js"); + deploymentAddress = address; saveProject(); deploymentComplete(); } From 90d309ec933bcf4b66bac98e6284597cfd51d55c Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 12 Feb 2015 16:37:06 +0100 Subject: [PATCH 56/71] style --- mix/CodeModel.cpp | 1 - mix/qml/ProjectModel.qml | 1 - mix/qml/main.qml | 2 -- 3 files changed, 4 deletions(-) diff --git a/mix/CodeModel.cpp b/mix/CodeModel.cpp index d2e06f6b3..aae9dac86 100644 --- a/mix/CodeModel.cpp +++ b/mix/CodeModel.cpp @@ -82,7 +82,6 @@ CompilationResult::CompilationResult(CompilationResult const& _prev, QString con m_codeHighlighter(_prev.m_codeHighlighter) {} - QString CompilationResult::codeHex() const { return QString::fromStdString(toJS(m_bytes)); diff --git a/mix/qml/ProjectModel.qml b/mix/qml/ProjectModel.qml index 7e65f3174..e74be7a9b 100644 --- a/mix/qml/ProjectModel.qml +++ b/mix/qml/ProjectModel.qml @@ -100,7 +100,6 @@ Item { } } - ListModel { id: projectListModel } diff --git a/mix/qml/main.qml b/mix/qml/main.qml index 38f58c852..71d8c24bf 100644 --- a/mix/qml/main.qml +++ b/mix/qml/main.qml @@ -310,6 +310,4 @@ ApplicationWindow { enabled: !projectModel.isEmpty && codeModel.hasContract onTriggered: projectModel.deployProject(); } - - } From 518ef9c18b3380408f2c878302d20b57d2aaaabe Mon Sep 17 00:00:00 2001 From: yann300 Date: Thu, 12 Feb 2015 17:57:45 +0100 Subject: [PATCH 57/71] ui improvement --- mix/AppContext.cpp | 4 ++-- mix/qml/CommonSeparator.qml | 2 +- mix/qml/DefaultLabel.qml | 5 +++-- mix/qml/StateDialog.qml | 2 ++ mix/qml/Style.qml | 7 +++++-- mix/qml/TransactionDialog.qml | 1 + 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/mix/AppContext.cpp b/mix/AppContext.cpp index 7fe22106f..64bc59ff8 100644 --- a/mix/AppContext.cpp +++ b/mix/AppContext.cpp @@ -60,6 +60,8 @@ AppContext::~AppContext() void AppContext::load() { m_applicationEngine->rootContext()->setContextProperty("appContext", this); + QFont f; + m_applicationEngine->rootContext()->setContextProperty("systemPointSize", f.pointSize()); qmlRegisterType("org.ethereum.qml", 1, 0, "FileIo"); m_applicationEngine->rootContext()->setContextProperty("codeModel", m_codeModel.get()); m_applicationEngine->rootContext()->setContextProperty("fileIo", m_fileIo.get()); @@ -81,8 +83,6 @@ void AppContext::load() BOOST_THROW_EXCEPTION(exception); } m_applicationEngine->rootContext()->setContextProperty("projectModel", projectModel); - QFont f; - m_applicationEngine->rootContext()->setContextProperty("systemPointSize", f.pointSize()); qmlRegisterType("CodeEditorExtensionManager", 1, 0, "CodeEditorExtensionManager"); qmlRegisterType("HttpServer", 1, 0, "HttpServer"); m_applicationEngine->load(QUrl("qrc:/qml/main.qml")); diff --git a/mix/qml/CommonSeparator.qml b/mix/qml/CommonSeparator.qml index 4d081e05f..c81a81f63 100644 --- a/mix/qml/CommonSeparator.qml +++ b/mix/qml/CommonSeparator.qml @@ -3,7 +3,7 @@ import "." Rectangle { - height: 3 + height: 1 color: Style.generic.layout.separatorColor } diff --git a/mix/qml/DefaultLabel.qml b/mix/qml/DefaultLabel.qml index da5495863..a1304e673 100644 --- a/mix/qml/DefaultLabel.qml +++ b/mix/qml/DefaultLabel.qml @@ -1,11 +1,12 @@ import QtQuick 2.0 import QtQuick.Controls 1.1 +import "." Label { text: text font.family: regularFont.name - - SourceSansProRegular + font.pointSize: Style.generic.size.titlePointSize + SourceSansProLight { id: regularFont } diff --git a/mix/qml/StateDialog.qml b/mix/qml/StateDialog.qml index ebcf5fd1d..eeda2ae22 100644 --- a/mix/qml/StateDialog.qml +++ b/mix/qml/StateDialog.qml @@ -132,6 +132,7 @@ Window { spacing: 0 RowLayout { + Layout.preferredWidth: 150 DefaultLabel { text: qsTr("Transactions: ") } @@ -142,6 +143,7 @@ Window { action: newTrAction width: 10 height: 10 + anchors.right: parent.right } Action { diff --git a/mix/qml/Style.qml b/mix/qml/Style.qml index 4dd5d978c..9e4b6f268 100644 --- a/mix/qml/Style.qml +++ b/mix/qml/Style.qml @@ -9,8 +9,11 @@ QtObject { } property QtObject generic: QtObject { - property QtObject layout : QtObject { - property string separatorColor: "#f7f7f7" + property QtObject layout: QtObject { + property string separatorColor: "#808080" + } + property QtObject size: QtObject { + property string titlePointSize: absoluteSize(0) } } } diff --git a/mix/qml/TransactionDialog.qml b/mix/qml/TransactionDialog.qml index cef82ecc9..a38886354 100644 --- a/mix/qml/TransactionDialog.qml +++ b/mix/qml/TransactionDialog.qml @@ -424,6 +424,7 @@ Window { CommonSeparator { Layout.fillWidth: true + visible: paramsModel.count > 0 } } From a98c86f797ea427ef96bc21c717198aff46eae6c Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 12 Feb 2015 20:08:01 +0100 Subject: [PATCH 58/71] added web3 to deploy object --- mix/qml/Debugger.qml | 2 +- mix/qml/html/WebContainer.html | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mix/qml/Debugger.qml b/mix/qml/Debugger.qml index 2bf23ccef..61bf5e8cc 100644 --- a/mix/qml/Debugger.qml +++ b/mix/qml/Debugger.qml @@ -426,7 +426,7 @@ Rectangle { Layout.minimumHeight: parent.height Text { anchors.centerIn: parent - anchors.leftMargin: 5() + anchors.leftMargin: 5 font.family: "monospace" color: "#4a4a4a" text: styleData.row; diff --git a/mix/qml/html/WebContainer.html b/mix/qml/html/WebContainer.html index e48668041..372860209 100644 --- a/mix/qml/html/WebContainer.html +++ b/mix/qml/html/WebContainer.html @@ -24,7 +24,8 @@ updateContract = function(address, contractFace) { window.deploy = { contractAddress: address, contractInterface: contractFace, - contract: window.contract + contract: window.contract, + web3: window.web3 }; } }; From 02f2070ddb1fa7e6ee07e1f6fe9ffb95151c785b Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 12 Feb 2015 18:38:07 +0100 Subject: [PATCH 59/71] Copying structs. --- libsolidity/AST.cpp | 6 +- libsolidity/ExpressionCompiler.cpp | 165 ++++++++++++++----------- libsolidity/ExpressionCompiler.h | 33 ++--- libsolidity/Types.cpp | 2 +- test/SolidityEndToEndTest.cpp | 60 +++++++++ test/SolidityNameAndTypeResolution.cpp | 2 +- 6 files changed, 175 insertions(+), 93 deletions(-) diff --git a/libsolidity/AST.cpp b/libsolidity/AST.cpp index 33cf8c12d..3c6b6007c 100644 --- a/libsolidity/AST.cpp +++ b/libsolidity/AST.cpp @@ -402,10 +402,8 @@ void Assignment::checkTypeRequirements() { m_leftHandSide->checkTypeRequirements(); m_leftHandSide->requireLValue(); - //@todo later, assignments to structs might be possible, but not to mappings - if (m_leftHandSide->getType()->getCategory() != Type::Category::ByteArray && - !m_leftHandSide->getType()->isValueType() && !m_leftHandSide->isLocalLValue()) - BOOST_THROW_EXCEPTION(createTypeError("Assignment to non-local non-value lvalue.")); + if (m_leftHandSide->getType()->getCategory() == Type::Category::Mapping) + BOOST_THROW_EXCEPTION(createTypeError("Mappings cannot be assigned to.")); m_type = m_leftHandSide->getType(); if (m_assigmentOperator == Token::Assign) m_rightHandSide->expectType(*m_type); diff --git a/libsolidity/ExpressionCompiler.cpp b/libsolidity/ExpressionCompiler.cpp index 3dbb4012d..90f860a1e 100644 --- a/libsolidity/ExpressionCompiler.cpp +++ b/libsolidity/ExpressionCompiler.cpp @@ -70,12 +70,12 @@ bool ExpressionCompiler::visit(Assignment const& _assignment) solAssert(_assignment.getType()->isValueType(), "Compound operators not implemented for non-value types."); if (m_currentLValue.storesReferenceOnStack()) m_context << eth::Instruction::SWAP1 << eth::Instruction::DUP2; - m_currentLValue.retrieveValue(_assignment.getType(), _assignment.getLocation(), true); + m_currentLValue.retrieveValue(_assignment.getLocation(), true); appendOrdinaryBinaryOperatorCode(Token::AssignmentToBinaryOp(op), *_assignment.getType()); if (m_currentLValue.storesReferenceOnStack()) m_context << eth::Instruction::SWAP1; } - m_currentLValue.storeValue(_assignment, *_assignment.getRightHandSide().getType()); + m_currentLValue.storeValue(*_assignment.getRightHandSide().getType(), _assignment.getLocation()); m_currentLValue.reset(); return false; @@ -105,13 +105,13 @@ bool ExpressionCompiler::visit(UnaryOperation const& _unaryOperation) break; case Token::Delete: // delete solAssert(m_currentLValue.isValid(), "LValue not retrieved."); - m_currentLValue.setToZero(_unaryOperation, *_unaryOperation.getSubExpression().getType()); + m_currentLValue.setToZero(_unaryOperation.getLocation()); m_currentLValue.reset(); break; case Token::Inc: // ++ (pre- or postfix) case Token::Dec: // -- (pre- or postfix) solAssert(m_currentLValue.isValid(), "LValue not retrieved."); - m_currentLValue.retrieveValue(_unaryOperation.getType(), _unaryOperation.getLocation()); + m_currentLValue.retrieveValue(_unaryOperation.getLocation()); if (!_unaryOperation.isPrefixOperation()) { if (m_currentLValue.storesReferenceOnStack()) @@ -128,7 +128,8 @@ bool ExpressionCompiler::visit(UnaryOperation const& _unaryOperation) // Stack for postfix: *ref [ref] (*ref)+-1 if (m_currentLValue.storesReferenceOnStack()) m_context << eth::Instruction::SWAP1; - m_currentLValue.storeValue(_unaryOperation, *_unaryOperation.getType(), !_unaryOperation.isPrefixOperation()); + m_currentLValue.storeValue(*_unaryOperation.getType(), _unaryOperation.getLocation(), + !_unaryOperation.isPrefixOperation()); m_currentLValue.reset(); break; case Token::Add: // + @@ -484,7 +485,7 @@ void ExpressionCompiler::endVisit(MemberAccess const& _memberAccess) { StructType const& type = dynamic_cast(*_memberAccess.getExpression().getType()); m_context << type.getStorageOffsetOfMember(member) << eth::Instruction::ADD; - m_currentLValue = LValue(m_context, LValue::LValueType::Storage, *_memberAccess.getType()); + m_currentLValue = LValue(m_context, LValue::LValueType::Storage, _memberAccess.getType()); m_currentLValue.retrieveValueIfLValueNotRequested(_memberAccess); break; } @@ -530,7 +531,7 @@ bool ExpressionCompiler::visit(IndexAccess const& _indexAccess) appendTypeMoveToMemory(IntegerType(256)); m_context << u256(0) << eth::Instruction::SHA3; - m_currentLValue = LValue(m_context, LValue::LValueType::Storage, *_indexAccess.getType()); + m_currentLValue = LValue(m_context, LValue::LValueType::Storage, _indexAccess.getType()); m_currentLValue.retrieveValueIfLValueNotRequested(_indexAccess); return false; @@ -939,8 +940,8 @@ void ExpressionCompiler::appendStateVariableAccessor(VariableDeclaration const& m_context << eth::Instruction::DUP1 << structType->getStorageOffsetOfMember(names[i]) << eth::Instruction::ADD; - m_currentLValue = LValue(m_context, LValue::LValueType::Storage, *types[i]); - m_currentLValue.retrieveValue(types[i], Location(), true); + m_currentLValue = LValue(m_context, LValue::LValueType::Storage, types[i]); + m_currentLValue.retrieveValue(Location(), true); solAssert(types[i]->getSizeOnStack() == 1, "Returning struct elements with stack size != 1 not yet implemented."); m_context << eth::Instruction::SWAP1; retSizeOnStack += types[i]->getSizeOnStack(); @@ -951,27 +952,51 @@ void ExpressionCompiler::appendStateVariableAccessor(VariableDeclaration const& { // simple value solAssert(accessorType.getReturnParameterTypes().size() == 1, ""); - m_currentLValue = LValue(m_context, LValue::LValueType::Storage, *returnType); - m_currentLValue.retrieveValue(returnType, Location(), true); + m_currentLValue = LValue(m_context, LValue::LValueType::Storage, returnType); + m_currentLValue.retrieveValue(Location(), true); retSizeOnStack = returnType->getSizeOnStack(); } solAssert(retSizeOnStack <= 15, "Stack too deep."); m_context << eth::dupInstruction(retSizeOnStack + 1) << eth::Instruction::JUMP; } -ExpressionCompiler::LValue::LValue(CompilerContext& _compilerContext, LValueType _type, Type const& _dataType, - unsigned _baseStackOffset): - m_context(&_compilerContext), m_type(_type), m_baseStackOffset(_baseStackOffset) +ExpressionCompiler::LValue::LValue(CompilerContext& _compilerContext, LValueType _type, + TypePointer const& _dataType, unsigned _baseStackOffset): + m_context(&_compilerContext), m_type(_type), m_dataType(_dataType), + m_baseStackOffset(_baseStackOffset) { //@todo change the type cast for arrays - solAssert(_dataType.getStorageSize() <= numeric_limits::max(), "The storage size of " +_dataType.toString() + " should fit in unsigned"); + solAssert(m_dataType->getStorageSize() <= numeric_limits::max(), + "The storage size of " + m_dataType->toString() + " should fit in unsigned"); if (m_type == LValueType::Storage) - m_size = unsigned(_dataType.getStorageSize()); + m_size = unsigned(m_dataType->getStorageSize()); else - m_size = unsigned(_dataType.getSizeOnStack()); + m_size = unsigned(m_dataType->getSizeOnStack()); } -void ExpressionCompiler::LValue::retrieveValue(TypePointer const& _type, Location const& _location, bool _remove) const +void ExpressionCompiler::LValue::fromIdentifier(Identifier const& _identifier, Declaration const& _declaration) +{ + if (m_context->isLocalVariable(&_declaration)) + { + m_type = LValueType::Stack; + m_dataType = _identifier.getType(); + m_size = m_dataType->getSizeOnStack(); + m_baseStackOffset = m_context->getBaseStackOffsetOfVariable(_declaration); + } + else if (m_context->isStateVariable(&_declaration)) + { + *m_context << m_context->getStorageLocationOfVariable(_declaration); + m_type = LValueType::Storage; + m_dataType = _identifier.getType(); + solAssert(m_dataType->getStorageSize() <= numeric_limits::max(), + "The storage size of " + m_dataType->toString() + " should fit in an unsigned"); + m_size = unsigned(m_dataType->getStorageSize()); } + else + BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_sourceLocation(_identifier.getLocation()) + << errinfo_comment("Identifier type not supported or identifier not found.")); +} + +void ExpressionCompiler::LValue::retrieveValue(Location const& _location, bool _remove) const { switch (m_type) { @@ -986,10 +1011,10 @@ void ExpressionCompiler::LValue::retrieveValue(TypePointer const& _type, Locatio break; } case LValueType::Storage: - retrieveValueFromStorage(_type, _remove); + retrieveValueFromStorage(_remove); break; case LValueType::Memory: - if (!_type->isValueType()) + if (!m_dataType->isValueType()) break; // no distinction between value and reference for non-value types BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_sourceLocation(_location) << errinfo_comment("Location type not yet implemented.")); @@ -1001,9 +1026,9 @@ void ExpressionCompiler::LValue::retrieveValue(TypePointer const& _type, Locatio } } -void ExpressionCompiler::LValue::retrieveValueFromStorage(TypePointer const& _type, bool _remove) const +void ExpressionCompiler::LValue::retrieveValueFromStorage(bool _remove) const { - if (!_type->isValueType()) + if (!m_dataType->isValueType()) return; // no distinction between value and reference for non-value types if (!_remove) *m_context << eth::Instruction::DUP1; @@ -1020,7 +1045,7 @@ void ExpressionCompiler::LValue::retrieveValueFromStorage(TypePointer const& _ty } } -void ExpressionCompiler::LValue::storeValue(Expression const& _expression, Type const& _sourceType, bool _move) const +void ExpressionCompiler::LValue::storeValue(Type const& _sourceType, Location const& _location, bool _move) const { switch (m_type) { @@ -1028,23 +1053,23 @@ void ExpressionCompiler::LValue::storeValue(Expression const& _expression, Type { unsigned stackDiff = m_context->baseToCurrentStackOffset(unsigned(m_baseStackOffset)) - m_size + 1; if (stackDiff > 16) - BOOST_THROW_EXCEPTION(CompilerError() << errinfo_sourceLocation(_expression.getLocation()) + BOOST_THROW_EXCEPTION(CompilerError() << errinfo_sourceLocation(_location) << errinfo_comment("Stack too deep.")); else if (stackDiff > 0) for (unsigned i = 0; i < m_size; ++i) *m_context << eth::swapInstruction(stackDiff) << eth::Instruction::POP; if (!_move) - retrieveValue(_expression.getType(), _expression.getLocation()); + retrieveValue(_location); break; } case LValueType::Storage: // stack layout: value value ... value target_ref - if (_expression.getType()->isValueType()) + if (m_dataType->isValueType()) { if (!_move) // copy values { if (m_size + 1 > 16) - BOOST_THROW_EXCEPTION(CompilerError() << errinfo_sourceLocation(_expression.getLocation()) + BOOST_THROW_EXCEPTION(CompilerError() << errinfo_sourceLocation(_location) << errinfo_comment("Stack too deep.")); for (unsigned i = 0; i < m_size; ++i) *m_context << eth::dupInstruction(m_size + 1) << eth::Instruction::SWAP1; @@ -1064,36 +1089,60 @@ void ExpressionCompiler::LValue::storeValue(Expression const& _expression, Type } else { - solAssert(!_move, "Move assign for non-value types not implemented."); - solAssert(_sourceType.getCategory() == _expression.getType()->getCategory(), ""); - if (_expression.getType()->getCategory() == Type::Category::ByteArray) + solAssert(_sourceType.getCategory() == m_dataType->getCategory(), ""); + if (m_dataType->getCategory() == Type::Category::ByteArray) CompilerUtils(*m_context).copyByteArrayToStorage( - dynamic_cast(*_expression.getType()), + dynamic_cast(*m_dataType), dynamic_cast(_sourceType)); - else if (_expression.getType()->getCategory() == Type::Category::Struct) + else if (m_dataType->getCategory() == Type::Category::Struct) { - //@todo - solAssert(false, "Struct copy not yet implemented."); + // stack layout: source_ref target_ref + auto const& structType = dynamic_cast(*m_dataType); + solAssert(structType == _sourceType, "Struct assignment with conversion."); + for (auto const& member: structType.getMembers()) + { + // assign each member that is not a mapping + TypePointer const& memberType = member.second; + if (memberType->getCategory() == Type::Category::Mapping) + continue; + *m_context << structType.getStorageOffsetOfMember(member.first) + << eth::Instruction::DUP3 << eth::Instruction::DUP2 + << eth::Instruction::ADD; + LValue rightHandSide(*m_context, LValueType::Storage, memberType); + rightHandSide.retrieveValue(_location, true); + // stack: source_ref target_ref offset source_value... + *m_context << eth::dupInstruction(2 + memberType->getSizeOnStack()) + << eth::dupInstruction(2 + memberType->getSizeOnStack()) + << eth::Instruction::ADD; + LValue memberLValue(*m_context, LValueType::Storage, memberType); + memberLValue.storeValue(*memberType, _location, true); + *m_context << eth::Instruction::POP; + } + if (_move) + *m_context << eth::Instruction::POP; + else + *m_context << eth::Instruction::SWAP1; + *m_context << eth::Instruction::POP; } else - BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_sourceLocation(_expression.getLocation()) + BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_sourceLocation(_location) << errinfo_comment("Invalid non-value type for assignment.")); } break; case LValueType::Memory: - if (!_expression.getType()->isValueType()) + if (!m_dataType->isValueType()) break; // no distinction between value and reference for non-value types - BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_sourceLocation(_expression.getLocation()) + BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_sourceLocation(_location) << errinfo_comment("Location type not yet implemented.")); break; default: - BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_sourceLocation(_expression.getLocation()) + BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_sourceLocation(_location) << errinfo_comment("Unsupported location type.")); break; } } -void ExpressionCompiler::LValue::setToZero(Expression const& _expression, Type const& _type) const +void ExpressionCompiler::LValue::setToZero(Location const& _location) const { switch (m_type) { @@ -1101,7 +1150,7 @@ void ExpressionCompiler::LValue::setToZero(Expression const& _expression, Type c { unsigned stackDiff = m_context->baseToCurrentStackOffset(unsigned(m_baseStackOffset)); if (stackDiff > 16) - BOOST_THROW_EXCEPTION(CompilerError() << errinfo_sourceLocation(_expression.getLocation()) + BOOST_THROW_EXCEPTION(CompilerError() << errinfo_sourceLocation(_location) << errinfo_comment("Stack too deep.")); solAssert(stackDiff >= m_size - 1, ""); for (unsigned i = 0; i < m_size; ++i) @@ -1110,8 +1159,8 @@ void ExpressionCompiler::LValue::setToZero(Expression const& _expression, Type c break; } case LValueType::Storage: - if (_type.getCategory() == Type::Category::ByteArray) - CompilerUtils(*m_context).clearByteArray(dynamic_cast(_type)); + if (m_dataType->getCategory() == Type::Category::ByteArray) + CompilerUtils(*m_context).clearByteArray(dynamic_cast(*m_dataType)); else { if (m_size == 0) @@ -1125,11 +1174,11 @@ void ExpressionCompiler::LValue::setToZero(Expression const& _expression, Type c } break; case LValueType::Memory: - BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_sourceLocation(_expression.getLocation()) + BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_sourceLocation(_location) << errinfo_comment("Location type not yet implemented.")); break; default: - BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_sourceLocation(_expression.getLocation()) + BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_sourceLocation(_location) << errinfo_comment("Unsupported location type.")); break; } @@ -1139,36 +1188,10 @@ void ExpressionCompiler::LValue::retrieveValueIfLValueNotRequested(Expression co { if (!_expression.lvalueRequested()) { - retrieveValue(_expression.getType(), _expression.getLocation(), true); + retrieveValue(_expression.getLocation(), true); reset(); } } -void ExpressionCompiler::LValue::fromStateVariable(Declaration const& _varDecl, TypePointer const& _type) -{ - m_type = LValueType::Storage; - solAssert(_type->getStorageSize() <= numeric_limits::max(), "The storage size of " + _type->toString() + " should fit in an unsigned"); - *m_context << m_context->getStorageLocationOfVariable(_varDecl); - m_size = unsigned(_type->getStorageSize()); -} - -void ExpressionCompiler::LValue::fromIdentifier(Identifier const& _identifier, Declaration const& _declaration) -{ - if (m_context->isLocalVariable(&_declaration)) - { - m_type = LValueType::Stack; - m_size = _identifier.getType()->getSizeOnStack(); - m_baseStackOffset = m_context->getBaseStackOffsetOfVariable(_declaration); - } - else if (m_context->isStateVariable(&_declaration)) - { - fromStateVariable(_declaration, _identifier.getType()); - } - else - BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_sourceLocation(_identifier.getLocation()) - << errinfo_comment("Identifier type not supported or identifier not found.")); -} - - } } diff --git a/libsolidity/ExpressionCompiler.h b/libsolidity/ExpressionCompiler.h index e0cc75ce8..734da50de 100644 --- a/libsolidity/ExpressionCompiler.h +++ b/libsolidity/ExpressionCompiler.h @@ -22,8 +22,10 @@ */ #include +#include #include #include +#include #include namespace dev { @@ -37,6 +39,7 @@ namespace solidity { class CompilerContext; class Type; class IntegerType; +class ByteArrayType; class StaticStringType; /** @@ -119,14 +122,13 @@ private: enum class LValueType { None, Stack, Memory, Storage }; explicit LValue(CompilerContext& _compilerContext): m_context(&_compilerContext) { reset(); } - LValue(CompilerContext& _compilerContext, LValueType _type, Type const& _dataType, unsigned _baseStackOffset = 0); + LValue(CompilerContext& _compilerContext, LValueType _type, + std::shared_ptr const& _dataType, unsigned _baseStackOffset = 0); /// Set type according to the declaration and retrieve the reference. /// @a _expression is the current expression void fromIdentifier(Identifier const& _identifier, Declaration const& _declaration); - /// Convenience function to set type for a state variable and retrieve the reference - void fromStateVariable(Declaration const& _varDecl, TypePointer const& _type); - void reset() { m_type = LValueType::None; m_baseStackOffset = 0; m_size = 0; } + void reset() { m_type = LValueType::None; m_dataType.reset(); m_baseStackOffset = 0; m_size = 0; } bool isValid() const { return m_type != LValueType::None; } bool isInOnStack() const { return m_type == LValueType::Stack; } @@ -138,30 +140,29 @@ private: /// Copies the value of the current lvalue to the top of the stack and, if @a _remove is true, /// also removes the reference from the stack (note that is does not reset the type to @a NONE). - /// @a _type is the type of the current expression and @ _location its location, used for error reporting. - /// @a _location can be a nullptr for expressions that don't have an actual ASTNode equivalent - void retrieveValue(TypePointer const& _type, Location const& _location, bool _remove = false) const; - /// Stores a value (from the stack directly beneath the reference, which is assumed to - /// be on the top of the stack, if any) in the lvalue and removes the reference. - /// Also removes the stored value from the stack if @a _move is - /// true. @a _expression is the current expression, used for error reporting. - /// @a _sourceType is the type of the expression that is assigned. - void storeValue(Expression const& _expression, Type const& _sourceType, bool _move = false) const; + /// @a _location source location of the current expression, used for error reporting. + void retrieveValue(Location const& _location, bool _remove = false) const; + /// Moves a value from the stack to the lvalue. Removes the value if @a _move is true. + /// @a _location is the source location of the expression that caused this operation. + /// Stack pre: [lvalue_ref] value + /// Stack post if !_move: value_of(lvalue_ref) + void storeValue(Type const& _sourceType, Location const& _location = Location(), bool _move = false) const; /// Stores zero in the lvalue. - /// @a _expression is the current expression, used for error reporting. - void setToZero(Expression const& _expression, Type const& _type) const; + /// @a _location is the source location of the requested operation + void setToZero(Location const& _location = Location()) const; /// Convenience function to convert the stored reference to a value and reset type to NONE if /// the reference was not requested by @a _expression. void retrieveValueIfLValueNotRequested(Expression const& _expression); private: /// Convenience function to retrieve Value from Storage. Specific version of @ref retrieveValue - void retrieveValueFromStorage(TypePointer const& _type, bool _remove = false) const; + void retrieveValueFromStorage(bool _remove = false) const; /// Copies from a byte array to a byte array in storage, both references on the stack. void copyByteArrayToStorage(ByteArrayType const& _targetType, ByteArrayType const& _sourceType) const; CompilerContext* m_context; LValueType m_type = LValueType::None; + std::shared_ptr m_dataType; /// If m_type is STACK, this is base stack offset (@see /// CompilerContext::getBaseStackOffsetOfVariable) of a local variable. unsigned m_baseStackOffset = 0; diff --git a/libsolidity/Types.cpp b/libsolidity/Types.cpp index 33cc8a1ec..16514b148 100644 --- a/libsolidity/Types.cpp +++ b/libsolidity/Types.cpp @@ -653,7 +653,7 @@ u256 StructType::getStorageOffsetOfMember(string const& _name) const { //@todo cache member offset? u256 offset; - for (ASTPointer variable: m_struct.getMembers()) + for (ASTPointer const& variable: m_struct.getMembers()) { if (variable->getName() == _name) return offset; diff --git a/test/SolidityEndToEndTest.cpp b/test/SolidityEndToEndTest.cpp index 587d4193f..abfc97b73 100644 --- a/test/SolidityEndToEndTest.cpp +++ b/test/SolidityEndToEndTest.cpp @@ -2424,6 +2424,66 @@ BOOST_AUTO_TEST_CASE(bytes_length_member) BOOST_CHECK(callContractFunction("getLength()") == encodeArgs(4+32+32)); } +BOOST_AUTO_TEST_CASE(struct_copy) +{ + char const* sourceCode = R"( + contract c { + struct Nested { uint x; uint y; } + struct Struct { uint a; mapping(uint => Struct) b; Nested nested; uint c; } + mapping(uint => Struct) public data; + function set(uint k) returns (bool) { + data[k].a = 1; + data[k].nested.x = 3; + data[k].nested.y = 4; + data[k].c = 2; + return true; + } + function copy(uint from, uint to) returns (bool) { + data[to] = data[from]; + return true; + } + function retrieve(uint k) returns (uint a, uint x, uint y, uint c) + { + a = data[k].a; + x = data[k].nested.x; + y = data[k].nested.y; + c = data[k].c; + } + } + )"; + compileAndRun(sourceCode); + BOOST_CHECK(callContractFunction("set(uint256)", 7) == encodeArgs(true)); + BOOST_CHECK(callContractFunction("retrieve(uint256)", 7) == encodeArgs(1, 3, 4, 2)); + BOOST_CHECK(callContractFunction("copy(uint256,uint256)", 7, 8) == encodeArgs(true)); + BOOST_CHECK(callContractFunction("retrieve(uint256)", 7) == encodeArgs(1, 3, 4, 2)); + BOOST_CHECK(callContractFunction("retrieve(uint256)", 8) == encodeArgs(1, 3, 4, 2)); + BOOST_CHECK(callContractFunction("copy(uint256,uint256)", 0, 7) == encodeArgs(true)); + BOOST_CHECK(callContractFunction("retrieve(uint256)", 7) == encodeArgs(0, 0, 0, 0)); + BOOST_CHECK(callContractFunction("retrieve(uint256)", 8) == encodeArgs(1, 3, 4, 2)); + BOOST_CHECK(callContractFunction("copy(uint256,uint256)", 7, 8) == encodeArgs(true)); + BOOST_CHECK(callContractFunction("retrieve(uint256)", 8) == encodeArgs(0, 0, 0, 0)); +} + +BOOST_AUTO_TEST_CASE(struct_copy_via_local) +{ + char const* sourceCode = R"( + contract c { + struct Struct { uint a; uint b; } + Struct data1; + Struct data2; + function test() returns (bool) { + data1.a = 1; + data1.b = 2; + var x = data1; + data2 = x; + return data2.a == data1.a && data2.b == data1.b; + } + } + )"; + compileAndRun(sourceCode); + BOOST_CHECK(callContractFunction("test()") == encodeArgs(true)); +} + BOOST_AUTO_TEST_SUITE_END() } diff --git a/test/SolidityNameAndTypeResolution.cpp b/test/SolidityNameAndTypeResolution.cpp index d013f5c5e..3ead6f1c5 100644 --- a/test/SolidityNameAndTypeResolution.cpp +++ b/test/SolidityNameAndTypeResolution.cpp @@ -332,7 +332,7 @@ BOOST_AUTO_TEST_CASE(assignment_to_struct) " data = a;\n" " }\n" "}\n"; - BOOST_CHECK_THROW(parseTextAndResolveNames(text), TypeError); + BOOST_CHECK_NO_THROW(parseTextAndResolveNames(text)); } BOOST_AUTO_TEST_CASE(returns_in_constructor) From f18c2aa378fd60a977649480bb3cd9fa80e77ddc Mon Sep 17 00:00:00 2001 From: winsvega Date: Thu, 12 Feb 2015 22:26:10 +0300 Subject: [PATCH 60/71] New Tests Solidity test Transaction test update --- test/stSolidityTestFiller.json | 237 ++++++++++++++++++++++++++++++ test/state.cpp | 5 + test/ttTransactionTestFiller.json | 70 ++++++--- 3 files changed, 292 insertions(+), 20 deletions(-) create mode 100644 test/stSolidityTestFiller.json diff --git a/test/stSolidityTestFiller.json b/test/stSolidityTestFiller.json new file mode 100644 index 000000000..fc0770c5e --- /dev/null +++ b/test/stSolidityTestFiller.json @@ -0,0 +1,237 @@ +{ + "SolidityTest" : { + "env" : { + "currentCoinbase" : "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty" : "45678256", + "currentGasLimit" : "1000000000000000000000", + "currentNumber" : "120", + "currentTimestamp" : 1, + "previousHash" : "5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6" + }, + "pre" : + { + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "10000000000000000", + "code" : "", + "nonce" : "0", + "storage" : { + } + }, + + "d94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "100000", + "//" : " ", + "//" : "contract TestContract ", + "//" : "{ ", + "//" : " function testMethod() returns (int res) ", + "//" : " { ", + "//" : " return 225; ", + "//" : " } ", + "//" : " ", + "//" : " function destroy(address sendFoundsTo) ", + "//" : " { ", + "//" : " suicide(sendFoundsTo); ", + "//" : " } ", + "//" : "} ", + "//" : " ", + "//" : "contract TestSolidityContracts ", + "//" : "{ ", + "//" : "struct StructTest ", + "//" : " { ", + "//" : " address addr; ", + "//" : " int amount; ", + "//" : " string32 str; ", + "//" : " mapping (uint => address) funders; ", + "//" : " } ", + "//" : " ", + "//" : " int globalValue; ", + "//" : " StructTest globalData; ", + "//" : " function runSolidityTests() returns (hash res) ", + "//" : " { ", + "//" : " //res is a mask of failing tests given the first byte is first test ", + "//" : " res = 0x0000000000000000000000000000000000000000000000000000000000000000; ", + "//" : " ", + "//" : " createContractFromMethod(); ", + "//" : " ", + "//" : " if (!testKeywords()) ", + "//" : " res = hash(int(res) + int(0xf000000000000000000000000000000000000000000000000000000000000000)); ", + "//" : " ", + "//" : " if (!testContractInteraction()) ", + "//" : " res = hash(int(res) + int(0x0f00000000000000000000000000000000000000000000000000000000000000)); ", + "//" : " ", + "//" : " if (!testContractSuicide()) ", + "//" : " res = hash(int(res) + int(0x00f0000000000000000000000000000000000000000000000000000000000000)); ", + "//" : " ", + "//" : " if (!testBlockAndTransactionProperties()) ", + "//" : " res = hash(int(res) + int(0x000f000000000000000000000000000000000000000000000000000000000000)); ", + "//" : " ", + "//" : " globalValue = 255; ", + "//" : " globalData.addr = 0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b; ", + "//" : " globalData.amount = 255; ", + "//" : " globalData.str = 'global data 32 length string'; ", + "//" : " globalData.funders[0] = 0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b; ", + "//" : " if (!testStructuresAndVariabless()) ", + "//" : " res = hash(int(res) + int(0x0000f00000000000000000000000000000000000000000000000000000000000)); ", + "//" : " ", + "//" : " //Tested 27.01.2015 ", + "//" : " //should run out of gas ", + "//" : " //if (!testInfiniteLoop()) ", + "//" : " // res = hash(int(res) + int(0x000f000000000000000000000000000000000000000000000000000000000000)); ", + "//" : " // ", + "//" : " //should run out of gas ", + "//" : " //if (!testRecursiveMethods()) ", + "//" : " // res = hash(int(res) + int(0x0000000000000000000000000000000000000000000000000000000000000000)); ", + "//" : " } ", + "//" : " ", + "//" : " function testStructuresAndVariabless() returns (bool res) ", + "//" : " { ", + "//" : " res = true; ", + "//" : " if (globalValue != 255) ", + "//" : " return false; ", + "//" : " ", + "//" : " if (globalValue != globalData.amount) ", + "//" : " return false; ", + "//" : " ", + "//" : " if (globalData.addr != 0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b) ", + "//" : " return false; ", + "//" : " ", + "//" : " if (globalData.str != 'global data 32 length string') ", + "//" : " return false; ", + "//" : " ", + "//" : " if (globalData.funders[0] != 0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b) ", + "//" : " return false; ", + "//" : " } ", + "//" : " ", + "//" : " function testBlockAndTransactionProperties() returns (bool res) ", + "//" : " { ", + "//" : " res = true; ", + "//" : " if (block.coinbase != 0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba) ", + "//" : " return false; ", + "//" : " ", + "//" : " if (block.difficulty != 45678256) ", + "//" : " return false; ", + "//" : " ", + "//" : " //for some reason does not work 27.01.2015 ", + "//" : " //if (block.gaslimit != 1000000000000000000000) ", + "//" : " // return false; ", + "//" : " ", + "//" : " if (block.number != 120) ", + "//" : " return false; ", + "//" : " ", + "//" : " //try to call this ", + "//" : " block.blockhash(120); ", + "//" : " block.timestamp; ", + "//" : " msg.gas; ", + "//" : " ", + "//" : " if (msg.sender != 0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b) ", + "//" : " return false; ", + "//" : " ", + "//" : " if (msg.value != 100) ", + "//" : " return false; ", + "//" : " ", + "//" : " if (tx.gasprice != 1) ", + "//" : " return false; ", + "//" : " ", + "//" : " if (tx.origin != 0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b) ", + "//" : " return false; ", + "//" : " ", + "//" : " } ", + "//" : " ", + "//" : " function testInfiniteLoop() returns (bool res) ", + "//" : " { ", + "//" : " res = false; ", + "//" : " while(true){} ", + "//" : " return true; ", + "//" : " } ", + "//" : " ", + "//" : " function testRecursiveMethods() returns (bool res) ", + "//" : " { ", + "//" : " res = false; ", + "//" : " testRecursiveMethods2(); ", + "//" : " return true; ", + "//" : " } ", + "//" : " function testRecursiveMethods2() ", + "//" : " { ", + "//" : " testRecursiveMethods(); ", + "//" : "} ", + "//" : " ", + "//" : " ", + "//" : " function testContractSuicide() returns (bool res) ", + "//" : " { ", + "//" : " TestContract a = new TestContract(); ", + "//" : " a.destroy(block.coinbase); ", + "//" : " if (a.testMethod() == 225) //we should be able to call a contract ", + "//" : " return true; ", + "//" : " return false; ", + "//" : " } ", + "//" : " ", + "//" : " function testContractInteraction() returns (bool res) ", + "//" : " { ", + "//" : " TestContract a = new TestContract(); ", + "//" : " if (a.testMethod() == 225) ", + "//" : " return true; ", + "//" : " return false; ", + "//" : " } ", + "//" : " ", + "//" : " function testKeywords() returns (bool res) ", + "//" : " { ", + "//" : " //some simple checks for the if statemnt ", + "//" : " //if, else, while, for, break, continue, return ", + "//" : " int i = 0; ", + "//" : " res = false; ", + "//" : " ", + "//" : " if (i == 0) ", + "//" : " { ", + "//" : " if( i <= -25) ", + "//" : " { ", + "//" : " return false; ", + "//" : " } ", + "//" : " else ", + "//" : " { ", + "//" : " while(i < 10) ", + "//" : " i++; ", + "//" : " ", + "//" : " if (i == 10) ", + "//" : " { ", + "//" : " for(var j=10; j>0; j--) ", + "//" : " { ", + "//" : " i--; ", + "//" : " } ", + "//" : " } ", + "//" : " } ", + "//" : " } ", + "//" : " ", + "//" : " ", + "//" : " if (i == 0) ", + "//" : " return true; ", + "//" : " ", + "//" : " return false; ", + "//" : " } ", + "//" : " ", + "//" : " function createContractFromMethod() returns (TestContract a) ", + "//" : " { ", + "//" : " a = new TestContract(); ", + "//" : " } ", + "//" : "} ", + "code" : "0x60e060020a6000350480630c4c9a8014610078578063296df0df1461008a5780632a9afb831461009c578063380e4396146100ae5780634893d88a146100c05780637ee17e12146100ce578063981a3165146100dc578063a60eedda146100ee578063e97384dc14610100578063ed973fe91461011257005b610080610431565b8060005260206000f35b6100926103f7565b8060005260206000f35b6100a46105d1565b8060005260206000f35b6100b6610220565b8060005260206000f35b6100c8610426565b60006000f35b6100d66102df565b60006000f35b6100e4610411565b8060005260206000f35b6100f6610124565b8060005260206000f35b6101086102f5565b8060005260206000f35b61011a6101be565b8060005260206000f35b60006000605f6106be600039605f60006000f0905080600160a060020a031662f55d9d8060e060020a0260005241600160a060020a0316600452600060006024600060008660155a03f150505080600160a060020a031663b9c3d0a58060e060020a02600052602060006004600060008660155a03f150505060005160e1146101ac576101b5565b600191506101ba565b600091505b5090565b60006000605f6106be600039605f60006000f0905080600160a060020a031663b9c3d0a58060e060020a02600052602060006004600060008660155a03f150505060005160e11461020e57610217565b6001915061021c565b600091505b5090565b60006000600060009150600092508160001461023b576102bf565b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe78213156102b5575b600a82121561027a578180600101925050610264565b81600a14610287576102b0565b600a90505b60008160ff1611156102af5781806001900392505080806001900391505061028c565b5b6102be565b600092506102da565b5b816000146102cc576102d5565b600192506102da565b600092505b505090565b6000605f6106be600039605f60006000f0905090565b60006001905041600160a060020a0316732adc25665018aa1fe0e6bc666dac8fc2697ff9ba14156103255761032e565b600090506103f4565b446302b8feb0141561033f57610348565b600090506103f4565b43607814156103565761035f565b600090506103f4565b33600160a060020a031673a94f5374fce5edbc8e2a8697c15331677e6ebf0b141561038957610392565b600090506103f4565b34606414156103a0576103a9565b600090506103f4565b3a600114156103b7576103c0565b600090506103f4565b32600160a060020a031673a94f5374fce5edbc8e2a8697c15331677e6ebf0b14156103ea576103f3565b600090506103f4565b5b90565b6000600090505b60011561040a576103fe565b6001905090565b60006000905061041f610426565b6001905090565b61042e610411565b50565b60006000905061043f6102df565b50610448610220565b1561045257610478565b7ff000000000000000000000000000000000000000000000000000000000000000810190505b6104806101be565b1561048a576104b0565b7f0f00000000000000000000000000000000000000000000000000000000000000810190505b6104b8610124565b156104c2576104e7565b7ef0000000000000000000000000000000000000000000000000000000000000810190505b6104ef6102f5565b156104f95761051e565b7e0f000000000000000000000000000000000000000000000000000000000000810190505b60ff60008190555073a94f5374fce5edbc8e2a8697c15331677e6ebf0b60018190555060ff6002819055507f676c6f62616c2064617461203332206c656e67746820737472696e670000000060038190555073a94f5374fce5edbc8e2a8697c15331677e6ebf0b600460006000526020526040600020819055506105a06105d1565b156105aa576105ce565b7df00000000000000000000000000000000000000000000000000000000000810190505b90565b60006001905060005460ff14156105e7576105f0565b600090506106ba565b60025460005414156106015761060a565b600090506106ba565b600154600160a060020a031673a94f5374fce5edbc8e2a8697c15331677e6ebf0b14156106365761063f565b600090506106ba565b6003547f676c6f62616c2064617461203332206c656e67746820737472696e6700000000141561066e57610677565b600090506106ba565b60046000600052602052604060002054600160a060020a031673a94f5374fce5edbc8e2a8697c15331677e6ebf0b14156106b0576106b9565b600090506106ba565b5b905600605380600c6000396000f30060e060020a600035048062f55d9d14601d578063b9c3d0a514602c57005b60266004356045565b60006000f35b6032603c565b8060005260206000f35b600060e1905090565b80600160a060020a0316ff5056", + "nonce" : "0", + "storage" : { + } + } + }, + + "transaction" : + { + "//" : "createContractFromMethod()", + "data" : "0x7ee17e12", + "//" : "runSolidityTests()", + "data" : "0x0c4c9a80", + "gasLimit" : "465224", + "gasPrice" : "1", + "nonce" : "0", + "secretKey" : "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8", + "to" : "d94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "value" : "100" + } + } +} diff --git a/test/state.cpp b/test/state.cpp index fb54a62ae..03f01d0fb 100644 --- a/test/state.cpp +++ b/test/state.cpp @@ -159,6 +159,11 @@ BOOST_AUTO_TEST_CASE(stBlockHashTest) dev::test::executeTests("stBlockHashTest", "/StateTests", dev::test::doStateTests); } +BOOST_AUTO_TEST_CASE(stSolidityTest) +{ + dev::test::executeTests("stSolidityTest", "/StateTests", dev::test::doStateTests); +} + BOOST_AUTO_TEST_CASE(stCreateTest) { for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i) diff --git a/test/ttTransactionTestFiller.json b/test/ttTransactionTestFiller.json index 49831261a..e9ae27188 100644 --- a/test/ttTransactionTestFiller.json +++ b/test/ttTransactionTestFiller.json @@ -44,6 +44,21 @@ } }, + "WrongVRSTestIncorrectSize" : { + "transaction" : + { + "data" : "", + "gasLimit" : "2000", + "gasPrice" : "1", + "nonce" : "0", + "to" : "b94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "value" : "10", + "v" : "28", + "r" : "0x98ff921201554726367d2be8c804a7ff89ccf285ebc57dff8ae4c44b9c19ac4a02c3", + "s" : "0x8887321be575c8095f789dd4c743dfe42c1820f9231f98a962b210e3ac2452a302c3" + } + }, + "SenderTest" : { "//" : "sender a94f5374fce5edbc8e2a8697c15331677e6ebf0b", "transaction" : @@ -196,33 +211,18 @@ } }, - "RLPElementsWithZeros" : { + "AddressMoreThan20PrefixedBy0" : { "transaction" : { - "data" : "0x0000011222333", - "gasLimit" : "1000", - "gasPrice" : "00123", - "nonce" : "0054", - "to" : "095e7baea6a6c7c4c2dfeb977efac326af552d87", - "value" : "00000011", - "v" : "27", - "r" : "0x48b55bfa915ac795c431978d8a6a992b628d557da5ff759b307d495a36649353", - "s" : "0xefffd310ac743f371de3b9f7f9cb56c0b28ad43601b4ab949f53faa07bd2c804" - } - }, - - "RLPWrongHexElements" : { - "transaction" : - { - "data" : "0x0000000012", + "data" : "0x12", "gasLimit" : "1000", "gasPrice" : "123", "nonce" : "54", - "to" : "095e7baea6a6c7c4c2dfeb977efac326af552d87", + "to" : "0x0000000000000000095e7baea6a6c7c4c2dfeb977efac326af552d87", "value" : "11", "v" : "27", - "r" : "0x0048b55bfa915ac795c431978d8a6a992b628d557da5ff759b307d495a36649353", - "s" : "0x00efffd310ac743f371de3b9f7f9cb56c0b28ad43601b4ab949f53faa07bd2c804" + "r" : "0x48b55bfa915ac795c431978d8a6a992b628d557da5ff759b307d495a36649353", + "s" : "0xefffd310ac743f371de3b9f7f9cb56c0b28ad43601b4ab949f53faa07bd2c804" } }, @@ -299,5 +299,35 @@ "r" : "0x98ff921201554726367d2be8c804a7ff89ccf285ebc57dff8ae4c44b9c19ac4a", "s" : "0x8887321be575c8095f789dd4c743dfe42c1820f9231f98a962b210e3ac2452a3" } + }, + + "ValuesAsHex" : { + "transaction" : + { + "data" : "", + "gasLimit" : "0xadc053", + "gasPrice" : "1", + "nonce" : "0xffdc5", + "to" : "b94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "value" : "0xfffdc12c", + "v" : "28", + "r" : "0x98ff921201554726367d2be8c804a7ff89ccf285ebc57dff8ae4c44b9c19ac4a", + "s" : "0x8887321be575c8095f789dd4c743dfe42c1820f9231f98a962b210e3ac2452a3" + } + }, + + "ValuesAsDec" : { + "transaction" : + { + "data" : "", + "gasLimit" : "11386963", + "gasPrice" : "1", + "nonce" : "1048005", + "to" : "b94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "value" : "4501151495864620", + "v" : "28", + "r" : "0x98ff921201554726367d2be8c804a7ff89ccf285ebc57dff8ae4c44b9c19ac4a", + "s" : "0x8887321be575c8095f789dd4c743dfe42c1820f9231f98a962b210e3ac2452a3" + } } } From 48dee28af5703c4db78515df60b07446a2b0beff Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 12 Feb 2015 20:57:25 +0100 Subject: [PATCH 61/71] 'web3' and 'contracts' for JS context instead of 'deploy' --- mix/qml/WebPreview.qml | 4 ++-- mix/qml/html/WebContainer.html | 16 +++++++--------- mix/qml/js/ProjectModel.js | 14 +++++++------- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/mix/qml/WebPreview.qml b/mix/qml/WebPreview.qml index 08f25b3df..10d0b3c83 100644 --- a/mix/qml/WebPreview.qml +++ b/mix/qml/WebPreview.qml @@ -28,7 +28,7 @@ Item { } function updateContract() { - webView.runJavaScript("updateContract(\"" + clientModel.contractAddress + "\", " + codeModel.code.contractInterface + ")"); + webView.runJavaScript("updateContract(\"" + codeModel.code.contract.name + "\", \"" + clientModel.contractAddress + "\", " + codeModel.code.contractInterface + ")"); } function reloadOnSave() { @@ -136,7 +136,7 @@ Item { content = fileIo.readFile(projectModel.getDocument(documentId).path); if (documentId === pageListModel.get(pageCombo.currentIndex).documentId) { //root page, inject deployment script - content = "\n" + content; + content = "\n" + content; _request.setResponseContentType("text/html"); } _request.setResponse(content); diff --git a/mix/qml/html/WebContainer.html b/mix/qml/html/WebContainer.html index 372860209..09a8734d5 100644 --- a/mix/qml/html/WebContainer.html +++ b/mix/qml/html/WebContainer.html @@ -15,17 +15,15 @@ reloadPage = function() { preview.contentWindow.location.reload(); }; -updateContract = function(address, contractFace) { +updateContract = function(name, address, contractFace) { if (window.web3) { window.web3.provider.polls = []; - window.contractAddress = address; - window.contractInterface = contractFace; - window.contract = window.web3.eth.contract(address, contractFace); - window.deploy = { - contractAddress: address, - contractInterface: contractFace, - contract: window.contract, - web3: window.web3 + var contract = window.web3.eth.contract(address, contractFace); + window.contracts = {}; + window.contracts[name] = { + address: address, + interface: contractFace, + contract: contract, }; } }; diff --git a/mix/qml/js/ProjectModel.js b/mix/qml/js/ProjectModel.js index fb6872bd4..be6c07c5b 100644 --- a/mix/qml/js/ProjectModel.js +++ b/mix/qml/js/ProjectModel.js @@ -326,16 +326,16 @@ function finalizeDeployment(deploymentId, address) { fileIo.copyFile(doc.path, deploymentDir + doc.fileName); } //write deployment js + var contractAccessor = "contracts[\"" + codeModel.code.contract.name + "\"]"; var deploymentJs = "// Autogenerated by Mix\n" + - "var web3 = require(\"web3\");\n" + - "var contractInterface = " + codeModel.code.contractInterface + ";\n" + - "deploy = {\n" + - "\tweb3: web3,\n" + - "\tcontractAddress: \"" + address + "\",\n" + - "\tcontractInterface: contractInterface,\n" + + "web3 = require(\"web3\");\n" + + "contracts = {};\n" + + contractAccessor + " = {\n" + + "\tinterface: " + codeModel.code.contractInterface + ",\n" + + "\taddress: \"" + address + "\"\n" + "};\n" + - "deploy.contract = web3.eth.contract(deploy.contractAddress, deploy.contractInterface);\n"; + contractAccessor + ".contract = web3.eth.contract(" + contractAccessor + ".address, " + contractAccessor + ".interface);\n"; fileIo.writeFile(deploymentDir + "deployment.js", deploymentJs); //copy scripts fileIo.copyFile("qrc:///js/bignumber.min.js", deploymentDir + "bignumber.min.js"); From 29abb4fb551f17442955e56160ff86ea4a622892 Mon Sep 17 00:00:00 2001 From: winsvega Date: Thu, 12 Feb 2015 23:43:57 +0300 Subject: [PATCH 62/71] New Tests Transaction with RLP object and catching exceptions from importByteArray --- test/TestHelper.cpp | 2 +- test/stTransactionTestFiller.json | 152 +++++++++++++++++++++++++++++- test/transaction.cpp | 11 ++- 3 files changed, 157 insertions(+), 8 deletions(-) diff --git a/test/TestHelper.cpp b/test/TestHelper.cpp index 8a00a5462..5e7472105 100644 --- a/test/TestHelper.cpp +++ b/test/TestHelper.cpp @@ -247,7 +247,7 @@ byte toByte(json_spirit::mValue const& _v) bytes importByteArray(std::string const& _str) { - return fromHex(_str.substr(0, 2) == "0x" ? _str.substr(2) : _str); + return fromHex(_str.substr(0, 2) == "0x" ? _str.substr(2) : _str, ThrowType::Throw); } bytes importData(json_spirit::mObject& _o) diff --git a/test/stTransactionTestFiller.json b/test/stTransactionTestFiller.json index e5369f0cd..214a0ae7c 100644 --- a/test/stTransactionTestFiller.json +++ b/test/stTransactionTestFiller.json @@ -189,6 +189,53 @@ } }, + "InternalCallHittingGasLimit" : { + "env" : { + "currentCoinbase" : "a94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "currentDifficulty" : "45678256", + "currentGasLimit" : "1100", + "currentNumber" : "0", + "currentTimestamp" : 1, + "previousHash" : "5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6" + }, + "pre" : + { + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "1000000", + "code" : "", + "nonce" : "0", + "storage" : { + } + }, + + "b94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "1000000", + "code" : "{ (CALL 5000 0xc94f5374fce5edbc8e2a8697c15331677e6ebf0b 1 0 0 0 0) }", + "nonce" : "0", + "storage" : { + } + }, + + "c94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "0", + "code" : "{[[1]]55}", + "nonce" : "0", + "storage" : { + } + } + }, + "transaction" : + { + "data" : "", + "gasLimit" : "1100", + "gasPrice" : "1", + "nonce" : "", + "secretKey" : "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8", + "to" : "b94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "value" : "10" + } + }, + "TransactionFromCoinbaseHittingBlockGasLimit" : { "env" : { "currentCoinbase" : "a94f5374fce5edbc8e2a8697c15331677e6ebf0b", @@ -623,7 +670,7 @@ "balance" : "100000", "code" : "", "nonce" : "115792089237316195423570985008687907853269984665640564039457584007913129639935", - "nonce" : "10000000", + "nonce" : "10000", "storage" : { } } @@ -634,7 +681,7 @@ "gasLimit" : "1000", "gasPrice" : "1", "nonce" : "115792089237316195423570985008687907853269984665640564039457584007913129639935", - "nonce" : "10000000", + "nonce" : "10000", "secretKey" : "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8", "to" : "b94f5374fce5edbc8e2a8697c15331677e6ebf0b", "value" : "100" @@ -864,5 +911,106 @@ "to" : "0xffffffffffffffffffffffffffffffffffffffff", "value" : "100" } + }, + + "CreateTransactionReverted" : { + "env" : { + "currentCoinbase" : "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty" : "45678256", + "currentGasLimit" : "1000000000000", + "currentNumber" : "0", + "currentTimestamp" : 1, + "previousHash" : "5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6" + }, + "pre" : + { + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "10000", + "code" : "", + "nonce" : "0", + "storage" : { + } + } + }, + "transaction" : + { + "data" : "0x602280600c6000396000f30060e060020a600035048063f8a8fd6d14601457005b601a6020565b60006000f35b56", + "gasLimit" : "882", + "gasPrice" : "1", + "nonce" : "0", + "secretKey" : "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8", + "to" : "", + "value" : "" + } + }, + + "CreateTransactionWorking" : { + "env" : { + "currentCoinbase" : "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty" : "45678256", + "currentGasLimit" : "1000000000000", + "currentNumber" : "0", + "currentTimestamp" : 1, + "previousHash" : "5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6" + }, + "pre" : + { + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "10000", + "code" : "", + "nonce" : "0", + "storage" : { + } + } + }, + "transaction" : + { + "data" : "0x602280600c6000396000f30060e060020a600035048063f8a8fd6d14601457005b601a6020565b60006000f35b56", + "gasLimit" : "883", + "gasPrice" : "1", + "nonce" : "0", + "secretKey" : "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8", + "to" : "", + "value" : "100" + } + }, + + "CreateMessageReverted" : { + "env" : { + "currentCoinbase" : "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty" : "45678256", + "currentGasLimit" : "1000000000000", + "currentNumber" : "0", + "currentTimestamp" : 1, + "previousHash" : "5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6" + }, + "pre" : + { + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "10000", + "code" : "", + "nonce" : "0", + "storage" : { + } + }, + + "b94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "0", + "code" : "{(MSTORE 0 0x600c600055) (CREATE 0 27 5)}", + "nonce" : "0", + "storage" : { + } + } + }, + "transaction" : + { + "data" : "", + "gasLimit" : "882", + "gasPrice" : "1", + "nonce" : "0", + "secretKey" : "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8", + "to" : "b94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "value" : "100" + } } } diff --git a/test/transaction.cpp b/test/transaction.cpp index b51494d84..9f9429534 100644 --- a/test/transaction.cpp +++ b/test/transaction.cpp @@ -64,7 +64,7 @@ RLPStream createRLPStreamFromTransactionFields(mObject& _tObj) rlpStream << bigint(_tObj["r"].get_str()); if (_tObj.count("s") > 0) - rlpStream << bigint(_tObj["s"].get_str()); + rlpStream << bigint(_tObj["s"].get_str()); if (_tObj.count("extrafield") > 0) rlpStream << bigint(_tObj["extrafield"].get_str()); @@ -82,18 +82,18 @@ void doTransactionTests(json_spirit::mValue& _v, bool _fillin) if (_fillin == false) { BOOST_REQUIRE(o.count("rlp") > 0); - bytes rlpReaded = importByteArray(o["rlp"].get_str()); Transaction txFromRlp; - try { - txFromRlp = Transaction(rlpReaded, CheckSignature::Sender); + bytes stream = importByteArray(o["rlp"].get_str()); + RLP rlp(stream); + txFromRlp = Transaction(rlp.data(), CheckSignature::Sender); if (!txFromRlp.signature().isValid()) BOOST_THROW_EXCEPTION(Exception() << errinfo_comment("transaction from RLP signature is invalid") ); } catch(...) { - BOOST_CHECK_MESSAGE(o.count("transaction") == 0, "A transction object should not be defined because the RLP is invalid!"); + BOOST_CHECK_MESSAGE(o.count("transaction") == 0, "A transaction object should not be defined because the RLP is invalid!"); return; } @@ -115,6 +115,7 @@ void doTransactionTests(json_spirit::mValue& _v, bool _fillin) Address addressReaded = Address(o["sender"].get_str()); BOOST_CHECK_MESSAGE(txFromFields.sender() == addressReaded || txFromRlp.sender() == addressReaded, "Signature address of sender does not match given sender address!"); + } else { From e1666c85e5ccbfaa7d2cd757fb5172b4feb8511d Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 12 Feb 2015 21:52:30 +0100 Subject: [PATCH 63/71] Ctrl+C/Cmd+C to copy data from debug panes --- mix/qml/DebugInfoList.qml | 11 ++++++++++- mix/qml/Debugger.qml | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/mix/qml/DebugInfoList.qml b/mix/qml/DebugInfoList.qml index c1ab11596..ae7e6fabe 100644 --- a/mix/qml/DebugInfoList.qml +++ b/mix/qml/DebugInfoList.qml @@ -8,7 +8,7 @@ ColumnLayout { property string title property variant listModel; property bool collapsible; - property bool enableSelection; + property bool enableSelection: false; property real storedHeight: 0; property Component itemDelegate signal rowActivated(int index) @@ -116,6 +116,15 @@ ColumnLayout { } } onActivated: rowActivated(row); + Keys.onPressed: { + if ((event.modifiers & Qt.ControlModifier) && event.key === Qt.Key_C && currentRow >=0 && currentRow < listModel.length) { + var str = ""; + for (var i = 0; i < listModel.length; i++) + str += listModel[i] + "\n"; + appContext.toClipboard(str); + } + } + TableViewColumn { role: "modelData" width: parent.width diff --git a/mix/qml/Debugger.qml b/mix/qml/Debugger.qml index 61bf5e8cc..df91aa1c4 100644 --- a/mix/qml/Debugger.qml +++ b/mix/qml/Debugger.qml @@ -529,7 +529,7 @@ Rectangle { Text { anchors.leftMargin: 5 width: parent.width - 5 - wrapMode: Text.Wrap + wrapMode: Text.NoWrap anchors.left: parent.left font.family: "monospace" anchors.verticalCenter: parent.verticalCenter @@ -600,7 +600,7 @@ Rectangle { Text { anchors.leftMargin: 5 width: parent.width - 5 - wrapMode: Text.Wrap + wrapMode: Text.NoWrap anchors.left: parent.left font.family: "monospace" anchors.verticalCenter: parent.verticalCenter From 1ffdd5d7ff32558437a6c1277a9b33b6daad2e88 Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 13 Feb 2015 01:29:21 +0100 Subject: [PATCH 64/71] Copying calldata directly to memory. --- libsolidity/CompilerUtils.cpp | 63 ++++++++++++++++++++--------------- test/SolidityEndToEndTest.cpp | 15 +++++++++ 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/libsolidity/CompilerUtils.cpp b/libsolidity/CompilerUtils.cpp index 73be38176..dda1736d6 100644 --- a/libsolidity/CompilerUtils.cpp +++ b/libsolidity/CompilerUtils.cpp @@ -76,35 +76,44 @@ void CompilerUtils::storeInMemoryDynamic(Type const& _type, bool _padToWordBound if (_type.getCategory() == Type::Category::ByteArray) { auto const& type = dynamic_cast(_type); - solAssert(type.getLocation() == ByteArrayType::Location::Storage, "Non-storage byte arrays not yet implemented."); - m_context << eth::Instruction::DUP1 << eth::Instruction::SLOAD; - // stack here: memory_offset storage_offset length_bytes - // jump to end if length is zero - m_context << eth::Instruction::DUP1 << eth::Instruction::ISZERO; - eth::AssemblyItem loopEnd = m_context.newTag(); - m_context.appendConditionalJumpTo(loopEnd); - // compute memory end offset - m_context << eth::Instruction::DUP3 << eth::Instruction::ADD << eth::Instruction::SWAP2; - // actual array data is stored at SHA3(storage_offset) - m_context << eth::Instruction::SWAP1; - CompilerUtils(m_context).computeHashStatic(); - m_context << eth::Instruction::SWAP1; + if (type.getLocation() == ByteArrayType::Location::CallData) + { + m_context << eth::Instruction::CALLDATASIZE << u256(0) << eth::Instruction::DUP3 + << eth::Instruction::CALLDATACOPY + << eth::Instruction::CALLDATASIZE << eth::Instruction::ADD; + } + else + { + solAssert(type.getLocation() == ByteArrayType::Location::Storage, "Memory byte arrays not yet implemented."); + m_context << eth::Instruction::DUP1 << eth::Instruction::SLOAD; + // stack here: memory_offset storage_offset length_bytes + // jump to end if length is zero + m_context << eth::Instruction::DUP1 << eth::Instruction::ISZERO; + eth::AssemblyItem loopEnd = m_context.newTag(); + m_context.appendConditionalJumpTo(loopEnd); + // compute memory end offset + m_context << eth::Instruction::DUP3 << eth::Instruction::ADD << eth::Instruction::SWAP2; + // actual array data is stored at SHA3(storage_offset) + m_context << eth::Instruction::SWAP1; + CompilerUtils(m_context).computeHashStatic(); + m_context << eth::Instruction::SWAP1; - // stack here: memory_end_offset storage_data_offset memory_offset - eth::AssemblyItem loopStart = m_context.newTag(); - m_context << loopStart - // load and store - << eth::Instruction::DUP2 << eth::Instruction::SLOAD - << eth::Instruction::DUP2 << eth::Instruction::MSTORE - // increment storage_data_offset by 1 - << eth::Instruction::SWAP1 << u256(1) << eth::Instruction::ADD - // increment memory offset by 32 - << eth::Instruction::SWAP1 << u256(32) << eth::Instruction::ADD - // check for loop condition - << eth::Instruction::DUP1 << eth::Instruction::DUP4 << eth::Instruction::GT; - m_context.appendConditionalJumpTo(loopStart); - m_context << loopEnd << eth::Instruction::POP << eth::Instruction::POP; + // stack here: memory_end_offset storage_data_offset memory_offset + eth::AssemblyItem loopStart = m_context.newTag(); + m_context << loopStart + // load and store + << eth::Instruction::DUP2 << eth::Instruction::SLOAD + << eth::Instruction::DUP2 << eth::Instruction::MSTORE + // increment storage_data_offset by 1 + << eth::Instruction::SWAP1 << u256(1) << eth::Instruction::ADD + // increment memory offset by 32 + << eth::Instruction::SWAP1 << u256(32) << eth::Instruction::ADD + // check for loop condition + << eth::Instruction::DUP1 << eth::Instruction::DUP4 << eth::Instruction::GT; + m_context.appendConditionalJumpTo(loopStart); + m_context << loopEnd << eth::Instruction::POP << eth::Instruction::POP; + } } else { diff --git a/test/SolidityEndToEndTest.cpp b/test/SolidityEndToEndTest.cpp index abfc97b73..259123db6 100644 --- a/test/SolidityEndToEndTest.cpp +++ b/test/SolidityEndToEndTest.cpp @@ -2273,6 +2273,21 @@ BOOST_AUTO_TEST_CASE(store_bytes) BOOST_CHECK(callContractFunction("save()", "abcdefg") == encodeArgs(24)); } +BOOST_AUTO_TEST_CASE(bytes_from_calldata_to_memory) +{ + char const* sourceCode = R"( + contract C { + function() returns (hash) { + return sha3("abc", msg.data); + } + } + )"; + compileAndRun(sourceCode); + bytes calldata = bytes(61, 0x22) + bytes(12, 0x12); + sendMessage(calldata, false); + BOOST_CHECK(m_output == encodeArgs(dev::sha3(bytes{'a', 'b', 'c'} + calldata))); +} + BOOST_AUTO_TEST_CASE(call_forward_bytes) { char const* sourceCode = R"( From 483ce3a2ea136f840ae6f1eac9dbf7fd0258a8e9 Mon Sep 17 00:00:00 2001 From: CJentzsch Date: Fri, 13 Feb 2015 09:00:51 +0100 Subject: [PATCH 65/71] performance test --- test/vm.cpp | 28 +++++++++++++++++++++++ test/vmPerformanceTestFiller.json | 38 +++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 test/vmPerformanceTestFiller.json diff --git a/test/vm.cpp b/test/vm.cpp index f15dc048a..af946777f 100644 --- a/test/vm.cpp +++ b/test/vm.cpp @@ -32,6 +32,8 @@ using namespace dev; using namespace dev::eth; using namespace dev::test; +using Millisecs = chrono::duration; + FakeExtVM::FakeExtVM(eth::BlockInfo const& _previousBlock, eth::BlockInfo const& _currentBlock, unsigned _depth): /// TODO: XXX: remove the default argument & fix. ExtVMFace(Address(), Address(), Address(), 0, 1, bytesConstRef(), bytes(), _previousBlock, _currentBlock, test::lastHashes(_currentBlock.number), _depth) {} @@ -513,13 +515,39 @@ BOOST_AUTO_TEST_CASE(vmSystemOperationsTest) dev::test::executeTests("vmSystemOperationsTest", "/VMTests", dev::test::doVMTests); } +BOOST_AUTO_TEST_CASE(vmPerformanceTest) +{ + for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i) + { + string arg = boost::unit_test::framework::master_test_suite().argv[i]; + if (arg == "--performance") + { + chrono::steady_clock::time_point start = chrono::steady_clock::now(); + + dev::test::executeTests("vmPerformanceTest", "/VMTests", dev::test::doVMTests); + + chrono::steady_clock::time_point end = chrono::steady_clock::now(); + Millisecs duration(chrono::duration_cast(end - start)); + cnote << "test duration: " << duration.count() << " milliseconds.\n"; + } + } +} + BOOST_AUTO_TEST_CASE(vmInputLimitsTest1) { for (int i = 1; i < boost::unit_test::framework::master_test_suite().argc; ++i) { string arg = boost::unit_test::framework::master_test_suite().argv[i]; if (arg == "--inputlimits") + { + chrono::steady_clock::time_point start = chrono::steady_clock::now(); + dev::test::executeTests("vmInputLimitsTest1", "/VMTests", dev::test::doVMTests); + + chrono::steady_clock::time_point end = chrono::steady_clock::now(); + Millisecs duration(chrono::duration_cast(end - start)); + cnote << "test duration: " << duration.count() << " milliseconds.\n"; + } } } diff --git a/test/vmPerformanceTestFiller.json b/test/vmPerformanceTestFiller.json new file mode 100644 index 000000000..7979bfbdb --- /dev/null +++ b/test/vmPerformanceTestFiller.json @@ -0,0 +1,38 @@ +{ + "ackermann33": { + "env" : { + "previousHash" : "5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6", + "currentNumber" : "0", + "currentGasLimit" : "100000000000", + "currentDifficulty" : "256", + "currentTimestamp" : "1", + "currentCoinbase" : "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba" + }, + "pre" : { + "0f572e5295c57f15886f9b263e2f6d2d6c7b5ec6" : { + "balance" : "1000000000000000000", + "nonce" : "0", + "//" : "contract PerformanceTester {", + "//" : " function ackermann(uint m, uint n) returns (uint) {", + "//" : " if (m == 0)", + "//" : " return n + 1;", + "//" : " if (n == 0)", + "//" : " return ackermann(m - 1, 1);", + "//" : " return ackermann(m - 1, ackermann(m, n - 1));", + "//" : " }", + "//" : "}", + "code" : "0x60e060020a6000350480632839e92814601457005b6020600435602435602a565b8060005260206000f35b6000826000146037576041565b8160010190506076565b81600014604c57605e565b6058600184036001602a565b90506076565b607360018403606f8560018603602a565b602a565b90505b9291505056", + "storage": {} + } + }, + "exec" : { + "address" : "0f572e5295c57f15886f9b263e2f6d2d6c7b5ec6", + "origin" : "cd1722f3947def4cf144679da39c4c32bdc35681", + "caller" : "cd1722f3947def4cf144679da39c4c32bdc35681", + "value" : "1000000000000000000", + "data" : "0x2839e92800000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000000000000000000000000000000000003", + "gasPrice" : "100000000000000", + "gas" : "100000000000" + } + } +} From 0520e0f4db7fa8b49b6c294135b689b1fdce4427 Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Fri, 13 Feb 2015 09:03:03 +0100 Subject: [PATCH 66/71] fixed issue with including wrong json/json.h file --- alethzero/CMakeLists.txt | 4 ++-- libsolidity/CMakeLists.txt | 4 ++-- lllc/CMakeLists.txt | 4 ++-- mix/CMakeLists.txt | 3 ++- solc/CMakeLists.txt | 4 ++-- test/CMakeLists.txt | 5 +++-- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/alethzero/CMakeLists.txt b/alethzero/CMakeLists.txt index ed2d8fa5e..c18a76c36 100644 --- a/alethzero/CMakeLists.txt +++ b/alethzero/CMakeLists.txt @@ -10,9 +10,9 @@ endif() set(CMAKE_INCLUDE_CURRENT_DIR ON) aux_source_directory(. SRC_LIST) -include_directories(${JSON_RPC_CPP_INCLUDE_DIRS}) -include_directories(${JSONCPP_INCLUDE_DIRS}) include_directories(..) +include_directories(BEFORE ${JSONCPP_INCLUDE_DIRS}) +include_directories(${JSON_RPC_CPP_INCLUDE_DIRS}) qt5_wrap_ui(ui_Main.h Main.ui) diff --git a/libsolidity/CMakeLists.txt b/libsolidity/CMakeLists.txt index 9c0b50775..cccc5e165 100644 --- a/libsolidity/CMakeLists.txt +++ b/libsolidity/CMakeLists.txt @@ -11,9 +11,9 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSTATICLIB") aux_source_directory(. SRC_LIST) -include_directories(${Boost_INCLUDE_DIRS}) -include_directories(${JSONCPP_INCLUDE_DIRS}) include_directories(..) +include_directories(BEFORE ${JSONCPP_INCLUDE_DIRS}) +include_directories(${Boost_INCLUDE_DIRS}) set(EXECUTABLE solidity) diff --git a/lllc/CMakeLists.txt b/lllc/CMakeLists.txt index 5aaca0ccc..2157fea74 100644 --- a/lllc/CMakeLists.txt +++ b/lllc/CMakeLists.txt @@ -3,9 +3,9 @@ set(CMAKE_AUTOMOC OFF) aux_source_directory(. SRC_LIST) -include_directories(${Boost_INCLUDE_DIRS}) -include_directories(${JSONCPP_INCLUDE_DIRS}) include_directories(..) +include_directories(BEFORE ${JSONCPP_INCLUDE_DIRS}) +include_directories(${Boost_INCLUDE_DIRS}) set(EXECUTABLE lllc) diff --git a/mix/CMakeLists.txt b/mix/CMakeLists.txt index 755f3df04..4390a4eb0 100644 --- a/mix/CMakeLists.txt +++ b/mix/CMakeLists.txt @@ -10,8 +10,9 @@ endif() set(CMAKE_INCLUDE_CURRENT_DIR ON) aux_source_directory(. SRC_LIST) -include_directories(${JSONCPP_INCLUDE_DIRS}) + include_directories(..) +include_directories(BEFORE ${JSONCPP_INCLUDE_DIRS}) find_package (Qt5WebEngine QUIET) qt5_add_resources(UI_RESOURCES res.qrc) diff --git a/solc/CMakeLists.txt b/solc/CMakeLists.txt index 2a7bd7b6d..747effad0 100644 --- a/solc/CMakeLists.txt +++ b/solc/CMakeLists.txt @@ -3,9 +3,9 @@ set(CMAKE_AUTOMOC OFF) aux_source_directory(. SRC_LIST) -include_directories(${Boost_INCLUDE_DIRS}) -include_directories(${JSONCPP_INCLUDE_DIRS}) include_directories(..) +include_directories(BEFORE ${JSONCPP_INCLUDE_DIRS}) +include_directories(${Boost_INCLUDE_DIRS}) set(EXECUTABLE solc) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ab8afcd70..6ba07f280 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -4,11 +4,12 @@ aux_source_directory(. SRC_LIST) list(REMOVE_ITEM SRC_LIST "./createRandomTest.cpp") list(REMOVE_ITEM SRC_LIST "./checkRandomTest.cpp") +include_directories(..) +include_directories(BEFORE ${JSONCPP_INCLUDE_DIRS}) include_directories(${Boost_INCLUDE_DIRS}) include_directories(${CRYPTOPP_INCLUDE_DIRS}) -include_directories(${JSONCPP_INCLUDE_DIRS}) include_directories(${JSON_RPC_CPP_INCLUDE_DIRS}) -include_directories(..) + file(GLOB HEADERS "*.h") add_executable(testeth ${SRC_LIST} ${HEADERS}) From d892a39cb21a9fcd871977d61ceecf8e23c0e052 Mon Sep 17 00:00:00 2001 From: CJentzsch Date: Fri, 13 Feb 2015 09:07:07 +0100 Subject: [PATCH 67/71] style fix --- test/vmPerformanceTestFiller.json | 64 +++++++++++++++---------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/test/vmPerformanceTestFiller.json b/test/vmPerformanceTestFiller.json index 7979bfbdb..7cfe659d5 100644 --- a/test/vmPerformanceTestFiller.json +++ b/test/vmPerformanceTestFiller.json @@ -1,38 +1,38 @@ { "ackermann33": { - "env" : { - "previousHash" : "5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6", - "currentNumber" : "0", - "currentGasLimit" : "100000000000", - "currentDifficulty" : "256", - "currentTimestamp" : "1", - "currentCoinbase" : "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba" - }, - "pre" : { - "0f572e5295c57f15886f9b263e2f6d2d6c7b5ec6" : { - "balance" : "1000000000000000000", - "nonce" : "0", - "//" : "contract PerformanceTester {", - "//" : " function ackermann(uint m, uint n) returns (uint) {", - "//" : " if (m == 0)", - "//" : " return n + 1;", - "//" : " if (n == 0)", - "//" : " return ackermann(m - 1, 1);", - "//" : " return ackermann(m - 1, ackermann(m, n - 1));", - "//" : " }", - "//" : "}", - "code" : "0x60e060020a6000350480632839e92814601457005b6020600435602435602a565b8060005260206000f35b6000826000146037576041565b8160010190506076565b81600014604c57605e565b6058600184036001602a565b90506076565b607360018403606f8560018603602a565b602a565b90505b9291505056", - "storage": {} - } - }, + "env" : { + "previousHash" : "5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6", + "currentNumber" : "0", + "currentGasLimit" : "100000000000", + "currentDifficulty" : "256", + "currentTimestamp" : "1", + "currentCoinbase" : "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba" + }, + "pre" : { + "0f572e5295c57f15886f9b263e2f6d2d6c7b5ec6" : { + "balance" : "1000000000000000000", + "nonce" : "0", + "//" : "contract PerformanceTester {", + "//" : " function ackermann(uint m, uint n) returns (uint) {", + "//" : " if (m == 0)", + "//" : " return n + 1;", + "//" : " if (n == 0)", + "//" : " return ackermann(m - 1, 1);", + "//" : " return ackermann(m - 1, ackermann(m, n - 1));", + "//" : " }", + "//" : "}", + "code" : "0x60e060020a6000350480632839e92814601457005b6020600435602435602a565b8060005260206000f35b6000826000146037576041565b8160010190506076565b81600014604c57605e565b6058600184036001602a565b90506076565b607360018403606f8560018603602a565b602a565b90505b9291505056", + "storage": {} + } + }, "exec" : { - "address" : "0f572e5295c57f15886f9b263e2f6d2d6c7b5ec6", - "origin" : "cd1722f3947def4cf144679da39c4c32bdc35681", - "caller" : "cd1722f3947def4cf144679da39c4c32bdc35681", - "value" : "1000000000000000000", + "address" : "0f572e5295c57f15886f9b263e2f6d2d6c7b5ec6", + "origin" : "cd1722f3947def4cf144679da39c4c32bdc35681", + "caller" : "cd1722f3947def4cf144679da39c4c32bdc35681", + "value" : "1000000000000000000", "data" : "0x2839e92800000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000000000000000000000000000000000003", - "gasPrice" : "100000000000000", + "gasPrice" : "100000000000000", "gas" : "100000000000" - } - } + } + } } From 0696ee87da8d34d09fd1286cc7d08e535b514ce3 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 13 Feb 2015 10:25:56 +0100 Subject: [PATCH 68/71] Windows compile fix. --- libp2p/Host.cpp | 2 +- libp2p/Session.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index f09468e0d..f117df37a 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -54,7 +54,7 @@ Host::Host(std::string const& _clientVersion, NetworkPreferences const& _n, byte m_ioService(2), m_tcp4Acceptor(m_ioService), m_alias(networkAlias(_restoreNetwork)), - m_lastPing(chrono::time_point::min()) + m_lastPing(chrono::steady_clock::time_point::min()) { for (auto address: m_ifAddresses) if (address.is_v4()) diff --git a/libp2p/Session.cpp b/libp2p/Session.cpp index cb7225f0f..6ff765cf6 100644 --- a/libp2p/Session.cpp +++ b/libp2p/Session.cpp @@ -40,10 +40,10 @@ Session::Session(Host* _s, bi::tcp::socket _socket, std::shared_ptr const& m_server(_s), m_socket(std::move(_socket)), m_peer(_n), - m_info({NodeId(), "?", m_socket.remote_endpoint().address().to_string(), 0, std::chrono::steady_clock::duration(0), CapDescSet(), 0, map()}), - m_ping(chrono::time_point::max()) + m_info({NodeId(), "?", m_socket.remote_endpoint().address().to_string(), 0, chrono::steady_clock::duration(0), CapDescSet(), 0, map()}), + m_ping(chrono::steady_clock::time_point::max()) { - m_lastReceived = m_connect = std::chrono::steady_clock::now(); + m_lastReceived = m_connect = chrono::steady_clock::now(); } Session::~Session() From f5022ee58f2262624b991487006e3fd56991ce07 Mon Sep 17 00:00:00 2001 From: CJentzsch Date: Fri, 13 Feb 2015 10:27:38 +0100 Subject: [PATCH 69/71] use auto for steady clock and chrono::milliseconds --- test/vm.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/vm.cpp b/test/vm.cpp index af946777f..faab48129 100644 --- a/test/vm.cpp +++ b/test/vm.cpp @@ -32,8 +32,6 @@ using namespace dev; using namespace dev::eth; using namespace dev::test; -using Millisecs = chrono::duration; - FakeExtVM::FakeExtVM(eth::BlockInfo const& _previousBlock, eth::BlockInfo const& _currentBlock, unsigned _depth): /// TODO: XXX: remove the default argument & fix. ExtVMFace(Address(), Address(), Address(), 0, 1, bytesConstRef(), bytes(), _previousBlock, _currentBlock, test::lastHashes(_currentBlock.number), _depth) {} @@ -522,12 +520,12 @@ BOOST_AUTO_TEST_CASE(vmPerformanceTest) string arg = boost::unit_test::framework::master_test_suite().argv[i]; if (arg == "--performance") { - chrono::steady_clock::time_point start = chrono::steady_clock::now(); + auto start = chrono::steady_clock::now(); dev::test::executeTests("vmPerformanceTest", "/VMTests", dev::test::doVMTests); - chrono::steady_clock::time_point end = chrono::steady_clock::now(); - Millisecs duration(chrono::duration_cast(end - start)); + auto end = chrono::steady_clock::now(); + chrono::milliseconds duration(chrono::duration_cast(end - start)); cnote << "test duration: " << duration.count() << " milliseconds.\n"; } } @@ -540,12 +538,12 @@ BOOST_AUTO_TEST_CASE(vmInputLimitsTest1) string arg = boost::unit_test::framework::master_test_suite().argv[i]; if (arg == "--inputlimits") { - chrono::steady_clock::time_point start = chrono::steady_clock::now(); + auto start = chrono::steady_clock::now(); dev::test::executeTests("vmInputLimitsTest1", "/VMTests", dev::test::doVMTests); - chrono::steady_clock::time_point end = chrono::steady_clock::now(); - Millisecs duration(chrono::duration_cast(end - start)); + auto end = chrono::steady_clock::now(); + chrono::milliseconds duration(chrono::duration_cast(end - start)); cnote << "test duration: " << duration.count() << " milliseconds.\n"; } } From 5d15c00aba00bacac7748b7dbf9b6c814018d131 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 13 Feb 2015 10:43:46 +0100 Subject: [PATCH 70/71] Windows warning fix. --- libp2p/NodeTable.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libp2p/NodeTable.h b/libp2p/NodeTable.h index 04e8d009c..0217c700e 100644 --- a/libp2p/NodeTable.h +++ b/libp2p/NodeTable.h @@ -163,7 +163,7 @@ public: std::list snapshot() const; /// Returns true if node id is in node table. - bool haveNode(NodeId const& _id) { Guard l(x_nodes); return m_nodes.count(_id); } + bool haveNode(NodeId const& _id) { Guard l(x_nodes); return m_nodes.count(_id) > 0; } /// Returns the Node to the corresponding node id or the empty Node if that id is not found. Node node(NodeId const& _id); @@ -418,4 +418,4 @@ struct NodeTableEgress: public LogChannel { static const char* name() { return " struct NodeTableIngress: public LogChannel { static const char* name() { return "< Date: Fri, 13 Feb 2015 11:24:14 +0100 Subject: [PATCH 71/71] auto reccommendations. --- CodingStandards.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CodingStandards.txt b/CodingStandards.txt index f883c9147..8acea78ae 100644 --- a/CodingStandards.txt +++ b/CodingStandards.txt @@ -96,12 +96,18 @@ e. Always pass non-trivial parameters with a const& suffix. f. If a function returns multiple values, use std::tuple (std::pair acceptable). Prefer not using */& arguments, except where efficiency requires. g. Never use a macro where adequate non-preprocessor C++ can be written. h. Prefer "using NewType = OldType" to "typedef OldType NewType". +i. Make use of auto whenever type is clear or unimportant: +- Always avoid doubly-stating the type. +- Use to avoid vast and unimportant type declarations + (WRONG) const double d = 0; int i, j; char *s; float meanAndSigma(std::vector _v, float* _sigma); +Derived* x(dynamic_cast(base)); +for (map::iterator i = l.begin(); i != l.end(); ++l) {} (CORRECT) double const d = 0; @@ -109,7 +115,8 @@ int i; int j; char* s; std::tuple meanAndSigma(std::vector const& _v); - +auto x = dynamic_cast(base); +for (auto i = x.begin(); i != x.end(); ++i) {} 7. Structs & classes