From 730fd563860b7fad5c27a9d7fc5840abffd0c364 Mon Sep 17 00:00:00 2001 From: shsmith Date: Sun, 12 Mar 2017 09:35:23 -0700 Subject: [PATCH 001/117] contribute notes for Raspberry Pi and Ubuntu 16.04 deployment --- contrib/README.md | 9 +++++ .../raspberrypi3deploy/install_electrumx.sh | 24 ++++++++++++ contrib/raspberrypi3deploy/run_electrumx.sh | 38 +++++++++++++++++++ contrib/ubuntu1604deploy/python-3.6.sh | 11 ++++++ 4 files changed, 82 insertions(+) create mode 100644 contrib/README.md create mode 100644 contrib/raspberrypi3deploy/install_electrumx.sh create mode 100644 contrib/raspberrypi3deploy/run_electrumx.sh create mode 100644 contrib/ubuntu1604deploy/python-3.6.sh diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000..d60046d --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,9 @@ +Installation Guides +------------------- + +### [Ubuntu 16.04](/contrib/ubuntu1604deploy) ### +Scripts and notes for deployment under Ubuntu 16.04. + +### [Raspberry Pi 3](/contrib/raspberrypi3deploy) ### +Scripts and notes for deployment on Raspberry Pi 3. + diff --git a/contrib/raspberrypi3deploy/install_electrumx.sh b/contrib/raspberrypi3deploy/install_electrumx.sh new file mode 100644 index 0000000..11b7ab9 --- /dev/null +++ b/contrib/raspberrypi3deploy/install_electrumx.sh @@ -0,0 +1,24 @@ +################### +# install electrumx +################### + +# upgrade raspbian to 'stretch' distribution for python 3.5 support +sudo echo 'deb http://mirrordirector.raspbian.org/raspbian/ testing main contrib non-free rpi' > /etc/apt/sources.list.d/stretch.list +sudo apt-get update +sudo apt-get dist-upgrade +sudo apt-get autoremove + +# install electrumx dependencies +sudo apt-get install python3-pip +sudo apt-get install build-essential libc6-dev +sudo apt-get install libncurses5-dev libncursesw5-dev libreadline6-dev +sudo apt-get install libleveldb-dev +sudo apt-get install git +sudo pip3 install plyvel +sudo pip3 install irc + +# install electrumx +git clone https://github.com/kyuupichan/electrumx.git +cd electrumx +sudo python3 setup.py install + diff --git a/contrib/raspberrypi3deploy/run_electrumx.sh b/contrib/raspberrypi3deploy/run_electrumx.sh new file mode 100644 index 0000000..f97b9f8 --- /dev/null +++ b/contrib/raspberrypi3deploy/run_electrumx.sh @@ -0,0 +1,38 @@ +############### +# run_electrumx +############### + +# configure electrumx +export COIN=Bitcoin +export DAEMON_URL=http://rpcuser:rpcpassword@127.0.0.1 +export NETWORK=mainnet +export CACHE_MB=400 +export DB_DIRECTORY=/home/username/.electrumx/db +export SSL_CERTFILE=/home/username/.electrumx/certfile.crt +export SSL_KEYFILE=/home/username/.electrumx/keyfile.key +export BANNER_FILE=/home/username/.electrumx/banner +export DONATION_ADDRESS=your-donation-address + +# connectivity +export HOST= +export TCP_PORT=50001 +export SSL_PORT=50002 + +# visibility +export IRC= +export IRC_NICK=hostname +export REPORT_HOST=hostname.com +export RPC_PORT=8000 + +# run electrumx +ulimit -n 10000 +/usr/local/bin/electrumx_server.py 2>> /home/username/.electrumx/electrumx.log >> /home/username/.electrumx/electrumx.log & + +###################### +# auto-start electrumx +###################### + +# add this line to crontab -e +# @reboot /path/to/run_electrumx.sh + + diff --git a/contrib/ubuntu1604deploy/python-3.6.sh b/contrib/ubuntu1604deploy/python-3.6.sh new file mode 100644 index 0000000..d83839d --- /dev/null +++ b/contrib/ubuntu1604deploy/python-3.6.sh @@ -0,0 +1,11 @@ +Installation of Python 3.6 +-------------------------- + +sudo add-apt-repository ppa:jonathonf/python-3.6 +sudo apt-get update && sudo apt-get install python3.6 python3.6-dev + +cd /home/username +git clone https://github.com/kyuupichan/electrumx.git +cd electrumx +sudo python3.6 setup.py install + From 5ef25976c40b2d8f181e514b7b571f5cefad941a Mon Sep 17 00:00:00 2001 From: shsmith Date: Sun, 12 Mar 2017 18:30:44 -0700 Subject: [PATCH 002/117] merge samples into contrib --- contrib/README.md | 9 ------ {samples => contrib}/daemontools/env/COIN | 0 .../daemontools/env/DAEMON_URL | 0 .../daemontools/env/DB_DIRECTORY | 0 .../daemontools/env/ELECTRUMX | 0 {samples => contrib}/daemontools/env/NETWORK | 0 {samples => contrib}/daemontools/env/USERNAME | 0 {samples => contrib}/daemontools/log/run | 0 {samples => contrib}/daemontools/run | 0 .../python-3.6.sh | 0 .../install_electrumx.sh | 0 .../run_electrumx.sh | 0 {samples => contrib}/systemd/electrumx.conf | 0 .../systemd/electrumx.service | 0 docs/HOWTO.rst | 30 ++++++++++++++++--- 15 files changed, 26 insertions(+), 13 deletions(-) delete mode 100644 contrib/README.md rename {samples => contrib}/daemontools/env/COIN (100%) rename {samples => contrib}/daemontools/env/DAEMON_URL (100%) rename {samples => contrib}/daemontools/env/DB_DIRECTORY (100%) rename {samples => contrib}/daemontools/env/ELECTRUMX (100%) rename {samples => contrib}/daemontools/env/NETWORK (100%) rename {samples => contrib}/daemontools/env/USERNAME (100%) rename {samples => contrib}/daemontools/log/run (100%) mode change 100755 => 100644 rename {samples => contrib}/daemontools/run (100%) mode change 100755 => 100644 rename contrib/{ubuntu1604deploy => python3.6}/python-3.6.sh (100%) rename contrib/{raspberrypi3deploy => raspberrypi3}/install_electrumx.sh (100%) rename contrib/{raspberrypi3deploy => raspberrypi3}/run_electrumx.sh (100%) rename {samples => contrib}/systemd/electrumx.conf (100%) rename {samples => contrib}/systemd/electrumx.service (100%) diff --git a/contrib/README.md b/contrib/README.md deleted file mode 100644 index d60046d..0000000 --- a/contrib/README.md +++ /dev/null @@ -1,9 +0,0 @@ -Installation Guides -------------------- - -### [Ubuntu 16.04](/contrib/ubuntu1604deploy) ### -Scripts and notes for deployment under Ubuntu 16.04. - -### [Raspberry Pi 3](/contrib/raspberrypi3deploy) ### -Scripts and notes for deployment on Raspberry Pi 3. - diff --git a/samples/daemontools/env/COIN b/contrib/daemontools/env/COIN similarity index 100% rename from samples/daemontools/env/COIN rename to contrib/daemontools/env/COIN diff --git a/samples/daemontools/env/DAEMON_URL b/contrib/daemontools/env/DAEMON_URL similarity index 100% rename from samples/daemontools/env/DAEMON_URL rename to contrib/daemontools/env/DAEMON_URL diff --git a/samples/daemontools/env/DB_DIRECTORY b/contrib/daemontools/env/DB_DIRECTORY similarity index 100% rename from samples/daemontools/env/DB_DIRECTORY rename to contrib/daemontools/env/DB_DIRECTORY diff --git a/samples/daemontools/env/ELECTRUMX b/contrib/daemontools/env/ELECTRUMX similarity index 100% rename from samples/daemontools/env/ELECTRUMX rename to contrib/daemontools/env/ELECTRUMX diff --git a/samples/daemontools/env/NETWORK b/contrib/daemontools/env/NETWORK similarity index 100% rename from samples/daemontools/env/NETWORK rename to contrib/daemontools/env/NETWORK diff --git a/samples/daemontools/env/USERNAME b/contrib/daemontools/env/USERNAME similarity index 100% rename from samples/daemontools/env/USERNAME rename to contrib/daemontools/env/USERNAME diff --git a/samples/daemontools/log/run b/contrib/daemontools/log/run old mode 100755 new mode 100644 similarity index 100% rename from samples/daemontools/log/run rename to contrib/daemontools/log/run diff --git a/samples/daemontools/run b/contrib/daemontools/run old mode 100755 new mode 100644 similarity index 100% rename from samples/daemontools/run rename to contrib/daemontools/run diff --git a/contrib/ubuntu1604deploy/python-3.6.sh b/contrib/python3.6/python-3.6.sh similarity index 100% rename from contrib/ubuntu1604deploy/python-3.6.sh rename to contrib/python3.6/python-3.6.sh diff --git a/contrib/raspberrypi3deploy/install_electrumx.sh b/contrib/raspberrypi3/install_electrumx.sh similarity index 100% rename from contrib/raspberrypi3deploy/install_electrumx.sh rename to contrib/raspberrypi3/install_electrumx.sh diff --git a/contrib/raspberrypi3deploy/run_electrumx.sh b/contrib/raspberrypi3/run_electrumx.sh similarity index 100% rename from contrib/raspberrypi3deploy/run_electrumx.sh rename to contrib/raspberrypi3/run_electrumx.sh diff --git a/samples/systemd/electrumx.conf b/contrib/systemd/electrumx.conf similarity index 100% rename from samples/systemd/electrumx.conf rename to contrib/systemd/electrumx.conf diff --git a/samples/systemd/electrumx.service b/contrib/systemd/electrumx.service similarity index 100% rename from samples/systemd/electrumx.service rename to contrib/systemd/electrumx.service diff --git a/docs/HOWTO.rst b/docs/HOWTO.rst index e409859..dde2f81 100644 --- a/docs/HOWTO.rst +++ b/docs/HOWTO.rst @@ -108,7 +108,7 @@ to at least 2,500. Note that setting the limit in your shell does *NOT* affect ElectrumX unless you are invoking ElectrumX directly from your shell. If you are using `systemd`, you need to set it in the `.service` file (see -`samples/systemd/electrumx.service`_). +`contrib/systemd/electrumx.service`_). Using daemontools @@ -136,7 +136,7 @@ you might do:: Then copy the all sample scripts from the ElectrumX source tree there:: - cp -R /path/to/repo/electrumx/samples/daemontools ~/scripts/electrumx + cp -R /path/to/repo/electrumx/contrib/daemontools ~/scripts/electrumx This copies 3 things: the top level server run script, a log/ directory with the logger run script, an env/ directory. @@ -172,7 +172,7 @@ Using systemd This repository contains a sample systemd unit file that you can use to setup ElectrumX with systemd. Simply copy it to :code:`/etc/systemd/system`:: - cp samples/systemd/electrumx.service /etc/systemd/system/ + cp contrib/systemd/electrumx.service /etc/systemd/system/ The sample unit file assumes that the repository is located at :code:`/home/electrumx/electrumx`. If that differs on your system, you need to @@ -199,6 +199,24 @@ minutes to flush cached data to disk during initial sync. You should set TimeoutStopSec to *at least* 10 mins in your `.service` file. +Installing Python 3.6 under Ubuntu +---------------------------------- + +Many Ubuntu distributions have an incompatible Python version baked in. +Because of this, it is easier to install Python 3.6 rather than attempting +to update Python 3.5.2 to 3.5.3. See `contrib/python3.6/python-3.6.sh`_. + + +Installing on Raspberry Pi 3 +---------------------------- + +To install on the Raspberry Pi 3 you will need to update to the "stretch" distribution. +See the full procedure in `contrib/raspberrypi3/install_electrumx.sh`_. + +See also `contrib/raspberrypi3/run_electrumx.sh`_ for an easy way to configure and +launch electrumx. + + Sync Progress ============= @@ -377,10 +395,14 @@ copy of your certificate and key in case you need to restore them. .. _`ENVIRONMENT.rst`: https://github.com/kyuupichan/electrumx/blob/master/docs/ENVIRONMENT.rst -.. _`samples/systemd/electrumx.service`: https://github.com/kyuupichan/electrumx/blob/master/samples/systemd/electrumx.service +.. _`contrib/systemd/electrumx.service`: https://github.com/kyuupichan/electrumx/blob/master/contrib/systemd/electrumx.service .. _`daemontools`: http://cr.yp.to/daemontools.html .. _`runit`: http://smarden.org/runit/index.html .. _`aiohttp`: https://pypi.python.org/pypi/aiohttp .. _`pylru`: https://pypi.python.org/pypi/pylru .. _`IRC`: https://pypi.python.org/pypi/irc .. _`x11_hash`: https://pypi.python.org/pypi/x11_hash +.. _`contrib/python3.6/python-3.6.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/contrib/python3.6/python-3.6.sh +.. _`contrib/raspberrypi3/install_electrumx.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/contrib/raspberrypi3/install_electrumx.sh +.. _`contrib/raspberrypi3/run_electrumx.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/contrib/raspberrypi3/run_electrumx.sh + From 67c135a1946e28d1ad405030f9d7cbc81f82353a Mon Sep 17 00:00:00 2001 From: Samuel Smith Date: Tue, 14 Mar 2017 14:29:55 -0700 Subject: [PATCH 003/117] update raspberrypi3 install procedure (#148) install stable version of libreadline6 based on comment from MaxTG --- contrib/raspberrypi3/install_electrumx.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/raspberrypi3/install_electrumx.sh b/contrib/raspberrypi3/install_electrumx.sh index 11b7ab9..db64146 100644 --- a/contrib/raspberrypi3/install_electrumx.sh +++ b/contrib/raspberrypi3/install_electrumx.sh @@ -11,7 +11,8 @@ sudo apt-get autoremove # install electrumx dependencies sudo apt-get install python3-pip sudo apt-get install build-essential libc6-dev -sudo apt-get install libncurses5-dev libncursesw5-dev libreadline6-dev +sudo apt-get install libncurses5-dev libncursesw5-dev +sudo apt install libreadline6-dev/stable libreadline6/stable sudo apt-get install libleveldb-dev sudo apt-get install git sudo pip3 install plyvel From 11a3c77fdb64ecd979457de971a38aa63ba06494 Mon Sep 17 00:00:00 2001 From: pooler Date: Fri, 17 Mar 2017 12:16:44 +0100 Subject: [PATCH 004/117] Fix Litecoin parameters --- lib/coins.py | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index 2aae70a..99f5044 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -385,8 +385,8 @@ class Litecoin(Coin): NAME = "Litecoin" SHORTNAME = "LTC" NET = "mainnet" - XPUB_VERBYTES = bytes.fromhex("019da462") - XPRV_VERBYTES = bytes.fromhex("019d9cfe") + XPUB_VERBYTES = bytes.fromhex("0488b21e") + XPRV_VERBYTES = bytes.fromhex("0488ade4") P2PKH_VERBYTE = bytes.fromhex("30") P2SH_VERBYTE = bytes.fromhex("05") WIF_BYTE = bytes.fromhex("b0") @@ -399,18 +399,46 @@ class Litecoin(Coin): IRC_CHANNEL = "#electrum-ltc" RPC_PORT = 9332 REORG_LIMIT = 800 + PEERS = [ + 'elec.luggs.co s444', + 'electrum-ltc.bysh.me s t', + 'electrum-ltc.ddns.net s t', + 'electrum.cryptomachine.com p1000 s t', + 'electrum.ltc.xurious.com s t', + 'eywr5eubdbbe2laq.onion s50008 t50007', + 'us11.einfachmalnettsein.de s50008 t50007', + ] class LitecoinTestnet(Litecoin): SHORTNAME = "XLT" NET = "testnet" - XPUB_VERBYTES = bytes.fromhex("0436f6e1") - XPRV_VERBYTES = bytes.fromhex("0436ef7d") + IRC_PREFIX = None + XPUB_VERBYTES = bytes.fromhex("043587cf") + XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("6f") P2SH_VERBYTE = bytes.fromhex("c4") WIF_BYTE = bytes.fromhex("ef") - GENESIS_HASH = ('f5ae71e26c74beacc88382716aced69c' - 'ddf3dffff24f384e1808905e0188f68f') + GENESIS_HASH = ('4966625a4b2851d9fdee139e56211a0d' + '88575f59ed816ff5e6a63deb4e3e29a0') + TX_COUNT = 21772 + TX_COUNT_HEIGHT = 20800 + TX_PER_BLOCK = 2 + RPC_PORT = 19332 + REORG_LIMIT = 4000 + PEER_DEFAULT_PORTS = {'t': '51001', 's': '51002'} + PEERS = [ + 'electrum-ltc.bysh.me s t', + 'electrum.ltc.xurious.com s t', + ] + + +class LitecoinTestnetSegWit(LitecoinTestnet): + NET = "testnet-segwit" + + @classmethod + def deserializer(cls): + return DeserializerSegWit # Source: namecoin.org From fe544eae7f313a76ee74d2054c37dad7fc3c283f Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Sat, 18 Mar 2017 03:27:53 +0200 Subject: [PATCH 005/117] Set the correct XPUB, XPRV bytes for dogecoin --- lib/coins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index 2aae70a..2ac7c7b 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -470,8 +470,8 @@ class DogecoinTestnet(Dogecoin): NAME = "Dogecoin" SHORTNAME = "XDT" NET = "testnet" - XPUB_VERBYTES = bytes.fromhex("0432a9a8") - XPRV_VERBYTES = bytes.fromhex("0432a243") + XPUB_VERBYTES = bytes.fromhex("043587cf") + XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("71") P2SH_VERBYTE = bytes.fromhex("c4") WIF_BYTE = bytes.fromhex("f1") From af675365989f296ab0cad4dca21d41633b42e260 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 21 Mar 2017 20:04:51 +0900 Subject: [PATCH 006/117] Set reorg limit to 8k on testnet --- lib/coins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coins.py b/lib/coins.py index b6c7575..3971407 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -332,7 +332,7 @@ class BitcoinTestnet(Bitcoin): WIF_BYTE = bytes.fromhex("ef") GENESIS_HASH = ('000000000933ea01ad0ee984209779ba' 'aec3ced90fa3f408719526f8d77f4943') - REORG_LIMIT = 4000 + REORG_LIMIT = 8000 TX_COUNT = 12242438 TX_COUNT_HEIGHT = 1035428 TX_PER_BLOCK = 21 From 5f56689e9cfd2697abea1903a4cb50a458f1a794 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 21 Mar 2017 20:23:57 +0900 Subject: [PATCH 007/117] Don't permit common invalid REPORT_HOST values --- server/env.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/server/env.py b/server/env.py index 6b1831d..dcaaad2 100644 --- a/server/env.py +++ b/server/env.py @@ -9,6 +9,7 @@ from collections import namedtuple +from ipaddress import ip_address from os import environ from lib.coins import Coin @@ -68,8 +69,10 @@ class Env(LoggedClass): self.irc_nick = self.default('IRC_NICK', None) # Identities + report_host = self.default('REPORT_HOST', self.host) + self.check_report_host(report_host) main_identity = NetIdentity( - self.default('REPORT_HOST', self.host), + report_host, self.integer('REPORT_TCP_PORT', self.tcp_port) or None, self.integer('REPORT_SSL_PORT', self.ssl_port) or None, '' @@ -114,6 +117,16 @@ class Env(LoggedClass): raise self.Error('cannot convert envvar {} value {} to an integer' .format(envvar, value)) + def check_report_host(self, host): + try: + ip = ip_address(host) + except ValueError: + bad = not bool(host) + else: + bad = ip.is_multicast or ip.is_unspecified + if bad: + raise self.Error('{} is not a valid REPORT_HOST'.format(host)) + def obsolete(self, envvars): bad = [envvar for envvar in envvars if environ.get(envvar)] if bad: From 294212d4211fb375744f7f92f62b7aed16345305 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 21 Mar 2017 22:16:19 +0900 Subject: [PATCH 008/117] Fix discovery of base of reorgs --- server/block_processor.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/server/block_processor.py b/server/block_processor.py index 3093263..933cbae 100644 --- a/server/block_processor.py +++ b/server/block_processor.py @@ -291,11 +291,13 @@ class BlockProcessor(server.db.DB): The hashes are returned in order of increasing height.''' - def match_pos(hashes1, hashes2): + def diff_pos(hashes1, hashes2): + '''Returns the index of the first difference in the hash lists. + If both lists match returns their length.''' for n, (hash1, hash2) in enumerate(zip(hashes1, hashes2)): - if hash1 == hash2: + if hash1 != hash2: return n - return -1 + return len(hashes) if count is None: # A real reorg @@ -305,9 +307,9 @@ class BlockProcessor(server.db.DB): hashes = self.fs_block_hashes(start, count) hex_hashes = [hash_to_str(hash) for hash in hashes] d_hex_hashes = await self.daemon.block_hex_hashes(start, count) - n = match_pos(hex_hashes, d_hex_hashes) - if n >= 0: - start += n + 1 + n = diff_pos(hex_hashes, d_hex_hashes) + if n > 0: + start += n break count = min(count * 2, start) start -= count From e0fd64d29af1d99bb7e09dba7b8b05391861d5ad Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 22 Mar 2017 07:57:15 +0900 Subject: [PATCH 009/117] Rate-limit add_peer calls randomly Prepare 1.0.1 --- README.rst | 10 ++++++++++ server/controller.py | 11 +++++++++++ server/session.py | 6 ++---- server/version.py | 2 +- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 10b97d1..3a71dd0 100644 --- a/README.rst +++ b/README.rst @@ -127,6 +127,16 @@ Roadmap ChangeLog ========= +Version 1.0.1 +------------- + +* Rate-limit add_peer calls in a random way +* Fix discovery of base height in reorgs +* Don't permit common but invalid REPORT_HOST values +* Set reorg limit to 8000 blocks on testnet +* dogecoin / litecoin parameter fixes (erasmospunk, pooler) +* minor doc tweaks + Version 1.0 ----------- diff --git a/server/controller.py b/server/controller.py index c0cbf6b..a8d4d43 100644 --- a/server/controller.py +++ b/server/controller.py @@ -9,6 +9,7 @@ import asyncio import json import os import ssl +import random import time import traceback import warnings @@ -47,6 +48,7 @@ class Controller(util.LoggedClass): self.executor = ThreadPoolExecutor() self.loop.set_default_executor(self.executor) self.start_time = time.time() + self.next_add_peer_time = self.start_time self.coin = env.coin self.daemon = Daemon(env.coin.daemon_urls(env.daemon_url)) self.bp = BlockProcessor(env, self, self.daemon) @@ -133,6 +135,15 @@ class Controller(util.LoggedClass): def is_deprioritized(self, session): return self.session_priority(session) > self.BANDS + def permit_add_peer(self): + '''To prevent lots of add_peer requests filling up the peer + table, accept only one per random time interval.''' + now = time.time() + if now < self.next_add_peer_time: + return False + self.next_add_peer_time = now + random.randrange(0, 1800) + return True + async def run_in_executor(self, func, *args): '''Wait whilst running func in the executor.''' return await self.loop.run_in_executor(None, func, *args) diff --git a/server/session.py b/server/session.py index ee0466f..6ae9d66 100644 --- a/server/session.py +++ b/server/session.py @@ -45,7 +45,6 @@ class SessionBase(JSONSession): self.bw_time = self.start_time self.bw_interval = 3600 self.bw_used = 0 - self.peer_added = False def close_connection(self): '''Call this to close the connection.''' @@ -196,13 +195,12 @@ class ElectrumX(SessionBase): def add_peer(self, features): '''Add a peer.''' - if self.peer_added: + if not self.controller.permit_add_peer(): return False peer_mgr = self.controller.peer_mgr peer_info = self.peer_info() source = peer_info[0] if peer_info else 'unknown' - self.peer_added = peer_mgr.on_add_peer(features, source) - return self.peer_added + return peer_mgr.on_add_peer(features, source) def peers_subscribe(self): '''Return the server peers as a list of (ip, host, details) tuples.''' diff --git a/server/version.py b/server/version.py index 334e721..e9146f2 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0' +VERSION = 'ElectrumX 1.0.1' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From 30c91c69e1fe230ae6ea0b28c79126ad4e427840 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 22 Mar 2017 23:45:26 +0900 Subject: [PATCH 010/117] Update protocol docs --- docs/PROTOCOL.rst | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/docs/PROTOCOL.rst b/docs/PROTOCOL.rst index 5f3151e..d260c76 100644 --- a/docs/PROTOCOL.rst +++ b/docs/PROTOCOL.rst @@ -5,6 +5,8 @@ Electrum Protocol Until now there was no written specification of the Electrum protocol that I am aware of; this document is an attempt to fill that gap. It is intended to be a reference for client and server authors alike. +[Since writing this I learnt there has been a skeleton protocol +description on docs.github.io]. I have attempted to ensure what is written is correct for the three known server implementations: electrum-server, jelectrum and @@ -61,8 +63,8 @@ Protocol negotiation is not implemented in any client or server at present to the best of my knowledge, so care is needed to ensure current clients and servers continue to operate as expected. -Protocol versions are denoted by [major_number, minor_number] pairs, -for example protocol version 1.15 is [1, 15] as a pair. +Protocol versions are denoted by "m.n" strings, where *m* is the major +version number and *n* the minor version number. For example: "1.5". A party to a connection will speak all protocol versions in a range, say from `protocol_min` to `protocol_max`. This min and max may be @@ -189,7 +191,7 @@ Return the unconfirmed transactions of a bitcoin address. transaction is a dictionary with keys *height* , *tx_hash* and *fee*. *tx_hash* the transaction hash in hexadecimal, *height* is `0` if all inputs are confirmed, and `-1` otherwise, and *fee* is - the transaction fee in coin units. + the transaction fee in minimum coin units as an integer. **Response Examples** @@ -298,7 +300,7 @@ blockchain.block.get_header Return the *deserialized header* [2]_ of the block at the given height. - blockchain.block.get_chunk(**height**) + blockchain.block.get_header(**height**) **height** @@ -324,10 +326,10 @@ blockchain.block.get_chunk ========================== Return a concatenated chunk of block headers. A chunk consists of a -fixed number of block headers over at the end of which difficulty is -retargeted. +fixed number of block headers over which difficulty is constant, and +at the end of which difficulty is retargeted. -So in the case of Bitcoin a chunk is 2,016 headers, each of 80 bytes, +In the case of Bitcoin a chunk is 2,016 headers, each of 80 bytes, and chunk 5 is the block headers from height 10,080 to 12,095 inclusive. When encoded as hexadecimal, the response string is twice as long, so for Bitcoin it is 322,560 bytes long, making this a @@ -558,7 +560,7 @@ deprecated. **Response** A Base58 address string, or *null*. If the transaction doesn't - exist, the index is out of range, or the output is not paid to and + exist, the index is out of range, or the output is not paid to an address, *null* must be returned. If the output is spent *null* may be returned. @@ -600,7 +602,7 @@ subscription and the server must send no notifications. The first element is the IP address, the second is the host name (which might also be an IP address), and the third is a list of server features. Each feature and starts with a letter. 'v' - indicates the server minimum protocol version, 'p' its pruning limit + indicates the server maximum protocol version, 'p' its pruning limit and is omitted if it does not prune, 't' is the TCP port number, and 's' is the SSL port number. If a port is not given for 's' or 't' the default port for the coin network is implied. If 's' or 't' is @@ -615,7 +617,7 @@ following changes: * improved semantics of `server.version` to aid protocol negotiation * deprecated methods `blockchain.address.get_proof`, - 'blockchain.utxo.get_address' and `blockchain.numblocks.subscribe` + `blockchain.utxo.get_address` and `blockchain.numblocks.subscribe` have been removed. * method `blockchain.transaction.get` no longer takes a *height* argument @@ -631,7 +633,7 @@ server.version Identify the client and inform the server the range of understood protocol versions. - server.version(**client_name**, **protocol_version** = ((1, 1), (1, 1))) + server.version(**client_name**, **protocol_version** = ["1.1", "1,1"]) **client_name** @@ -639,31 +641,28 @@ protocol versions. **protocol_verion** - Optional with default value ((1, 1), (1, 1)). + Optional with default value ["1.1", "1,1"]. It must be a pair [`protocol_min`, `protocol_max`], each of which is - itself a [major_version, minor_version] pair. - - If a string was passed it should be interpreted as `protocol_min` and - `protocol_max` both being [1, 0]. + a string. The server should use the highest protocol version both support: protocol_version_to_use = min(client.protocol_max, server.protocol_max) -If this is below +If this is below the value min(client.protocol_min, server.protocol_min) -there is no protocol version in common and the server must close the -connection. Otherwise it should send a response appropriate for that -protocol version. +then there is no protocol version in common and the server must close +the connection. Otherwise it should send a response appropriate for +that protocol version. **Response** - A pair + A string - [identifying_string, protocol_version] + "m.n" identifying the server and the protocol version that will be used for future communication. @@ -672,7 +671,7 @@ protocol version. :: - server.version('2.7.11', ((1, 0), (2, 0))) + server.version('2.7.11', ["1.0", "2.0"]) server.add_peer From 9238fe397d921dd8107575754fe1c421cbd24df6 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 22 Mar 2017 23:45:44 +0900 Subject: [PATCH 011/117] Drop ports from top level of features --- lib/peer.py | 4 +--- server/controller.py | 8 +++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/peer.py b/lib/peer.py index c4bdcc8..0ea193d 100644 --- a/lib/peer.py +++ b/lib/peer.py @@ -39,9 +39,7 @@ class Peer(object): # metadata 'source', 'ip_addr', 'good_ports', 'last_connect', 'last_try', 'try_count') - PORTS = ('ssl_port', 'tcp_port') - FEATURES = PORTS + ('pruning', 'server_version', - 'protocol_min', 'protocol_max') + FEATURES = ('pruning', 'server_version', 'protocol_min', 'protocol_max') # This should be set by the application DEFAULT_PORTS = {} diff --git a/server/controller.py b/server/controller.py index a8d4d43..89ba171 100644 --- a/server/controller.py +++ b/server/controller.py @@ -513,10 +513,12 @@ class Controller(util.LoggedClass): 'Tries', 'Source', 'IP Address') for item in data: features = item['features'] - yield fmt.format(item['host'][:30], + hostname = item['host'] + host = features['hosts'][hostname] + yield fmt.format(hostname[:30], item['status'], - features['tcp_port'] or '', - features['ssl_port'] or '', + host['tcp_port'] or '', + host['ssl_port'] or '', features['server_version'] or 'unknown', features['protocol_min'], features['protocol_max'], From d198b95798f51e048d3452b0244009b6abe72532 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Thu, 23 Mar 2017 07:47:44 +0900 Subject: [PATCH 012/117] Reduce new peers per source limit to 2 by default --- server/peers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/peers.py b/server/peers.py index 080e1bc..70d070b 100644 --- a/server/peers.py +++ b/server/peers.py @@ -273,7 +273,7 @@ class PeerManager(util.LoggedClass): return [peer_data(peer) for peer in sorted(self.peers, key=peer_key)] - def add_peers(self, peers, limit=3, check_ports=False, source=None): + def add_peers(self, peers, limit=2, check_ports=False, source=None): '''Add a limited number of peers that are not already present.''' retry = False new_peers = [] From 060d32211fbea80d3ae4c0851d56fb880bbada2a Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Thu, 23 Mar 2017 08:02:49 +0900 Subject: [PATCH 013/117] Check height for all peers Closes #152 --- server/peers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/peers.py b/server/peers.py index 70d070b..1da3de1 100644 --- a/server/peers.py +++ b/server/peers.py @@ -80,6 +80,7 @@ class PeerSession(JSONSession): self.send_request(self.on_version, 'server.version', [version.VERSION, proto_ver]) self.send_request(self.on_features, 'server.features') + self.send_request(self.on_headers, 'blockchain.headers.subscribe') def connection_lost(self, exc): '''Handle disconnection.''' @@ -153,10 +154,6 @@ class PeerSession(JSONSession): self.peer_verified(True) self.peer.update_features(features) verified = True - # For legacy peers not implementing features, check their height - # as a proxy to determining they're on our network - if not verified and not self.peer.bad: - self.send_request(self.on_headers, 'blockchain.headers.subscribe') self.close_if_done() def on_headers(self, result, error): From 127b4de745c06d12826381878f6aa1e91f85aaa9 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 12 Mar 2017 12:34:40 +0900 Subject: [PATCH 014/117] Add new RPC method: add_peer --- docs/RPC-INTERFACE.rst | 11 +++++++++++ server/controller.py | 12 +++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/RPC-INTERFACE.rst b/docs/RPC-INTERFACE.rst index bb0aed9..b5dc960 100644 --- a/docs/RPC-INTERFACE.rst +++ b/docs/RPC-INTERFACE.rst @@ -156,6 +156,17 @@ The following commands are available: Currently peer data is obtained via a peer discovery protocol; it used to be taken from IRC. +* **add_peer** + + Add a peer to the peers list. ElectrumX will schdule an immediate + connection attempt. This command takes a single argument: the + peer's "real name" as it would advertise itself on IRC. + + .. code:: + + $ ./electrumx_rpc.py add_peer "ecdsa.net v1.0 s110 t" + "peer 'ecdsa.net v1.0 s110 t' added" + * **daemon_url** This command takes an optional argument that is interpreted diff --git a/server/controller.py b/server/controller.py index 89ba171..11c37ed 100644 --- a/server/controller.py +++ b/server/controller.py @@ -22,6 +22,7 @@ import pylru from lib.jsonrpc import JSONRPC, JSONSessionBase, RPCError from lib.hash import double_sha256, hash_to_str, hex_str_to_hash +from lib.peer import Peer import lib.util as util from server.block_processor import BlockProcessor from server.daemon import Daemon, DaemonError @@ -75,7 +76,7 @@ class Controller(util.LoggedClass): env.max_send = max(350000, env.max_send) self.setup_bands() # Set up the RPC request handlers - cmds = ('daemon_url disconnect getinfo groups log peers reorg ' + cmds = ('add_peer daemon_url disconnect getinfo groups log peers reorg ' 'sessions stop'.split()) self.rpc_handlers = {cmd: getattr(self, 'rpc_' + cmd) for cmd in cmds} # Set up the ElectrumX request handlers @@ -592,6 +593,15 @@ class Controller(util.LoggedClass): # Local RPC command handlers + def rpc_add_peer(self, real_name): + '''Add a peer. + + real_name: a real name, as would appear on IRC + ''' + peer = Peer.from_real_name(real_name, 'RPC') + self.peer_mgr.add_peers([peer]) + return "peer '{}' added".format(real_name) + def rpc_disconnect(self, session_ids): '''Disconnect sesssions. From ed7d8a319d779dd1fff4305d58c4404e37a56407 Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Tue, 14 Mar 2017 02:06:54 +0200 Subject: [PATCH 015/117] Refactor block parsing API --- lib/coins.py | 50 ++++++++++++++++++++------------------- lib/tx.py | 5 ++-- server/block_processor.py | 19 ++++++++------- server/db.py | 25 +++++++++++++------- 4 files changed, 56 insertions(+), 43 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index 3971407..d6d4596 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -30,6 +30,7 @@ Anything coin-specific should go in this file and be subclassed where necessary for appropriate handling. ''' +from collections import namedtuple import re import struct from decimal import Decimal @@ -40,6 +41,8 @@ from lib.hash import Base58, hash160, double_sha256, hash_to_str from lib.script import ScriptPubKey from lib.tx import Deserializer, DeserializerSegWit +Block = namedtuple("Block", "header transactions") + class CoinError(Exception): '''Exception raised for coin-related errors.''' @@ -53,6 +56,8 @@ class Coin(object): RPC_URL_REGEX = re.compile('.+@(\[[0-9a-fA-F:]+\]|[^:]+)(:[0-9]+)?') VALUE_PER_COIN = 100000000 CHUNK_SIZE = 2016 + BASIC_HEADER_SIZE = 80 + STATIC_BLOCK_HEADERS = True IRC_PREFIX = None IRC_SERVER = "irc.freenode.net" IRC_PORT = 6667 @@ -232,29 +237,33 @@ class Coin(object): return header[4:36] @classmethod - def header_offset(cls, height): + def static_header_offset(cls, height): '''Given a header height return its offset in the headers file. If header sizes change at some point, this is the only code that needs updating.''' - return height * 80 + assert cls.STATIC_BLOCK_HEADERS + return height * cls.BASIC_HEADER_SIZE @classmethod - def header_len(cls, height): + def static_header_len(cls, height): '''Given a header height return its length.''' - return cls.header_offset(height + 1) - cls.header_offset(height) + return cls.static_header_offset(height + 1) \ + - cls.static_header_offset(height) @classmethod def block_header(cls, block, height): '''Returns the block header given a block and its height.''' - return block[:cls.header_len(height)] + return block[:cls.static_header_len(height)] @classmethod - def block_txs(cls, block, height): - '''Returns a list of (deserialized_tx, tx_hash) pairs given a + def block_full(cls, block, height): + '''Returns (header, [(deserialized_tx, tx_hash), ...]) given a block and its height.''' + header = cls.block_header(block, height) deserializer = cls.deserializer() - return deserializer(block[cls.header_len(height):]).read_block() + txs = deserializer(block[len(header):]).read_tx_block() + return Block(header, txs) @classmethod def decimal_value(cls, value): @@ -635,8 +644,9 @@ class FairCoin(Coin): P2PKH_VERBYTE = bytes.fromhex("5f") P2SH_VERBYTE = bytes.fromhex("24") WIF_BYTE = bytes.fromhex("df") - GENESIS_HASH=('1f701f2b8de1339dc0ec908f3fb6e9b0' - 'b870b6f20ba893e120427e42bbc048d7') + GENESIS_HASH = ('1f701f2b8de1339dc0ec908f3fb6e9b0' + 'b870b6f20ba893e120427e42bbc048d7') + BASIC_HEADER_SIZE = 108 TX_COUNT = 1000 TX_COUNT_HEIGHT = 1000 TX_PER_BLOCK = 1 @@ -650,22 +660,14 @@ class FairCoin(Coin): ] @classmethod - def header_offset(cls, height): - '''Given a header height return its offset in the headers file. - If header sizes change at some point, this is the only code - that needs updating.''' - return height * 108 - - @classmethod - def block_txs(cls, block, height): - '''Returns a list of (deserialized_tx, tx_hash) pairs given a + def block_full(cls, block, height): + '''Returns (header, [(deserialized_tx, tx_hash), ...]) given a block and its height.''' - if height == 0: - return [] - - deserializer = cls.deserializer() - return deserializer(block[cls.header_len(height):]).read_block() + if height > 0: + return cls.block_full(block, height) + else: + return Block(cls.block_header(block, height), []) @classmethod def electrum_header(cls, header, height): diff --git a/lib/tx.py b/lib/tx.py index 9869512..d84e8af 100644 --- a/lib/tx.py +++ b/lib/tx.py @@ -1,4 +1,5 @@ # Copyright (c) 2016-2017, Neil Booth +# Copyright (c) 2017, the ElectrumX authors # # All rights reserved. # @@ -105,10 +106,10 @@ class Deserializer(object): self._read_le_uint32() # locktime ), double_sha256(self.binary[start:self.cursor]) - def read_block(self): + def read_tx_block(self): '''Returns a list of (deserialized_tx, tx_hash) pairs.''' read_tx = self.read_tx - txs = [read_tx() for n in range(self._read_varint())] + txs = [read_tx() for _ in range(self._read_varint())] # Some coins have excess data beyond the end of the transactions return txs diff --git a/server/block_processor.py b/server/block_processor.py index 933cbae..1d73df2 100644 --- a/server/block_processor.py +++ b/server/block_processor.py @@ -1,4 +1,5 @@ # Copyright (c) 2016-2017, Neil Booth +# Copyright (c) 2017, the ElectrumX authors # # All rights reserved. # @@ -231,15 +232,15 @@ class BlockProcessor(server.db.DB): .format(len(blocks), first, self.height + 1)) return - headers = [self.coin.block_header(block, first + n) - for n, block in enumerate(blocks)] + blocks = [self.coin.block_full(block, first + n) + for n, block in enumerate(blocks)] + headers = [b.header for b in blocks] hprevs = [self.coin.header_prevhash(h) for h in headers] chain = [self.tip] + [self.coin.header_hash(h) for h in headers[:-1]] if hprevs == chain: start = time.time() - await self.controller.run_in_executor(self.advance_blocks, - blocks, headers) + await self.controller.run_in_executor(self.advance_blocks, blocks) if not self.first_sync: s = '' if len(blocks) == 1 else 's' self.logger.info('processed {:,d} block{} in {:.1f}s' @@ -479,18 +480,18 @@ class BlockProcessor(server.db.DB): if utxo_MB + hist_MB >= self.cache_MB or hist_MB >= self.cache_MB // 5: self.flush(utxo_MB >= self.cache_MB * 4 // 5) - def advance_blocks(self, blocks, headers): + def advance_blocks(self, blocks): '''Synchronously advance the blocks. It is already verified they correctly connect onto our tip. ''' - block_txs = self.coin.block_txs + headers = [block.header for block in blocks] min_height = self.min_undo_height(self.daemon.cached_height()) height = self.height for block in blocks: height += 1 - undo_info = self.advance_txs(block_txs(block, height)) + undo_info = self.advance_txs(block.transactions) if height >= min_height: self.undo_infos.append((undo_info, height)) @@ -568,14 +569,14 @@ class BlockProcessor(server.db.DB): coin = self.coin for block in blocks: # Check and update self.tip - header = coin.block_header(block, self.height) + header, txs = coin.block_full(block, self.height) header_hash = coin.header_hash(header) if header_hash != self.tip: raise ChainError('backup block {} not tip {} at height {:,d}' .format(hash_to_str(header_hash), hash_to_str(self.tip), self.height)) self.tip = coin.header_prevhash(header) - self.backup_txs(coin.block_txs(block, self.height)) + self.backup_txs(txs) self.height -= 1 self.tx_counts.pop() diff --git a/server/db.py b/server/db.py index 6a3d4ef..2d65ea2 100644 --- a/server/db.py +++ b/server/db.py @@ -1,4 +1,5 @@ # Copyright (c) 2016, Neil Booth +# Copyright (c) 2017, the ElectrumX authors # # All rights reserved. # @@ -44,6 +45,13 @@ class DB(util.LoggedClass): self.env = env self.coin = env.coin + # Setup block header size handlers + if self.coin.STATIC_BLOCK_HEADERS: + self.header_offset = self.coin.static_header_offset + self.header_len = self.coin.static_header_len + else: + raise Exception("Non static headers are not supported") + self.logger.info('switching current directory to {}' .format(env.db_dir)) os.chdir(env.db_dir) @@ -191,24 +199,25 @@ class DB(util.LoggedClass): updated. These arrays are all append only, so in a crash we just pick up again from the DB height. ''' - blocks_done = len(self.headers) + blocks_done = len(headers) + height_start = fs_height + 1 new_height = fs_height + blocks_done prior_tx_count = (self.tx_counts[fs_height] if fs_height >= 0 else 0) cur_tx_count = self.tx_counts[-1] if self.tx_counts else 0 txs_done = cur_tx_count - prior_tx_count - assert len(self.tx_hashes) == blocks_done + assert len(block_tx_hashes) == blocks_done assert len(self.tx_counts) == new_height + 1 hashes = b''.join(block_tx_hashes) assert len(hashes) % 32 == 0 assert len(hashes) // 32 == txs_done # Write the headers, tx counts, and tx hashes - offset = self.coin.header_offset(fs_height + 1) + offset = self.header_offset(height_start) self.headers_file.write(offset, b''.join(headers)) - offset = (fs_height + 1) * self.tx_counts.itemsize + offset = height_start * self.tx_counts.itemsize self.tx_counts_file.write(offset, - self.tx_counts[fs_height + 1:].tobytes()) + self.tx_counts[height_start:].tobytes()) offset = prior_tx_count * 32 self.hashes_file.write(offset, hashes) @@ -220,8 +229,8 @@ class DB(util.LoggedClass): raise self.DBError('{:,d} headers starting at {:,d} not on disk' .format(count, start)) if disk_count: - offset = self.coin.header_offset(start) - size = self.coin.header_offset(start + disk_count) - offset + offset = self.header_offset(start) + size = self.header_offset(start + disk_count) - offset return self.headers_file.read(offset, size) return b'' @@ -241,7 +250,7 @@ class DB(util.LoggedClass): offset = 0 headers = [] for n in range(count): - hlen = self.coin.header_len(height + n) + hlen = self.header_len(height + n) headers.append(headers_concat[offset:offset + hlen]) offset += hlen From 37602d4eae6a4d419ce7fd3ca6d94c256ff38d17 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 14 Mar 2017 20:42:01 +0900 Subject: [PATCH 016/117] A couple of tweaks to the prior commit. --- lib/coins.py | 2 +- server/block_processor.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index d6d4596..701625c 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -665,7 +665,7 @@ class FairCoin(Coin): block and its height.''' if height > 0: - return cls.block_full(block, height) + return super().block_full(block, height) else: return Block(cls.block_header(block, height), []) diff --git a/server/block_processor.py b/server/block_processor.py index 1d73df2..fe26f63 100644 --- a/server/block_processor.py +++ b/server/block_processor.py @@ -485,7 +485,6 @@ class BlockProcessor(server.db.DB): It is already verified they correctly connect onto our tip. ''' - headers = [block.header for block in blocks] min_height = self.min_undo_height(self.daemon.cached_height()) height = self.height @@ -495,6 +494,7 @@ class BlockProcessor(server.db.DB): if height >= min_height: self.undo_infos.append((undo_info, height)) + headers = [block.header for block in blocks] self.height = height self.headers.extend(headers) self.tip = self.coin.header_hash(headers[-1]) @@ -569,14 +569,14 @@ class BlockProcessor(server.db.DB): coin = self.coin for block in blocks: # Check and update self.tip - header, txs = coin.block_full(block, self.height) - header_hash = coin.header_hash(header) + block_full = coin.block_full(block, self.height) + header_hash = coin.header_hash(block_full.header) if header_hash != self.tip: raise ChainError('backup block {} not tip {} at height {:,d}' .format(hash_to_str(header_hash), hash_to_str(self.tip), self.height)) - self.tip = coin.header_prevhash(header) - self.backup_txs(txs) + self.tip = coin.header_prevhash(block_full.header) + self.backup_txs(block_full.transactions) self.height -= 1 self.tx_counts.pop() From a820829e0ec63eb3b5223d8bce749c2c90f97860 Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Tue, 14 Mar 2017 02:27:32 +0200 Subject: [PATCH 017/117] Dynamic header support Block headers can have a dynamic size that is being indexed on a new meta file "headers_offsets". The offsets are 64 bits in order to accommodate coins with big headers that will accumulate GBs of header data after some years. Closes #128 --- docs/AUTHORS | 3 ++- server/db.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/AUTHORS b/docs/AUTHORS index 722c701..09871ce 100644 --- a/docs/AUTHORS +++ b/docs/AUTHORS @@ -1,2 +1,3 @@ Neil Booth: creator and maintainer -Johann Bauer: backend DB abstraction \ No newline at end of file +Johann Bauer: backend DB abstraction +John Jegutanis: alt-chain integrations \ No newline at end of file diff --git a/server/db.py b/server/db.py index 2d65ea2..cd42d66 100644 --- a/server/db.py +++ b/server/db.py @@ -50,7 +50,8 @@ class DB(util.LoggedClass): self.header_offset = self.coin.static_header_offset self.header_len = self.coin.static_header_len else: - raise Exception("Non static headers are not supported") + self.header_offset = self.dynamic_header_offset + self.header_len = self.dynamic_header_len self.logger.info('switching current directory to {}' .format(env.db_dir)) @@ -69,6 +70,12 @@ class DB(util.LoggedClass): self.headers_file = util.LogicalFile('meta/headers', 2, 16000000) self.tx_counts_file = util.LogicalFile('meta/txcounts', 2, 2000000) self.hashes_file = util.LogicalFile('meta/hashes', 4, 16000000) + if not self.coin.STATIC_BLOCK_HEADERS: + self.headers_offsets_file = util.LogicalFile( + 'meta/headers_offsets', 2, 16000000) + # Write the offset of the genesis block + if self.headers_offsets_file.read(0, 8) != b'\x00' * 8: + self.headers_offsets_file.write(0, b'\x00' * 8) # tx_counts[N] has the cumulative number of txs at the end of # height N. So tx_counts[0] is 1 - the genesis coinbase @@ -192,6 +199,28 @@ class DB(util.LoggedClass): self.clear_excess_history(self.utxo_flush_count) self.clear_excess_undo_info() + def fs_update_header_offsets(self, offset_start, height_start, headers): + if self.coin.STATIC_BLOCK_HEADERS: + return + offset = offset_start + offsets = [] + for h in headers: + offset += len(h) + offsets.append(pack(" Date: Tue, 14 Mar 2017 03:05:34 +0200 Subject: [PATCH 018/117] Fix support for Namecoin and Dogecoin, add Zcash support Closes #83 --- lib/coins.py | 70 +++++++++++++++++++++++++++++++++++++++-- lib/tx.py | 64 +++++++++++++++++++++++++++++++++++++ tests/test_addresses.py | 6 +++- 3 files changed, 136 insertions(+), 4 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index 701625c..8df7a20 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -39,7 +39,7 @@ from hashlib import sha256 import lib.util as util from lib.hash import Base58, hash160, double_sha256, hash_to_str from lib.script import ScriptPubKey -from lib.tx import Deserializer, DeserializerSegWit +from lib.tx import Deserializer, DeserializerSegWit, DeserializerAuxPow, DeserializerZcash Block = namedtuple("Block", "header transactions") @@ -293,6 +293,23 @@ class Coin(object): def deserializer(cls): return Deserializer +class CoinAuxPow(Coin): + # Set NAME and NET to avoid exception in Coin::lookup_coin_class + NAME = "" + NET = "" + STATIC_BLOCK_HEADERS = False + + @classmethod + def header_hash(cls, header): + '''Given a header return hash''' + return double_sha256(header[:cls.BASIC_HEADER_SIZE]) + + @classmethod + def block_header(cls, block, height): + '''Return the AuxPow block header bytes''' + block = DeserializerAuxPow(block) + return block.read_header(height, cls.BASIC_HEADER_SIZE) + class Bitcoin(Coin): NAME = "Bitcoin" @@ -451,7 +468,7 @@ class LitecoinTestnetSegWit(LitecoinTestnet): # Source: namecoin.org -class Namecoin(Coin): +class Namecoin(CoinAuxPow): NAME = "Namecoin" SHORTNAME = "NMC" NET = "mainnet" @@ -484,7 +501,7 @@ class NamecoinTestnet(Namecoin): # For DOGE there is disagreement across sites like bip32.org and # pycoin. Taken from bip32.org and bitmerchant on github -class Dogecoin(Coin): +class Dogecoin(CoinAuxPow): NAME = "Dogecoin" SHORTNAME = "DOGE" NET = "mainnet" @@ -682,3 +699,50 @@ class FairCoin(Coin): 'timestamp': timestamp, 'creatorId': creatorId, } + + +class Zcash(Coin): + NAME = "Zcash" + SHORTNAME = "ZEC" + NET = "mainnet" + XPUB_VERBYTES = bytes.fromhex("0488b21e") + XPRV_VERBYTES = bytes.fromhex("0488ade4") + P2PKH_VERBYTE = bytes.fromhex("1CB8") + P2SH_VERBYTE = bytes.fromhex("1CBD") + WIF_BYTE = bytes.fromhex("80") + GENESIS_HASH = ('00040fe8ec8471911baa1db1266ea15d' + 'd06b4a8a5c453883c000b031973dce08') + STATIC_BLOCK_HEADERS = False + BASIC_HEADER_SIZE = 140 # Excluding Equihash solution + TX_COUNT = 329196 + TX_COUNT_HEIGHT = 68379 + TX_PER_BLOCK = 5 + IRC_PREFIX = "E_" + IRC_CHANNEL = "#electrum-zcash" + RPC_PORT = 8232 + REORG_LIMIT = 800 + + @classmethod + def electrum_header(cls, header, height): + version, = struct.unpack(' 0 else False + + +class DeserializerZcash(Deserializer): + def read_header(self, height, static_header_size): + '''Return the block header bytes''' + start = self.cursor + # We are going to calculate the block size then read it as bytes + self.cursor += static_header_size + solution_size = self._read_varint() + self.cursor += solution_size + header_end = self.cursor + self.cursor = start + return self._read_nbytes(header_end) + + def read_tx(self): + start = self.cursor + base_tx = TxJoinSplit( + self._read_le_int32(), # version + self._read_inputs(), # inputs + self._read_outputs(), # outputs + self._read_le_uint32() # locktime + ) + if base_tx.version >= 2: + joinsplit_size = self._read_varint() + if joinsplit_size > 0: + self.cursor += joinsplit_size * 1802 # JSDescription + self.cursor += 32 # joinSplitPubKey + self.cursor += 64 # joinSplitSig + return base_tx, double_sha256(self.binary[start:self.cursor]) diff --git a/tests/test_addresses.py b/tests/test_addresses.py index 9ee81fd..be560d0 100644 --- a/tests/test_addresses.py +++ b/tests/test_addresses.py @@ -26,7 +26,7 @@ import pytest -from lib.coins import Litecoin, Bitcoin +from lib.coins import Litecoin, Bitcoin, Zcash from lib.hash import Base58 addresses = [ @@ -38,6 +38,10 @@ addresses = [ "206168f5322583ff37f8e55665a4789ae8963532", "b8cb80b26e8932f5b12a7e"), (Litecoin, "3GxRZWkJufR5XA8hnNJgQ2gkASSheoBcmW", "a773db925b09add367dcc253c1f9bbc1d11ec6fd", "062d8515e50cb92b8a3a73"), + (Zcash, "t1LppKe1sfPNDMysGSGuTjxoAsBcvvSYv5j", + "206168f5322583ff37f8e55665a4789ae8963532", "b8cb80b26e8932f5b12a7e"), + (Zcash, "t3Zq2ZrASszCg7oBbio7oXqnfR6dnSWqo76", + "a773db925b09add367dcc253c1f9bbc1d11ec6fd", "062d8515e50cb92b8a3a73"), ] From 50a829c3717283cb67b2a9df7aa9674fc1d4ef7f Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Tue, 14 Mar 2017 03:10:10 +0200 Subject: [PATCH 019/117] added raw block tests --- tests/blocks/bitcoin_mainnet_100000.json | 17 ++++++ tests/blocks/dogecoin_mainnet_371337.json | 19 +++++++ tests/blocks/litecoin_mainnet_900000.json | 18 ++++++ tests/blocks/namecoin_mainnet_19200.json | 14 +++++ tests/blocks/namecoin_mainnet_19204.json | 15 +++++ tests/blocks/zcash_mainnet_1000.json | 15 +++++ tests/test_blocks.py | 68 +++++++++++++++++++++++ 7 files changed, 166 insertions(+) create mode 100644 tests/blocks/bitcoin_mainnet_100000.json create mode 100644 tests/blocks/dogecoin_mainnet_371337.json create mode 100644 tests/blocks/litecoin_mainnet_900000.json create mode 100644 tests/blocks/namecoin_mainnet_19200.json create mode 100644 tests/blocks/namecoin_mainnet_19204.json create mode 100644 tests/blocks/zcash_mainnet_1000.json create mode 100644 tests/test_blocks.py diff --git a/tests/blocks/bitcoin_mainnet_100000.json b/tests/blocks/bitcoin_mainnet_100000.json new file mode 100644 index 0000000..6e0946f --- /dev/null +++ b/tests/blocks/bitcoin_mainnet_100000.json @@ -0,0 +1,17 @@ +{ + "hash": "000000000003ba27aa200b1cecaad478d2b00432346c3f1f3986da1afd33e506", + "size": 957, + "height": 100000, + "merkleroot": "f3e94742aca4b5ef85488dc37c06c3282295ffec960994b2c0d5ac2a25a95766", + "tx": [ + "8c14f0db3df150123e6f3dbbf30f8b955a8249b62ac1d1ff16284aefa3d06d87", + "fff2525b8931402dd09222c50775608f75787bd2b87e56995a7bdd30f79702c4", + "6359f0868171b1d194cbee1af2f16ea598ae8fad666d9b012c8ed2b79a236ec4", + "e9a66845e05d5abc0ad04ec80f774a7e585c6e8db975962d069a522137b80c1d" + ], + "time": 1293623863, + "nonce": 274148111, + "bits": "1b04864c", + "previousblockhash": "000000000002d01c1fccc21636b607dfd930d31d01c3a62104612a1719011250", + "block": "0100000050120119172a610421a6c3011dd330d9df07b63616c2cc1f1cd00200000000006657a9252aacd5c0b2940996ecff952228c3067cc38d4885efb5a4ac4247e9f337221b4d4c86041b0f2b57100401000000010000000000000000000000000000000000000000000000000000000000000000ffffffff08044c86041b020602ffffffff0100f2052a010000004341041b0e8c2567c12536aa13357b79a073dc4444acb83c4ec7a0e2f99dd7457516c5817242da796924ca4e99947d087fedf9ce467cb9f7c6287078f801df276fdf84ac000000000100000001032e38e9c0a84c6046d687d10556dcacc41d275ec55fc00779ac88fdf357a187000000008c493046022100c352d3dd993a981beba4a63ad15c209275ca9470abfcd57da93b58e4eb5dce82022100840792bc1f456062819f15d33ee7055cf7b5ee1af1ebcc6028d9cdb1c3af7748014104f46db5e9d61a9dc27b8d64ad23e7383a4e6ca164593c2527c038c0857eb67ee8e825dca65046b82c9331586c82e0fd1f633f25f87c161bc6f8a630121df2b3d3ffffffff0200e32321000000001976a914c398efa9c392ba6013c5e04ee729755ef7f58b3288ac000fe208010000001976a914948c765a6914d43f2a7ac177da2c2f6b52de3d7c88ac000000000100000001c33ebff2a709f13d9f9a7569ab16a32786af7d7e2de09265e41c61d078294ecf010000008a4730440220032d30df5ee6f57fa46cddb5eb8d0d9fe8de6b342d27942ae90a3231e0ba333e02203deee8060fdc70230a7f5b4ad7d7bc3e628cbe219a886b84269eaeb81e26b4fe014104ae31c31bf91278d99b8377a35bbce5b27d9fff15456839e919453fc7b3f721f0ba403ff96c9deeb680e5fd341c0fc3a7b90da4631ee39560639db462e9cb850fffffffff0240420f00000000001976a914b0dcbf97eabf4404e31d952477ce822dadbe7e1088acc060d211000000001976a9146b1281eec25ab4e1e0793ff4e08ab1abb3409cd988ac0000000001000000010b6072b386d4a773235237f64c1126ac3b240c84b917a3909ba1c43ded5f51f4000000008c493046022100bb1ad26df930a51cce110cf44f7a48c3c561fd977500b1ae5d6b6fd13d0b3f4a022100c5b42951acedff14abba2736fd574bdb465f3e6f8da12e2c5303954aca7f78f3014104a7135bfe824c97ecc01ec7d7e336185c81e2aa2c41ab175407c09484ce9694b44953fcb751206564a9c24dd094d42fdbfdd5aad3e063ce6af4cfaaea4ea14fbbffffffff0140420f00000000001976a91439aa3d569e06a1d7926dc4be1193c99bf2eb9ee088ac00000000" +} \ No newline at end of file diff --git a/tests/blocks/dogecoin_mainnet_371337.json b/tests/blocks/dogecoin_mainnet_371337.json new file mode 100644 index 0000000..a3da22e --- /dev/null +++ b/tests/blocks/dogecoin_mainnet_371337.json @@ -0,0 +1,19 @@ +{ + "hash": "60323982f9c5ff1b5a954eac9dc1269352835f47c2c5222691d80f0d50dcf053", + "size": 1704, + "height": 371337, + "merkleroot": "ee27b8fb782a5bfb99c975f0d4686440b9af9e16846603e5f2830e0b6fbf158a", + "tx": [ + "4547b14bc16db4184fa9f141d645627430dd3dfa662d0e6f418fba497091da75", + "a965dba2ed06827ed9a24f0568ec05b73c431bc7f0fb6913b144e62db7faa519", + "5e3ab18cb7ba3abc44e62fb3a43d4c8168d00cf0a2e0f8dbeb2636bb9a212d12", + "f022935ac7c4c734bd2c9c6a780f8e7280352de8bd358d760d0645b7fe734a93", + "ec063cc8025f9f30a6ed40fc8b1fe63b0cbd2ea2c62664eb26b365e6243828ca", + "02c16e3389320da3e77686d39773dda65a1ecdf98a2ef9cfb938c9f4b58f7a40" + ], + "time": 1410464577, + "nonce": 0, + "bits": "1b364184", + "previousblockhash": "46a8b109fb016fa41abd17a19186ca78d39c60c020c71fcd2690320d47036f0d", + "block": "020162000d6f03470d329026cd1fc720c0609cd378ca8691a117bd1aa46f01fb09b1a8468a15bf6f0b0e83f2e5036684169eafb9406468d4f075c999fb5b2a78fbb827ee41fb11548441361b0000000001000000010000000000000000000000000000000000000000000000000000000000000000ffffffff380345bf09fabe6d6d980ba42120410de0554d42a5b5ee58167bcd86bf7591f429005f24da45fb51cf0800000000000000cdb1f1ff0e000000ffffffff01800c0c2a010000001976a914aa3750aa18b8a0f3f0590731e1fab934856680cf88ac00000000b3e64e02fff596209c498f1b18f798d62f216f11c8462bf3922319000000000003a979a636db2450363972d211aee67b71387a3daaa3051be0fd260c5acd4739cd52a418d29d8a0e56c8714c95a0dc24e1c9624480ec497fe2441941f3fee8f9481a3370c334178415c83d1d0c2deeec727c2330617a47691fc5e79203669312d100000000036fa40307b3a439538195245b0de56a2c1db6ba3a64f8bdd2071d00bc48c841b5e77b98e5c7d6f06f92dec5cf6d61277ecb9a0342406f49f34c51ee8ce4abd678038129485de14238bd1ca12cd2de12ff0e383aee542d90437cd664ce139446a00000000002000000d2ec7dfeb7e8f43fe77aba3368df95ac2088034420402730ee0492a2084217083411b3fc91033bfdeea339bc11b9efc986e161c703e07a9045338c165673f09940fb11548b54021b58cc9ae50601000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0d0389aa050101062f503253482fffffffff010066f33caf050000232102b73438165461b826b30a46078f211aa005d1e7e430b1e0ed461678a5fe516c73ac000000000100000001ef2e86aa5f027e13d7fc1f0bd4a1fc677d698e42850680634ccd1834668ff320010000006b483045022100fcf5dc43afa85978a71e76a9f4c11cd6bf2a7d5677212f9001ad085d420a5d3a022068982e1e53e94fc6007cf8b60ff3919bcaf7f0b70fefb79112cb840777d8c7cf0121022b050b740dd02c1b4e1e7cdbffe6d836d987c9db4c4db734b58526f08942193bffffffff02004e7253000000001976a91435cb1f77e88e96fb3094d84e8d3b7789a092636d88ac00d4b7e8b00700001976a9146ca1f634daa4efc7871abab945c7cefd282b481f88ac0000000001000000010a6c24bbc92fd0ec32bb5b0a051c44eba0c1325f0b24d9523c109f8bb1281f49000000006a4730440220608577619fb3a0b826f09df5663ffbf121c8e0164f43b73d9affe2f9e4576bd0022040782c9a7df0a20afe1a7e3578bf27e1331c862253af21ced4fde5ef1b44b787012103e4f91ad831a87cc532249944bc7138a355f7d0aac25dc4737a8701181ce680a5ffffffff010019813f0d0000001976a91481db1aa49ebc6a71cad96949eb28e22af85eb0bd88ac0000000001000000017b82db0f644ecff378217d9b8dc0de8817eaf85ceefacab23bf344e2e495dca5010000006b483045022100f07ced6bfdbd6cdeb8b2c8fc92b9803f5798754b5b6c454c8f084198bea303f402205616f84d7ec882af9c34a3fd2457ca3fb81ec5a463a963a6e684edee427d4525012102c056b10494520dbd7b37e2e6bb8f72f98d73a609a926901221bfb114fa1d5a80ffffffff02f0501a22000000001976a914ca63ded8b23d0252158a3bdc816747ef89fb438988ac80b65ea1350700001976a914fb26a7c16ace531a8e7bbd925e46c67c3150c1c888ac000000000100000001c9bdba900e1579ebf4e44415fe8b9abec57a763f8c70a30604bea7fbe7c55d42000000006a47304402204ccbeeace0630e72102fdaf0836e41f8f6dcdde6a178f0fbc2d96a4d17a1df8f02207e4a91203a2abd87fdddee96510482ef96535741b6c17a1acae93c977ad248e5012103e0747583a342b76a5de9c21db138b9640d49b4f3b67a306d3b3f217416d49b55ffffffff020058850c020000001976a9144417c63a91208a02a5f46a0f7a2b806adc7d19a788ac0042dc06030000001976a9147b61c5adef0d559e5acf2901c2989294624b651988ac0000000001000000017c1423b198dfc3da37ae9a5fc11a3720e4343b3049d3b289b8285eb04595c04b000000006b483045022100b0c1cb9608bf644d7a8916bf61f36ced95bd045e97612804ca774f60e05e7bde022017c12255eecc474c8d8b05d0910013b2df8703af68212cf0962b6b8ee0e101ee01210341e154088c23b8ea943bca94c1d4f65361668a242b168522f00199365414b46affffffff01019891ad000000001976a91481db1aa49ebc6a71cad96949eb28e22af85eb0bd88ac00000000" +} \ No newline at end of file diff --git a/tests/blocks/litecoin_mainnet_900000.json b/tests/blocks/litecoin_mainnet_900000.json new file mode 100644 index 0000000..a768c9b --- /dev/null +++ b/tests/blocks/litecoin_mainnet_900000.json @@ -0,0 +1,18 @@ +{ + "hash": "545127eacc261629ae25ada99c7aadc1a929aed2da32f95ef866333f37c11e49", + "size": 1132, + "height": 900000, + "merkleroot": "11929e3e325f6346e9d24c0373dafbafcaaa7837aa862f33b7c529d457ca1229", + "tx": [ + "ad21fe3e94fd3da9a0920ed2fd112f7c805ac1b80274f4d999da3d2a5c6bd733", + "ea3b27388e968c413ef6af47be2843d649979e9b721331f593287b8d486be230", + "3b6b555a86471c5e5ee3d07838df04a6802f83b6f37c79922b86ef1983262d5e", + "026f93ffe84775b6c42b660944d25f7224c31b1175db837b664db32cd42e2300", + "7c274e298aa6feae7a0590dffca92d31b1f5f3697b26c6ceb477efc43f0afe39" + ], + "time": 1449526456, + "nonce": 685998084, + "bits": "1b014ec5", + "previousblockhash": "93819e801bbdaec2698e3dda35e12be0a0004759c635924fda7f007a358848be", + "block": "03000000be4888357a007fda4f9235c6594700a0e02be135da3d8e69c2aebd1b809e81932912ca57d429c5b7332f86aa3778aacaaffbda73034cd2e946635f323e9e9211b8046656c54e011b0480e3280501000000010000000000000000000000000000000000000000000000000000000000000000ffffffff6403a0bb0de4b883e5bda9e7a59ee4bb99e9b1bcfabe6d6d227c509f30b3ac49948323ce5974f89f6261ed913a701fc137bc08ead15179b940000000f09f909f4d696e6564206279206c6a6a38380000000000000000000000000000000000000000000000510000000176fa0795000000001976a914aa3750aa18b8a0f3f0590731e1fab934856680cf88ac5aa5893301000000015992d44c8d8790727c91055ce305e115373ff7fe32d632edc3f9939914b7d810000000006a47304402203ea789e265999b19b2155e4eb6135a50773d45836e1abb00a4959126c323e25d02207db24c9069683a6e4fc850a700717da08bf2c3ea80e8f3ee1ac75c1b702198800121033567eb9b5281b320bd8f20718b205e1808e7c0432d41991bdfad3eb5b53c49f9ffffffff02427e8837000000001976a914bc8d35412e239d91f9c95548afa15e22f094be3688ac0027b929000000001976a914b5e82238517f926b14467fbf8f90812b0eec8e5288ac000000000100000001ad3d610da30df966af2407b45bf0236a782f1e4444b829bf59da1679ceb16733000000006a47304402206b32468586635a1965fbb1c186799f1ccfce13549bd098845b97e75ea8bff473022021f35faf6e67428d51e58ed1895f9db2d40337d04e1b8819154c2bc71b0446af012102a740669302896fc4bdba32a951a67f95b3369fbc2ac97f1fda559999866d623bffffffff0245781300000000001976a9146f67216770c0af807e0597896a8c8ec306994e7b88ac80841e00000000001976a914b5e8223ec1e89b386cb5beb1c30cf165ac84e46388ac000000000100000001520f304eec49a1a9eeb0682da600b436a8dd43efc97ff4ed6ac2bcf0912e5caa000000006a473044022040218475e180db66cf71aa56668145b4f4d4d0a93b0e3777985039d87a53f881022047aaef5b4e262365c2dd2d7e1cbdf3016ff22468faef6104e4397540c199dfc6012103418a46f4534e7ec8a98146da6431550c370069777cacfdfbccc7a01f31abd1d0ffffffff02505bd425000000001976a9149f74e62f0f92663525050b56ad8b180048b4e80488ac408d1c1b000000001976a9149f7044d46304c187dc08d05864aeccb5a044e45588ac00000000010000000139c9bb7efca3fdd77ae18adf87614827d1c0bb1803a0d50ae42342e524ca99b7000000006a47304402205b75fd27c33c89346bc778d1369549b27f41ed0ded4947a19fb2884363a8ee7502206672bb1bd4e4a2a89cba62d1c5a93e1a6ae042f379e57380aebf14a693b42bea0121024f5b70c3309c77762c1b487f804c9666f5302545d7555d1808b63fdc9c17f840ffffffff01f3247d00000000001976a9149a20d4f533a7d7670cf14c77107dfd1eefddbd5388ac00000000" +} \ No newline at end of file diff --git a/tests/blocks/namecoin_mainnet_19200.json b/tests/blocks/namecoin_mainnet_19200.json new file mode 100644 index 0000000..ea99f04 --- /dev/null +++ b/tests/blocks/namecoin_mainnet_19200.json @@ -0,0 +1,14 @@ +{ + "hash": "d8a7c3e01e1e95bcee015e6fcc7583a2ca60b79e5a3aa0a171eddd344ada903d", + "size": 678, + "height": 19200, + "merkleroot": "88afdfdcc78f778f701835b62e432d3ba7d55b3e59ac4e7cab08d6bc49655c0f", + "tx": [ + "88afdfdcc78f778f701835b62e432d3ba7d55b3e59ac4e7cab08d6bc49655c0f" + ], + "time": 1318066829, + "nonce": 0, + "bits": "1b00b269", + "previousblockhash": "000000000000b19f0ad5cd46859fe8c9662e8828d8a75ff6da73167ac09a9036", + "block": "0101010036909ac07a1673daf65fa7d828882e66c9e89f8546cdd50a9fb10000000000000f5c6549bcd608ab7c4eac593e5bd5a73b2d432eb63518708f778fc7dcdfaf888d1a904e69b2001b0000000001000000010000000000000000000000000000000000000000000000000000000000000000ffffffff35045dee091a014d522cfabe6d6dd8a7c3e01e1e95bcee015e6fcc7583a2ca60b79e5a3aa0a171eddd344ada903d0100000000000000ffffffff0160a0102a01000000434104f8bbe97ed2acbc5bba11c68f6f1a0313f918f3d3c0e8475055e351e3bf442f8c8dcee682d2457bdc5351b70dd9e34026766eba18b06eaee2e102efd1ab634667ac00000000a903ef9de1918e4b44f6176a30c0e7c7e3439c96fb597327473d00000000000005050ac4a1a1e1bce0c48e555b1a9f935281968c72d6379b24729ca0425a3fc3cb433cd348b35ea22806cf21c7b146489aef6989551eb5ad2373ab6121060f30341d648757c0217d43e66c57eaed64fc1820ec65d157f33b741965183a5e0c8506ac2602dfe2f547012d1cc75004d48f97aba46bd9930ff285c9f276f5bd09f356df19724579d65ec7cb62bf97946dfc6fb0e3b2839b7fdab37cdb60e55122d35b0000000000000000000100000008be13295c03e67cb70d00dae81ea06e78b9014e5ceb7d9ba504000000000000e0fd42db8ef6d783f079d126bea12e2d10c104c0927cd68f954d856f9e8111e59a23904e5dee091a1c6550860101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff080469b2001b010152ffffffff0100f2052a0100000043410489fe91e62847575c98deeab020f65fdff17a3a870ebb05820b414f3d8097218ec9a65f1e0ae0ac35af7247bd79ed1f2a24675fffb5aa6f9620e1920ad4bf5aa6ac00000000" +} diff --git a/tests/blocks/namecoin_mainnet_19204.json b/tests/blocks/namecoin_mainnet_19204.json new file mode 100644 index 0000000..28edfca --- /dev/null +++ b/tests/blocks/namecoin_mainnet_19204.json @@ -0,0 +1,15 @@ +{ + "hash": "000000000000122ff239e71146bf57aee28ad913931d672cd124255e91351660", + "size": 475, + "height": 19204, + "merkleroot": "45d5bc5330dad21dd4dcf0daadefef4ba826fe81e2dd2fa38a4a49ea06c97b1d", + "tx": [ + "7752b6a596641bd90ae71d1bc054f3dd1ad36ce3fe7e7d3292ff8594feafb8a9", + "499dad7cd9e737c9f2f10bc4b3930b566d82288a8c02b68a50d2cf2694868bdd" + ], + "time": 1318073606, + "nonce": 3373003561, + "bits": "1b00b269", + "previousblockhash": "07d6d85d2f33fae0b52d84a90757d1600fdb3f2cf2f31f2a32eef59172245af6", + "block": "01000100f65a247291f5ee322a1ff3f22c3fdb0f60d15707a9842db5e0fa332f5dd8d6071d7bc906ea494a8aa32fdde281fe26a84befefaddaf0dcd41dd2da3053bcd5450635904e69b2001b29f30bc90201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff080469b2001b027829ffffffff0100f2052a0100000043410439cf5bc2e4b0d555178b3d19fa82d59aa998cc082086f874928af6e70c1093b300b6371d093ac9d41393e11907ed17d2489405e220a6f08feeecbce9f6cfcc0bac00000000010000000148efa1ba7512bbd538033b798da1064e724e21739f6bd8ea0c28e3d0d53136d6010000008c49304602210095cf1a495623ed7794746b7b0f2daa70a9783f635e24991487e8a6869b553c9b0221009dec7919115c3a84f03236c8aea6e175e8dd9ee3a6daa1f6c56ac1d6246ec5da014104d483cffe3907aefdb9a20dab73dd4c83f6d14d26bd9d21aeccd33b0be2068e4832fea66110606198728413ad88a6dd642bdd6ff72aefd79732a2375c3129f1fcffffffff0220ab6136000000001976a9141b5a80636dfa8c4e78c1d1150a2ba961d704911388ac0065cd1d000000001976a9143f47c122f3a71e70cb3a4c9d215e5ce7b740b96a88ac00000000" +} diff --git a/tests/blocks/zcash_mainnet_1000.json b/tests/blocks/zcash_mainnet_1000.json new file mode 100644 index 0000000..2c36dd6 --- /dev/null +++ b/tests/blocks/zcash_mainnet_1000.json @@ -0,0 +1,15 @@ +{ + "hash": "0000000b70480327694608408728c65c1f1a300bfe705b01baca0f5504092e1b", + "size": 3562, + "height": 1000, + "merkleroot": "48a46e38901fe503b4d0733d28f601443e388c789d5bcecda11ae29e25a4c980", + "tx": [ + "2a03a4110c62047af28a44ab78ec9af9d020c9b8128051b46d89c22cd34777d5", + "fae6c7ce358e018779b11a4fba4ec0850a57e1585102843f82ebca76df1c27b9" + ], + "time": 1477750705, + "nonce": "0200000000000000000000000000000000000000000000000005b2c70a9355a5", + "bits": "1d0fe28d", + "previousblockhash": "0000000aeb86e32aeb49ef9e40bfbd69585bd67ebd62d535e7e728ed2ba42064", + "block": "040000006420a42bed28e7e735d562bd7ed65b5869bdbf409eef49eb2ae386eb0a00000080c9a4259ee21aa1cdce5b9d788c383e4401f6283d73d0b403e51f90386ea4480000000000000000000000000000000000000000000000000000000000000000b1af14588de20f1da555930ac7b20500000000000000000000000000000000000000000000000002fd4005003bd1eed8ec2acfb38c728dcffafcc1c10716b2d518119615c95684fd2ee693a7134490f943c17ab4130bcc16663ee26667d79f32070d97c426e06f7a94c515ae7fbfd61cc0735eae12a6f6996c213d6691370a0b640f7a6143d431fa21f31c76315c9e0d60b6176735fa92942cd54547bba348f7a96dca66677a342b3c0c51677fd113762fc28df66c73ca4762fdd4f7f7303e18cd2d3b1e2f3716ec975dacc923fac0e55926dc03490b7b6cceea20c2bab9b2d678fe269ceb580ccb346e65fdd84dc0e38bc1660f90f917ce7f21fa10de034f4773500f311fdf4c76c16afd396338eadfd96e046bf70a20ea0cddb0b687d906bf1b6a12db3e411c041aa3d6c40d088b1040f3fd4d229e1a7f0777a3031efb030eee1de2d5bbbc84075972b8da18273802150c3c98fc48837784caf905b4556f1133afb65dedf82825852ea5e31bd7a45997a4034168771104f8b343015c1abd0716d10d6bc1f03cb905857dcc56f7e2fc6c1d83762c2065ebfe0d87f289fccbde95715557bb21cd02bc09cc5013028a523ab63e490281863e65ca2d7247ff7de13defdf5db5d5db3837723e1758318f062321750bc1c17371475509c3e1ef154be46f59551c136f50c06ef29b97930490022d75855cdf18278a185124020a13c2a1de456494a17838c77ede9dbfb025234e13b6d7b43cd561e6930fd3a709a5ff4f84dc05ac3ff32b687ed97db3644fb5cd752d4138784fb011fd65cac32a02eb93ad528127f41b494d6079b970399ffbd371ed93d78eaa08d10878bb1b44893d6ad14e5ceeb34ee044d7af1056e7ff64984763985ed6e40f42e2470e36af8dd0a87140f7eee9c8940d6e154b27a3db8a10a221e59a2bf42429561b3a9c4cfab6f114391b1fd02cf2f3ec4ac77acaf4cc0e9ddcb8b20919584cb11dab8305e76302da779f0d9ef0f358aac902270f857d215c032a766183cd27f1a9e4d25f59ac7360f424c82b6ea3dced576ed36c7707184e9cbe760bc953e4444e8967e4f0b84808fde99704153f8b2a1b3da14228d8e71d12229734ddddbdf208fc3444840f9bb334d21ae2b58ec0c669e4f93e4ec8d659e5d52492daea112423ef4433f40bdfba877db7e7db053515288fd1c4ded1af01c4d60e9bc39e6e78f9b9c6628beb1d7064e5c79fdb8a5a77ef78185aa822d95caf0ae776abd3cc813266ba93b58853e6cf637c7fb865164b1c58e665ecbbacd381b8c7fee0da7a21f6bd6e0f43f95c85e9fb6bbc59db59d4ef23fb7a821f04a50f79aa14ee862e36f671b5e3733b733ef2b69ecd880d843a3ea7864d134a6f32b502ac549ee2aebb2bde0e9826a971d7ce4d7937940468bb94917bcff560d33e9b04434cd9ce8edb5e370afedc34b3bcbd1de6c04cbc242f679e7916fc4da54f7e609ffe32b41d43d5044816d3b7cac77e78df188723574b5a62f6b7faf2263c0f3a469d683dba34087691f10dba3e745408d917ec5324072f2e5dc5a743d27d78b1590389ab54091809791df5e15ef9472c71886328856883ff49739707a0d8fcf2707d47864d85e16562ae8666a018ee4210f203fd98096584908d740ccc794b3aa48017f75f1e02a368698bd049faf62a4c8efda1838f295f88c741626fafa9de9dc17d75a6d1cf6256929fd6589fa00655dbdb41e3cb0b63e523d18fbb8662c14e777dae0bfe6a91618aa4f744b452e580a47c6a78f75dfcb425f399ebf06de31fe4f8252fddbdc48a97d6dc43484cb775f9f617945ffe7e181498c153de17f0bbc2fa0eac18e201a00145e64a02a7bfa866a166b45857281683ae34bb8e4bbe96a8b2305117169a2afed309921554bc551bcc9aac9488b28af5e10321ce53101e79348b3d200e8e3564d78084c017a6239a3dee765ebf0201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0402e80300ffffffff029017fb0200000000232103d2b11b34418e65f8358fb89bee21432b6505eb860614560ce7180fa362f8d965ac20bcbe000000000017a9147d46a730d31f97b1930d3368a967c309bd4d136a8700000000020000000001cb5f0800000000001976a91462781a2b98afe68df3969626aed5140083b281be88ac00000000010000000000000000db86080000000000d22e16e61fb5ffe03dbd17120fe278119283fc058630b151c7d57228b915d59b5a3807a619154f2e843190aef7d6178ca13c749049a1c187806b0a82045ebb075d93bee3646fef8d8354172f5033d33dee040ab22008a6dbc2f4cbcf0a535a14358d59e198a5cf851c1b8390dd1b11e1f3035c8de30af8debbeee05a711e3b3a5a2d42333cfbdaebb7c6f7149af180f5d3affaf9904d7bf0e9219f756e2934475f6c8d130ea154080d0b13f84b201bba94996ca5b63aebd1b5db126140cd431c5244d3f0c61c7a99120a667ee19eac1d40534b440562e4b62bda003fd78c00a0bb687912dc6d940ffc0f90913e1ad66f3b083deebd7fc7a84e1d436846446be306d97ea37bf0f6e6d604fba23c37cac3f0dabecb9fb6fa0efdbea5cf45b09711032981cf8ab112a4be60c7060217fb46ff2188563c422572769a4badd8fd07f4fd021850f8ee1116f6f0d0e5892c36882590b0bce2068416b2fcb6718fc2753bd3910a05431c92587fad8e0ef181a0250cca1f4b48fea77c248ca8801296ca8dc2aeb56bcdc951b381631daf2ebef7a209367cb7fc65a76af1ba1745bacc467ea1fbd0022cfe1d0bb59d4ce857cd5d093f764aa359449a7d6c9c0da7d8c3919ce49a3c5602287de3b88ba75425325a9877d50d39222f8feececebcba020b7631b561eb9b4e0319ba59eab765c93bcb8a6093e88e12a2651cb5edf421952e287b36ec3356bfbf020327062e850495d97b0b5dcd0016f43f7c111ae0024f31b6043eebecbc008921031fdc03bec3b1c845bbdcac9ef296885afabb94f3ff16e2ab6d8305936485434706cbfeff58e4e610c73ae96a4ec156cbf0118f4dc37b56ba25060368758acf48a30a9cffcff3cb8c515db2340ba29ab5f7d8ee98157c74150bf0d6449ba885dee98b6ab9987e3d4fd7ecb141c24897eb9aea17040df30e3b6191f509e10ae55e3481d4b49a2800c427eeb2b3119b782a99af3c9d4489f5d53d05d88790787cbf224dda53bc3377b77a09b46127ca69333dd1f265bf385751ad65def57079084c3292485891f2066903f6f86164acdf03b2419a280873581d0588e547af06f7da6bdc4c052008d4f21e278de9b72ff4c61909128dac8b3cc211060e7bbe7b8ec0067ddc7d1f273c677135436eb47bf9540e0a1821b2e1032fa47deaa1e0a27cd0c340ae4a6e5cdc178d637aab0d3f5825e2cd2b3c390ef2cedbe22482c150418400bb8d4a7a9c75affb3794b13837f3d5125316b4a2b70478d1dbc93da56f9aee295c32305efde3ae835288ba1d20e674f958556c46b4574695b853866c40a4357eca23fbdfaef5565341bcdfa204fa9d234e6a19138bddbdbb531face9175c76126c9bad3b4e1ee4300909485e9bc70063e2c6e0d86c6c1c585fbc202cf2af867079a0c528a9c541ae0f73833a0ba8616ae2e45b02d338c843748664aeafb6d1ba197f0a943fe8e1988c5b3e27127c14f45e6c1878446567e0ccde39c3a8ff361b17bf50dc295788e281d9d0051bd1d3ceab5fe494a0efdf957bb02ff139b752eccf92c7c454dafee9e83df912c602ffa32e59c334213df610f543b07bf547ba35d9b39b4079476b28e78e272def1e42bfafb91459e9d236eb7a4ddcb44bded2b3d2d9571e7c18ffbe7368fbf350116d350f9e8fee8afc30790fbd0e1fa3642644c92e018ed9ecbed99315d0e199bbe4e63aee1e66f1a8f1295b7476cfc02d5c41acd85fa2cc2699684ff897c9d52fb7a417a83597ff9e89be120a34ef7041131022f6494905fb78f60f31eca6a651fff2edee6f287871267ff1cbf84f12a73e9bf1bf6d16e794f37b7325c1da5c44774af8475bf7c9743251a4a111b977aca58d4d396c83aef0b05bbca73edacd6c691fef2a4371cf42d9292f416e72c0caa78d3f8543d70f999caaafb9ba465c64e9efc79398d79dc0a1cff4de789a563a3a4c7969d8497bf114870af1aa68d87a7cd2102b82c2f3eddb77a0b7e5ab5b618a50c9b642b968ab37b15b36c0266c87a4c2c98fa2bf5f368fd147d99924d615b83db2c4ebd6c93f42cd3d9b7260ccf6dc4ecac7e06262e5b6dd7eae1051236a89715d19c0bef814887d319f6f16f9bcd0425d83357b5b2e8dfcc2d4bd06d2d0390f1dbcea244700309fe870039e7548613c5ada8496c53d23be00399d3a126a48f0a966ce8c309541e80528d9fc66404359d2ccd69d808031678c5e3f44524f957b3de7d5ce9594671977e8d903921acf437d05f80a4785ed99eefc23c28797dd839a23b311d35206e9cc16ae16ed409cf90a36e997e5bfd23d20efed87491e5fd1cc0725ca9e021d48bf83af9bd82629b82651ec6a4e60d95e237ca9222a3d5e19111b064f2f4d79c149ba0f9b5af0600fa7ae58012aa8f456ae3af679405a2eb5a6e5a3fdda01891e129b5d756816803d92d742eca0dd81879ae2781577cc16cc109901311fc494fd4a80cf4cc41a68ec46d4583e912d61a7d8febe0d07b26a52c45a81932127a6e70a8c1002039a528f520549da16bb0ffa3a767adf69a0fc5e1a0bbb0cc03c78325d8585e754c5a10f9a9e055ac1ac3ca26f9b0bde90012402d91d84a1a006a43145900fde4966854315311cab068dcc9e893c23d308998fc688d63e0779e789eb711eee7c4a87d30503" +} diff --git a/tests/test_blocks.py b/tests/test_blocks.py new file mode 100644 index 0000000..ff8b756 --- /dev/null +++ b/tests/test_blocks.py @@ -0,0 +1,68 @@ +# Copyright (c) 2017, the ElectrumX authors +# +# All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# and warranty status of this software. + +import json +import os +from binascii import unhexlify + +import pytest + +from lib.coins import Coin +from lib.hash import hex_str_to_hash + +BLOCKS_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)), 'blocks') + +# Find out which db engines to test +# Those that are not installed will be skipped +blocks = [] + +for name in os.listdir(BLOCKS_DIR): + try: + name_parts = name.split("_") + coin = Coin.lookup_coin_class(name_parts[0], name_parts[1]) + with open(os.path.join(BLOCKS_DIR, name)) as f: + blocks.append((coin, json.load(f))) + except Exception as e: + blocks.append(pytest.mark.skip(name)) + + +@pytest.fixture(params=blocks) +def block_details(request): + return request.param + + +def test_block(block_details): + coin, block_info = block_details + + block = unhexlify(block_info['block']) + h, txs = coin.block_full(block, block_info['height']) + + assert coin.header_hash(h) == hex_str_to_hash(block_info['hash']) + assert coin.header_prevhash(h) == hex_str_to_hash(block_info['previousblockhash']) + for n, tx in enumerate(txs): + _, txid = tx + assert txid == hex_str_to_hash(block_info['tx'][n]) From 10a2c8fed5617d35d4ea4feb94a33cf0ac78fe5c Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 25 Mar 2017 10:32:08 +0900 Subject: [PATCH 020/117] Update docs --- contrib/daemontools/env/{NETWORK => NET} | 0 contrib/raspberrypi3/run_electrumx.sh | 4 +--- 2 files changed, 1 insertion(+), 3 deletions(-) rename contrib/daemontools/env/{NETWORK => NET} (100%) diff --git a/contrib/daemontools/env/NETWORK b/contrib/daemontools/env/NET similarity index 100% rename from contrib/daemontools/env/NETWORK rename to contrib/daemontools/env/NET diff --git a/contrib/raspberrypi3/run_electrumx.sh b/contrib/raspberrypi3/run_electrumx.sh index f97b9f8..e3d633f 100644 --- a/contrib/raspberrypi3/run_electrumx.sh +++ b/contrib/raspberrypi3/run_electrumx.sh @@ -5,7 +5,7 @@ # configure electrumx export COIN=Bitcoin export DAEMON_URL=http://rpcuser:rpcpassword@127.0.0.1 -export NETWORK=mainnet +export NET=mainnet export CACHE_MB=400 export DB_DIRECTORY=/home/username/.electrumx/db export SSL_CERTFILE=/home/username/.electrumx/certfile.crt @@ -34,5 +34,3 @@ ulimit -n 10000 # add this line to crontab -e # @reboot /path/to/run_electrumx.sh - - From 84c201f66586e4b527c2cc56276bb6b43f1c552c Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 25 Mar 2017 10:43:14 +0900 Subject: [PATCH 021/117] Improve diagnostic --- server/env.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/env.py b/server/env.py index dcaaad2..bbba097 100644 --- a/server/env.py +++ b/server/env.py @@ -78,9 +78,10 @@ class Env(LoggedClass): '' ) if not main_identity.host.strip(): - raise self.Error('IRC host is empty') + raise self.Error('REPORT_HOST host is empty') if main_identity.tcp_port == main_identity.ssl_port: - raise self.Error('IRC TCP and SSL ports are the same') + raise self.Error('REPORT_TCP_PORT and REPORT_SSL_PORT are both {}' + .format(main_identity.tcp_port)) self.identities = [main_identity] tor_host = self.default('REPORT_HOST_TOR', '') From 7a2f29aabeb3829c0b469c82d82b91abd857f507 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 25 Mar 2017 11:36:14 +0900 Subject: [PATCH 022/117] Be stricter accepting add_peer requests - rate-limit onion add_peer requests - for clearnet peers only accept if the peer resolves to the source address --- server/controller.py | 10 ---------- server/peers.py | 46 ++++++++++++++++++++++++++++++++++++-------- server/session.py | 10 +++------- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/server/controller.py b/server/controller.py index 11c37ed..81d663e 100644 --- a/server/controller.py +++ b/server/controller.py @@ -49,7 +49,6 @@ class Controller(util.LoggedClass): self.executor = ThreadPoolExecutor() self.loop.set_default_executor(self.executor) self.start_time = time.time() - self.next_add_peer_time = self.start_time self.coin = env.coin self.daemon = Daemon(env.coin.daemon_urls(env.daemon_url)) self.bp = BlockProcessor(env, self, self.daemon) @@ -136,15 +135,6 @@ class Controller(util.LoggedClass): def is_deprioritized(self, session): return self.session_priority(session) > self.BANDS - def permit_add_peer(self): - '''To prevent lots of add_peer requests filling up the peer - table, accept only one per random time interval.''' - now = time.time() - if now < self.next_add_peer_time: - return False - self.next_add_peer_time = now + random.randrange(0, 1800) - return True - async def run_in_executor(self, func, *args): '''Wait whilst running func in the executor.''' return await self.loop.run_in_executor(None, func, *args) diff --git a/server/peers.py b/server/peers.py index 1da3de1..5800d5c 100644 --- a/server/peers.py +++ b/server/peers.py @@ -14,6 +14,7 @@ import ssl import time from collections import defaultdict, Counter from functools import partial +from socket import SOCK_STREAM from lib.jsonrpc import JSONSession from lib.peer import Peer @@ -221,6 +222,7 @@ class PeerManager(util.LoggedClass): # any other peers with the same host name or IP address. self.peers = set() self.onion_peers = [] + self.permit_onion_peer_time = time.time() self.last_tor_retry_time = 0 self.tor_proxy = SocksProxy(env.tor_proxy_host, env.tor_proxy_port, loop=self.loop) @@ -302,15 +304,43 @@ class PeerManager(util.LoggedClass): if retry: self.retry_event.set() - def on_add_peer(self, features, source): - '''Add peers from an incoming connection.''' + def permit_new_onion_peer(self): + '''Accept a new onion peer only once per random time interval.''' + now = time.time() + if now < self.permit_onion_peer_time: + return False + self.permit_onion_peer_time = now + random.randrange(0, 1200) + return True + + async def on_add_peer(self, features, source_info): + '''Add a peer (but only if the peer resolves to the source).''' + if not source_info: + return False + source = source_info[0] peers = Peer.peers_from_features(features, source) - if peers: - hosts = [peer.host for peer in peers] - self.log_info('add_peer request from {} for {}' - .format(source, ', '.join(hosts))) - self.add_peers(peers, check_ports=True) - return bool(peers) + if not peers: + return False + + # Just look at the first peer, require it + peer = peers[0] + host = peer.host + if peer.is_tor: + permit = self.permit_new_onion_peer() + reason = 'rate limiting' + else: + infos = await self.loop.getaddrinfo(host, 80, type=SOCK_STREAM) + permit = any(source == info[-1][0] for info in infos) + reason = 'source-destination mismatch' + + if permit: + self.log_info('accepted add_peer request from {} for {}' + .format(source, host)) + self.add_peers([peer], check_ports=True) + else: + self.log_warning('rejected add_peer request from {} for {} ({})' + .format(source, host, reason)) + + return permit def on_peers_subscribe(self, is_tor): '''Returns the server peers as a list of (ip, host, details) tuples. diff --git a/server/session.py b/server/session.py index 6ae9d66..b7e4ddb 100644 --- a/server/session.py +++ b/server/session.py @@ -193,14 +193,10 @@ class ElectrumX(SessionBase): self.subscribe_height = True return self.height() - def add_peer(self, features): - '''Add a peer.''' - if not self.controller.permit_add_peer(): - return False + async def add_peer(self, features): + '''Add a peer (but only if the peer resolves to the source).''' peer_mgr = self.controller.peer_mgr - peer_info = self.peer_info() - source = peer_info[0] if peer_info else 'unknown' - return peer_mgr.on_add_peer(features, source) + return await peer_mgr.on_add_peer(features, self.peer_info()) def peers_subscribe(self): '''Return the server peers as a list of (ip, host, details) tuples.''' From be5397a853fbc67164890a9ecb9c94ead6d6d0f5 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 25 Mar 2017 11:51:29 +0900 Subject: [PATCH 023/117] Validate hostnames before accepting a new peer Don't retain non-public addresses Closes #157 --- lib/peer.py | 6 +++--- lib/util.py | 11 +++++++++++ server/peers.py | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/peer.py b/lib/peer.py index 0ea193d..1e3c630 100644 --- a/lib/peer.py +++ b/lib/peer.py @@ -28,7 +28,7 @@ import re from ipaddress import ip_address -from lib.util import cachedproperty +from lib.util import cachedproperty, is_valid_hostname class Peer(object): @@ -144,7 +144,7 @@ class Peer(object): if ip: return ((ip.is_global or ip.is_private) and not (ip.is_multicast or ip.is_unspecified)) - return True + return is_valid_hostname(self.host) @cachedproperty def is_public(self): @@ -152,7 +152,7 @@ class Peer(object): if ip: return self.is_valid and not ip.is_private else: - return self.host != 'localhost' + return self.is_valid and self.host != 'localhost' @cachedproperty def ip_address(self): diff --git a/lib/util.py b/lib/util.py index e35d3b8..e6fdd9a 100644 --- a/lib/util.py +++ b/lib/util.py @@ -31,6 +31,7 @@ import array import inspect from ipaddress import ip_address import logging +import re import sys from collections import Container, Mapping @@ -241,3 +242,13 @@ def address_string(address): if host.version == 6: fmt = '[{}]:{:d}' return fmt.format(host, port) + +# See http://stackoverflow.com/questions/2532053/validate-a-hostname-string +SEGMENT_REGEX = re.compile("(?!-)[A-Z\d-]{1,63}(? 255: + return False + # strip exactly one dot from the right, if present + if hostname[-1] == ".": + hostname = hostname[:-1] + return all(SEGMENT_REGEX.match(x) for x in hostname.split(".")) diff --git a/server/peers.py b/server/peers.py index 5800d5c..4525765 100644 --- a/server/peers.py +++ b/server/peers.py @@ -277,7 +277,7 @@ class PeerManager(util.LoggedClass): retry = False new_peers = [] for peer in peers: - if not peer.is_valid: + if not peer.is_public: continue matches = peer.matches(self.peers) if not matches: From 31755e1dace318c4e5475a73097cc78e9117d2de Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 25 Mar 2017 12:06:53 +0900 Subject: [PATCH 024/117] Update PROTOCOL.rst and PEER_DISCOVERY.rst --- docs/PEER_DISCOVERY.rst | 56 +++++++++++++++++++++++++++++-------- docs/PROTOCOL.rst | 61 +++++++++++++++++++++++++++++------------ 2 files changed, 87 insertions(+), 30 deletions(-) diff --git a/docs/PEER_DISCOVERY.rst b/docs/PEER_DISCOVERY.rst index a69b9c3..699f9e2 100644 --- a/docs/PEER_DISCOVERY.rst +++ b/docs/PEER_DISCOVERY.rst @@ -72,11 +72,12 @@ Maintaining the Peer Database In order to keep its peer database up-to-date and fresh, if some time has passed since the last successful connection to a peer, an Electrum server should make an attempt to connect, choosing either the TCP or -SSL port. On connecting it should issue **server.peers.subscribe** -and **server.features** RPC calls to collect information about the -server and its peers, and if it is the first time connecting to this -peer, a **server.add_peer** call to advertise itself. Once this is -done and replies received it should terminate the connection. +SSL port. On connecting it should issue **server.peers.subscribe**, +**blockchain.headers.subscribe**, and **server.features** RPC calls to +collect information about the server and its peers. If the peer seems +to not know of you, you can issue a **server.add_peer** call to +advertise yourself. Once this is done and replies received it should +terminate the connection. The peer database should view information obtained from an outgoing connection as authoritative, and prefer it to information obtained @@ -84,13 +85,12 @@ from any other source. On connecting, a server should confirm the peer is serving the same network, ideally via the genesis block hash of the **server.features** -RPC call below. If the peer does not implement that call, perhaps -instead check the **blockchain.headers.subscribe** RPC call returns a -peer block height within a small number of the expected value. If a -peer is on the wrong network it should never be advertised to clients -or other peers. Such invalid peers should perhaps be remembered for a -short time to prevent redundant revalidation if other peers persist in -advertising them, and later forgotten. +RPC call below. Also the height reported by the peer should be within +a small number of the expected value. If a peer is on the wrong +network it should never be advertised to clients or other peers. Such +invalid peers should perhaps be remembered for a short time to prevent +redundant revalidation if other peers persist in advertising them, and +later forgotten. If a connection attempt fails, subsequent reconnection attempts should follow some kind of exponential backoff. @@ -200,3 +200,35 @@ the hard-coded peer list used to seed this process should suffice. Any peer on IRC will report other peers on IRC, and so if any one of them is known to any single peer implementing this protocol, they will all become known to all peers quite rapidly. + + +Notes to Implementators +----------------------- + +* it is very important to only accept peers that appear to be on the + same network. At a minimum the genesis hash should be compared (if + the peer supports the *server.features* RPC call), and also that the + peer's reported height is within a few blocks of your own server's + height. +* care should be taken with the *add_peer* call. Consider only + accepting it once per connection. Clearnet peer requests should + check the peer resolves to the requesting IP address, to prevent + attackers from being able to trigger arbitrary outgoing connections + from your server. This doesn't work for onion peers so they should + be rate-limited. +* it should be possible for a peer to change their port assignments - + presumably connecting to the old ports to perform checks will not + work. +* peer host names should be checked for validity before accepting + them; and *localhost* should probably be rejected. If it is an IP + address it should be a normal public one (not private, multicast or + unspecified). +* you should limit the number of new peers accepted from any single + source to at most a handful, to limit the effectiveness of malicious + peers wanting to trigger arbitrary outgoing connections or fill your + peer tables with junk data. +* in the response to *server.peers.subscribe* calls, consider limiting + the number of peers on similar IP subnets to protect against sybil + attacks, and in the case of onion servers the total returned. +* you should not advertise a peer's IP address if it also advertises a + hostname (avoiding duplicates). diff --git a/docs/PROTOCOL.rst b/docs/PROTOCOL.rst index d260c76..b5d4e52 100644 --- a/docs/PROTOCOL.rst +++ b/docs/PROTOCOL.rst @@ -703,37 +703,62 @@ Get a list of features and services supported by the server. The following features MUST be reported by the server. Additional key-value pairs may be returned. - * **hosts** +* **hosts** - A dictionary of host names the server can be reached at. Each - value is a dictionary with keys "ssl_port" and "tcp_port" at which - the given host can be reached. If there is no open port for a - transport, its value should be *null*. + An dictionary, keyed by host name, that this server can be reached + at. Normally this will only have a single entry; other entries can + be used in case there are other connection routes (e.g. Tor). - * **server_version** + The value for a host is itself a dictionary, with the following + optional keys: - The same identifying string as returned in response to *server.version*. + * **ssl_port** - * **protocol_version** + An integer. Omit or set to *null* if SSL connectivity is not + provided. - A pair [`protocol_min`, `protocol_max`] of the protocols supported - by the server, each of which is itself a [major_version, - minor_version] pair. + * **tcp_port** - * **pruning** + An integer. Omit or set to *null* if TCP connectivity is not + provided. - The history pruning limit of the server as an integer. If the - server does not prune return *null*. + A server should ignore information provided about any host other + than the one it connected to. + +* **genesis_hash** + + The hash of the genesis block. This is used to detect if a peer is + connected to one serving a different network. + +* **server_version** + + A string that identifies the server software. Should be the same as + the response to **server.version** RPC call. + +* **protocol_max** +* **protocol_min** + + Strings that are the minimum and maximum Electrum protcol versions + this server speaks. The maximum value should be the same as what + would suffix the letter **v** in the IRC real name. Example: "1.1". + +* **pruning** + + An integer, the pruning limit. Omit or set to *null* if there is no + pruning limit. Should be the same as what would suffix the letter + **p** in the IRC real name. **Example Response** :: { - "server_version": "ElectrumX 0.10.14", - "protocol_version": [[1, 0], [1, 1]], - "hosts": {"14.3.140.101": {"ssl_port": 50002, "tcp_port": 50001}}, - "pruning": null + "genesis_hash": "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943", + "hosts": {"14.3.140.101": {"tcp_port": 51001, "ssl_port": 51002}}, + "protocol_max": "1.0", + "protocol_min": "1.0", + "pruning": null, + "server_version": "ElectrumX 1.0.1" } .. _JSON RPC 1.0: http://json-rpc.org/wiki/specification From 8236aaf2344c54ee8dba17c559aaff6e1b2086cb Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 25 Mar 2017 12:26:17 +0900 Subject: [PATCH 025/117] Be more strict on form of features dictionary --- lib/peer.py | 13 +++++++++---- server/controller.py | 4 ++-- server/peers.py | 7 ++++++- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/peer.py b/lib/peer.py index 1e3c630..5f94e03 100644 --- a/lib/peer.py +++ b/lib/peer.py @@ -49,6 +49,7 @@ class Peer(object): a dictionary of features, and a record of the source.''' assert isinstance(host, str) assert isinstance(features, dict) + assert host in features.get('hosts', {}) self.host = host self.features = features.copy() # Canonicalize / clean-up @@ -105,10 +106,14 @@ class Peer(object): def update_features(self, features): '''Update features in-place.''' - tmp = Peer(self.host, features) - self.features = tmp.features - for feature in self.FEATURES: - setattr(self, feature, getattr(tmp, feature)) + try: + tmp = Peer(self.host, features) + except Exception: + pass + else: + self.features = tmp.features + for feature in self.FEATURES: + setattr(self, feature, getattr(tmp, feature)) def connection_port_pairs(self): '''Return a list of (kind, port) pairs to try when making a diff --git a/server/controller.py b/server/controller.py index 81d663e..3bd8264 100644 --- a/server/controller.py +++ b/server/controller.py @@ -508,8 +508,8 @@ class Controller(util.LoggedClass): host = features['hosts'][hostname] yield fmt.format(hostname[:30], item['status'], - host['tcp_port'] or '', - host['ssl_port'] or '', + host.get('tcp_port') or '', + host.get('ssl_port') or '', features['server_version'] or 'unknown', features['protocol_min'], features['protocol_max'], diff --git a/server/peers.py b/server/peers.py index 4525765..e78845a 100644 --- a/server/peers.py +++ b/server/peers.py @@ -400,7 +400,12 @@ class PeerManager(util.LoggedClass): if data: version, items = ast.literal_eval(data) if version == 1: - peers = [Peer.deserialize(item) for item in items] + peers = [] + for item in items: + try: + peers.append(Peer.deserialize(item)) + except Exception: + pass self.add_peers(peers, source='peers file', limit=None) def import_peers(self): From f04ff6f5b317bd209d26925cb80a7b5f81c44226 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 25 Mar 2017 12:41:02 +0900 Subject: [PATCH 026/117] Don't add_peer to ourself --- server/peers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/peers.py b/server/peers.py index e78845a..e839aac 100644 --- a/server/peers.py +++ b/server/peers.py @@ -113,10 +113,12 @@ class PeerSession(JSONSession): self.peer_mgr.add_peers(peers) - # Announce ourself if not present. Don't if disabled or we - # are a non-public IP address. + # Announce ourself if not present. Don't if disabled, we + # are a non-public IP address, or to ourselves. if not self.peer_mgr.env.peer_announce: return + if self.peer in self.peer_mgr.myselves: + return my = self.peer_mgr.my_clearnet_peer() if not my.is_public: return From e4947cb9ef56a9d98985237759b3b3687b0ead20 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 25 Mar 2017 12:54:35 +0900 Subject: [PATCH 027/117] Sanitize the maximum number of sessions Reduce the maximum number of sessions to permit if running with a tight rlimit, to avoid hitting open file limits. Log when doing so. Closes #158 --- server/env.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/server/env.py b/server/env.py index bbba097..dbbc78d 100644 --- a/server/env.py +++ b/server/env.py @@ -8,6 +8,7 @@ '''Class for handling environment configuration and defaults.''' +import resource from collections import namedtuple from ipaddress import ip_address from os import environ @@ -60,7 +61,7 @@ class Env(LoggedClass): # Server limits to help prevent DoS self.max_send = self.integer('MAX_SEND', 1000000) self.max_subs = self.integer('MAX_SUBS', 250000) - self.max_sessions = self.integer('MAX_SESSIONS', 1000) + self.max_sessions = self.sane_max_sessions() self.max_session_subs = self.integer('MAX_SESSION_SUBS', 50000) self.bandwidth_limit = self.integer('BANDWIDTH_LIMIT', 2000000) self.session_timeout = self.integer('SESSION_TIMEOUT', 600) @@ -118,6 +119,20 @@ class Env(LoggedClass): raise self.Error('cannot convert envvar {} value {} to an integer' .format(envvar, value)) + def sane_max_sessions(self): + '''Return the maximum number of sessions to permit. Normally this + is MAX_SESSIONS. However, to prevent open file exhaustion, ajdust + downwards if running with a small open file rlimit.''' + env_value = self.integer('MAX_SESSIONS', 1000) + nofile_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0] + # We give the DB 250 files; allow ElectrumX 100 for itself + value = max(0, min(env_value, nofile_limit - 350)) + if value < env_value: + self.log_warning('lowered maximum seessions from {:,d} to {:,d} ' + 'because your open file limit is {:,d}' + .format(env_value, value, nofile_limit)) + return value + def check_report_host(self, host): try: ip = ip_address(host) From 7b263c5c49437a8775d81391c42c2fd2486f1f2f Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 25 Mar 2017 13:03:26 +0900 Subject: [PATCH 028/117] Prepare 1.0.2 --- README.rst | 62 +++++++++++++++-------------------------------- server/version.py | 2 +- 2 files changed, 20 insertions(+), 44 deletions(-) diff --git a/README.rst b/README.rst index 3a71dd0..701dfe7 100644 --- a/README.rst +++ b/README.rst @@ -127,6 +127,20 @@ Roadmap ChangeLog ========= +Version 1.0.2 +------------- + +* stricter acceptance of add_peer requests: rate-limit onion peers, + and require incoming requests to resolve to the requesting IP address +* validate peer hostnames (closes `#157`_) +* verify height for all peers (closes `#152`_) +* various improvements to peer handling +* various documentation tweaks +* limit the maximum number of sessions based on the process's + open file soft limit (closes `#158`_) +* improved altcoin support for variable-length block headers and AuxPoW + (erasmospunk) (closes `#128`_ and `#83`_) + Version 1.0.1 ------------- @@ -186,60 +200,22 @@ documentation updates. see `docs/ENVIRONMENT.rst`_. * add FairCoin (thokon00) -Version 0.11.4 --------------- - -* peer handling fixes / improvements based on suggestions of hsmiths - -Version 0.11.3 --------------- - -* fixed a typo in lib/peer.py pointed out by hsmiths - -Version 0.11.2 --------------- - -* Preliminary implementation of script hash subscriptions to enable - subscribing to updates of arbitrary scripts, not just those of - standard bitcoin addresses. I'll fully document once confirmed - working as expected. - Closes `#124`_. - -Version 0.11.1 --------------- - -* report unconfirmed parent tx status correctly, and notify if that - parent status changes. Fixes `#129`_. - -Version 0.11.0 --------------- - -* implementation of `docs/PEER_DISCOVERY.rst`_ for discovery of server - peers without using IRC. Closes `#104`_. Since all testnet peers - are ElectrumX servers, IRC advertising is now disabled on bitcoin - testnet. - - Thanks to bauerj, hsmiths and JWU42 for their help testing these - changes over the last month. -* you can now specify a tor proxy (or have it autodetected if local), - and if an incoming connection seems to be from the proxy a - tor-specific banner file is served. See **TOR_BANNER_FILE** in - `docs/ENVIRONMENT.rst`_. - **Neil Booth** kyuupichan@gmail.com https://github.com/kyuupichan 1BWwXJH3q6PRsizBkSGm2Uw4Sz1urZ5sCj +.. _#83: https://github.com/kyuupichan/electrumx/issues/83 .. _#100: https://github.com/kyuupichan/electrumx/issues/100 -.. _#104: https://github.com/kyuupichan/electrumx/issues/104 -.. _#124: https://github.com/kyuupichan/electrumx/issues/124 -.. _#129: https://github.com/kyuupichan/electrumx/issues/129 +.. _#128: https://github.com/kyuupichan/electrumx/issues/128 .. _#132: https://github.com/kyuupichan/electrumx/issues/132 .. _#135: https://github.com/kyuupichan/electrumx/issues/135 .. _#136: https://github.com/kyuupichan/electrumx/issues/136 .. _#138: https://github.com/kyuupichan/electrumx/issues/138 +.. _#152: https://github.com/kyuupichan/electrumx/issues/152 +.. _#157: https://github.com/kyuupichan/electrumx/issues/157 +.. _#158: https://github.com/kyuupichan/electrumx/issues/158 .. _docs/HOWTO.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/HOWTO.rst .. _docs/ENVIRONMENT.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/ENVIRONMENT.rst .. _docs/PEER_DISCOVERY.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/PEER_DISCOVERY.rst diff --git a/server/version.py b/server/version.py index e9146f2..26f48b6 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.1' +VERSION = 'ElectrumX 1.0.2' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From 32369891655dbdec6daf46bd2a3797ac66e7ca34 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 25 Mar 2017 20:22:38 +0900 Subject: [PATCH 029/117] Fix bad peer looping JWU42 pointed out an issue where peer discovery could get in a failure loop for bad peers; this fixes the the root cause and the immediate retries --- server/peers.py | 43 ++++++++++++++----------------------------- 1 file changed, 14 insertions(+), 29 deletions(-) diff --git a/server/peers.py b/server/peers.py index e839aac..943142b 100644 --- a/server/peers.py +++ b/server/peers.py @@ -56,7 +56,7 @@ class PeerSession(JSONSession): self.peer = peer self.peer_mgr = peer_mgr self.kind = kind - self.failed = False + self.bad = False self.log_prefix = '[{}] '.format(self.peer) async def wait_on_items(self): @@ -82,6 +82,7 @@ class PeerSession(JSONSession): [version.VERSION, proto_ver]) self.send_request(self.on_features, 'server.features') self.send_request(self.on_headers, 'blockchain.headers.subscribe') + self.send_request(self.on_peers_subscribe, 'server.peers.subscribe') def connection_lost(self, exc): '''Handle disconnection.''' @@ -91,7 +92,7 @@ class PeerSession(JSONSession): def on_peers_subscribe(self, result, error): '''Handle the response to the peers.subcribe message.''' if error: - self.failed = True + self.bad = True self.log_error('server.peers.subscribe: {}'.format(error)) else: self.check_remote_peers(result) @@ -133,54 +134,40 @@ class PeerSession(JSONSession): '''Handle the response to the add_peer message.''' self.close_if_done() - def peer_verified(self, is_good): - '''Call when it has been determined whether or not the peer seems to - be on the same network. - ''' - if is_good: - self.send_request(self.on_peers_subscribe, - 'server.peers.subscribe') - else: - self.peer.mark_bad() - self.failed = True - def on_features(self, features, error): # Several peers don't implement this. If they do, check they are # the same network with the genesis hash. - verified = False if not error and isinstance(features, dict): our_hash = self.peer_mgr.env.coin.GENESIS_HASH if our_hash != features.get('genesis_hash'): - self.peer_verified(False) + self.bad = True self.log_warning('incorrect genesis hash') else: - self.peer_verified(True) self.peer.update_features(features) - verified = True self.close_if_done() def on_headers(self, result, error): '''Handle the response to the version message.''' if error: - self.failed = True + self.bad = True self.log_error('blockchain.headers.subscribe returned an error') elif not isinstance(result, dict): + self.bad = True self.log_error('bad blockchain.headers.subscribe response') - self.peer_verified(False) else: our_height = self.peer_mgr.controller.bp.db_height their_height = result.get('block_height') is_good = (isinstance(their_height, int) and abs(our_height - their_height) <= 5) - self.peer_verified(is_good) if not is_good: + self.bad = True self.log_warning('bad height {}'.format(their_height)) self.close_if_done() def on_version(self, result, error): '''Handle the response to the version message.''' if error: - self.failed = True + self.bad = True self.log_error('server.version returned an error') elif isinstance(result, str): self.peer.server_version = result @@ -189,14 +176,15 @@ class PeerSession(JSONSession): def close_if_done(self): if not self.has_pending_requests(): - is_good = not self.failed - self.peer_mgr.set_connection_status(self.peer, is_good) + if self.bad: + peer.mark_bad() + self.peer_mgr.set_connection_status(self.peer, not self.bad) if self.peer.is_tor: how = 'via {} over Tor'.format(self.kind) else: how = 'via {} at {}'.format(self.kind, self.peer_addr(anon=False)) - status = 'verified' if is_good else 'failed to verify' + status = 'failed to verify' if self.bad else 'verified' elapsed = time.time() - self.peer.last_try self.log_info('{} {} in {:.1f}s'.format(status, how, elapsed)) self.close_connection() @@ -559,11 +547,8 @@ class PeerManager(util.LoggedClass): def maybe_forget_peer(self, peer): '''Forget the peer if appropriate, e.g. long-term unreachable.''' - if peer.bad: - forget = peer.last_connect < time.time() - STALE_SECS // 2 - else: - try_limit = 10 if peer.last_connect else 3 - forget = peer.try_count >= try_limit + try_limit = 10 if peer.last_connect else 3 + forget = peer.try_count >= try_limit if forget: desc = 'bad' if peer.bad else 'unreachable' From fa1a5bd3e4e4771f0e239ed58be93a16f69c44cf Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 25 Mar 2017 20:27:33 +0900 Subject: [PATCH 030/117] Prepare 1.0.3 --- README.rst | 5 +++++ server/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 701dfe7..056d544 100644 --- a/README.rst +++ b/README.rst @@ -127,6 +127,11 @@ Roadmap ChangeLog ========= +Version 1.0.3 +------------- + +* fix a verification loop that happened occasionally with bad peers + Version 1.0.2 ------------- diff --git a/server/version.py b/server/version.py index 26f48b6..a7f88f1 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.2' +VERSION = 'ElectrumX 1.0.3' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From 79bce5335e952d58d03911df94907af377de89c0 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 25 Mar 2017 20:52:26 +0900 Subject: [PATCH 031/117] Fix missing self --- server/peers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/peers.py b/server/peers.py index 943142b..2bfe546 100644 --- a/server/peers.py +++ b/server/peers.py @@ -177,7 +177,7 @@ class PeerSession(JSONSession): def close_if_done(self): if not self.has_pending_requests(): if self.bad: - peer.mark_bad() + self.peer.mark_bad() self.peer_mgr.set_connection_status(self.peer, not self.bad) if self.peer.is_tor: how = 'via {} over Tor'.format(self.kind) From 122d78d25b6c0ce8b6d207b61db54efc44133640 Mon Sep 17 00:00:00 2001 From: protonn Date: Sat, 25 Mar 2017 19:02:24 -0500 Subject: [PATCH 032/117] Argentum; support for AuxPow --- lib/coins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coins.py b/lib/coins.py index 8df7a20..101acdd 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -588,7 +588,7 @@ class DashTestnet(Dash): ] -class Argentum(Coin): +class Argentum(CoinAuxPow): NAME = "Argentum" SHORTNAME = "ARG" NET = "mainnet" From 2650459012efe859f2da7e6bbc27b912c96a3a7a Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 26 Mar 2017 11:36:19 +0900 Subject: [PATCH 033/117] Peer discovery fixes Change last_connect to mean last connection as its name implies, not last connection that wasn't bad. Keep bad peers around for 3 tries. Improve diagnostic --- server/peers.py | 98 +++++++++++++++++++++++++++---------------------- 1 file changed, 54 insertions(+), 44 deletions(-) diff --git a/server/peers.py b/server/peers.py index 2bfe546..0c70fc0 100644 --- a/server/peers.py +++ b/server/peers.py @@ -56,7 +56,9 @@ class PeerSession(JSONSession): self.peer = peer self.peer_mgr = peer_mgr self.kind = kind + self.failed = False self.bad = False + self.remote_peers = None self.log_prefix = '[{}] '.format(self.peer) async def wait_on_items(self): @@ -92,44 +94,12 @@ class PeerSession(JSONSession): def on_peers_subscribe(self, result, error): '''Handle the response to the peers.subcribe message.''' if error: - self.bad = True + self.failed = True self.log_error('server.peers.subscribe: {}'.format(error)) else: - self.check_remote_peers(result) + self.remote_peers = result self.close_if_done() - def check_remote_peers(self, updates): - '''When a peer gives us a peer update. - - Each update is expected to be of the form: - [ip_addr, hostname, ['v1.0', 't51001', 's51002']] - ''' - try: - real_names = [' '.join([u[1]] + u[2]) for u in updates] - peers = [Peer.from_real_name(real_name, str(self.peer)) - for real_name in real_names] - except Exception: - self.log_error('bad server.peers.subscribe response') - return - - self.peer_mgr.add_peers(peers) - - # Announce ourself if not present. Don't if disabled, we - # are a non-public IP address, or to ourselves. - if not self.peer_mgr.env.peer_announce: - return - if self.peer in self.peer_mgr.myselves: - return - my = self.peer_mgr.my_clearnet_peer() - if not my.is_public: - return - for peer in my.matches(peers): - if peer.tcp_port == my.tcp_port and peer.ssl_port == my.ssl_port: - return - - self.log_info('registering ourself with server.add_peer') - self.send_request(self.on_add_peer, 'server.add_peer', [my.features]) - def on_add_peer(self, result, error): '''Handle the response to the add_peer message.''' self.close_if_done() @@ -149,7 +119,7 @@ class PeerSession(JSONSession): def on_headers(self, result, error): '''Handle the response to the version message.''' if error: - self.bad = True + self.failed = True self.log_error('blockchain.headers.subscribe returned an error') elif not isinstance(result, dict): self.bad = True @@ -157,34 +127,72 @@ class PeerSession(JSONSession): else: our_height = self.peer_mgr.controller.bp.db_height their_height = result.get('block_height') - is_good = (isinstance(their_height, int) and - abs(our_height - their_height) <= 5) - if not is_good: + if not isinstance(their_height, int): + self.log_warning('invalid height {}'.format(their_height)) + self.bad = True + elif abs(our_height - their_height) > 5: + self.log_warning('bad height {:,d} (ours: {:,d})' + .format(their_height, our_height)) self.bad = True - self.log_warning('bad height {}'.format(their_height)) self.close_if_done() def on_version(self, result, error): '''Handle the response to the version message.''' if error: - self.bad = True + self.failed = True self.log_error('server.version returned an error') elif isinstance(result, str): self.peer.server_version = result self.peer.features['server_version'] = result self.close_if_done() + def check_remote_peers(self): + '''When a peer gives us a peer update. + + Each update is expected to be of the form: + [ip_addr, hostname, ['v1.0', 't51001', 's51002']] + ''' + try: + real_names = [' '.join([u[1]] + u[2]) for u in self.remote_peers] + peers = [Peer.from_real_name(real_name, str(self.peer)) + for real_name in real_names] + except Exception: + self.log_error('bad server.peers.subscribe response') + return + + self.peer_mgr.add_peers(peers) + + # Announce ourself if not present. Don't if disabled, we + # are a non-public IP address, or to ourselves. + if not self.peer_mgr.env.peer_announce: + return + if self.peer in self.peer_mgr.myselves: + return + my = self.peer_mgr.my_clearnet_peer() + if not my.is_public: + return + for peer in my.matches(peers): + if peer.tcp_port == my.tcp_port and peer.ssl_port == my.ssl_port: + return + + self.log_info('registering ourself with server.add_peer') + self.send_request(self.on_add_peer, 'server.add_peer', [my.features]) + def close_if_done(self): if not self.has_pending_requests(): if self.bad: self.peer.mark_bad() - self.peer_mgr.set_connection_status(self.peer, not self.bad) + elif self.remote_peers: + self.check_remote_peers() + self.peer.last_connect = time.time() + is_good = not (self.failed or self.bad) + self.peer_mgr.set_connection_status(self.peer, is_good) if self.peer.is_tor: how = 'via {} over Tor'.format(self.kind) else: how = 'via {} at {}'.format(self.kind, self.peer_addr(anon=False)) - status = 'failed to verify' if self.bad else 'verified' + status = 'verified' if is_good else 'failed to verify' elapsed = time.time() - self.peer.last_try self.log_info('{} {} in {:.1f}s'.format(status, how, elapsed)) self.close_connection() @@ -536,7 +544,6 @@ class PeerManager(util.LoggedClass): '''Called when a connection succeeded or failed.''' if good: peer.try_count = 0 - peer.last_connect = time.time() peer.source = 'peer' # Remove matching IP addresses for match in peer.matches(self.peers): @@ -547,7 +554,10 @@ class PeerManager(util.LoggedClass): def maybe_forget_peer(self, peer): '''Forget the peer if appropriate, e.g. long-term unreachable.''' - try_limit = 10 if peer.last_connect else 3 + if peer.last_connect and not peer.bad: + try_limit = 10 + else: + try_limit = 3 forget = peer.try_count >= try_limit if forget: From a88fc75610eca486b0a01edda4d696de24805d99 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 26 Mar 2017 11:49:57 +0900 Subject: [PATCH 034/117] Tweaks to coins.py --- lib/coins.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index 101acdd..afa29e8 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -295,8 +295,6 @@ class Coin(object): class CoinAuxPow(Coin): # Set NAME and NET to avoid exception in Coin::lookup_coin_class - NAME = "" - NET = "" STATIC_BLOCK_HEADERS = False @classmethod @@ -499,8 +497,6 @@ class NamecoinTestnet(Namecoin): 'a4cccff2a4767a8eee39c11db367b008') -# For DOGE there is disagreement across sites like bip32.org and -# pycoin. Taken from bip32.org and bitmerchant on github class Dogecoin(CoinAuxPow): NAME = "Dogecoin" SHORTNAME = "DOGE" From 860a4e8e93496fdbe69c83a2beb8abbcb9b68e8d Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 26 Mar 2017 11:51:18 +0900 Subject: [PATCH 035/117] Prepare 1.0.4 --- README.rst | 5 +++++ server/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 056d544..d2a9b68 100644 --- a/README.rst +++ b/README.rst @@ -127,6 +127,11 @@ Roadmap ChangeLog ========= +Version 1.0.4 +------------- + +* fix another unwanted loop in peer discovery, tweak diagnostics + Version 1.0.3 ------------- diff --git a/server/version.py b/server/version.py index a7f88f1..855964b 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.3' +VERSION = 'ElectrumX 1.0.4' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From 9620aa8bb6dc93cc7e926ad744a096c287273719 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 26 Mar 2017 12:29:04 +0900 Subject: [PATCH 036/117] Restore dummy NAME and NET --- lib/coins.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/coins.py b/lib/coins.py index afa29e8..330463c 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -295,6 +295,8 @@ class Coin(object): class CoinAuxPow(Coin): # Set NAME and NET to avoid exception in Coin::lookup_coin_class + NAME = '' + NET = '' STATIC_BLOCK_HEADERS = False @classmethod From 23a408c572c4feba0b9f9d660e6064e20fc34447 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Mon, 27 Mar 2017 13:04:59 +0900 Subject: [PATCH 037/117] More logging --- server/peers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/peers.py b/server/peers.py index 0c70fc0..75ef51d 100644 --- a/server/peers.py +++ b/server/peers.py @@ -313,10 +313,12 @@ class PeerManager(util.LoggedClass): async def on_add_peer(self, features, source_info): '''Add a peer (but only if the peer resolves to the source).''' if not source_info: + self.log_info('ignored add_peer request: no source info') return False source = source_info[0] peers = Peer.peers_from_features(features, source) if not peers: + self.log_info('ignored add_peer request: no peers given') return False # Just look at the first peer, require it From 8e00affc1a19522fe8fc245c1f3ed188159b05c8 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 28 Mar 2017 11:13:33 +0900 Subject: [PATCH 038/117] Fix the diagnostic looping in PeerSession Fixes #160 --- server/peers.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/server/peers.py b/server/peers.py index 75ef51d..abe5e9e 100644 --- a/server/peers.py +++ b/server/peers.py @@ -97,12 +97,14 @@ class PeerSession(JSONSession): self.failed = True self.log_error('server.peers.subscribe: {}'.format(error)) else: + # Save for later analysis self.remote_peers = result self.close_if_done() def on_add_peer(self, result, error): - '''Handle the response to the add_peer message.''' - self.close_if_done() + '''We got a response the add_peer message.''' + # This is the last thing we were waiting for; shutdown the connection + self.shutdown_connection() def on_features(self, features, error): # Several peers don't implement this. If they do, check they are @@ -147,10 +149,12 @@ class PeerSession(JSONSession): self.close_if_done() def check_remote_peers(self): - '''When a peer gives us a peer update. + '''Check the peers list we got from a remote peer. Each update is expected to be of the form: [ip_addr, hostname, ['v1.0', 't51001', 's51002']] + + Call add_peer if the remote doesn't appear to know about us. ''' try: real_names = [' '.join([u[1]] + u[2]) for u in self.remote_peers] @@ -184,18 +188,23 @@ class PeerSession(JSONSession): self.peer.mark_bad() elif self.remote_peers: self.check_remote_peers() - self.peer.last_connect = time.time() - is_good = not (self.failed or self.bad) - self.peer_mgr.set_connection_status(self.peer, is_good) - if self.peer.is_tor: - how = 'via {} over Tor'.format(self.kind) - else: - how = 'via {} at {}'.format(self.kind, - self.peer_addr(anon=False)) - status = 'verified' if is_good else 'failed to verify' - elapsed = time.time() - self.peer.last_try - self.log_info('{} {} in {:.1f}s'.format(status, how, elapsed)) - self.close_connection() + # We might now be waiting for an add_peer response + if not self.has_pending_requests(): + self.shutdown_connection() + + def shutdown_connection(self): + self.peer.last_connect = time.time() + is_good = not (self.failed or self.bad) + self.peer_mgr.set_connection_status(self.peer, is_good) + if self.peer.is_tor: + how = 'via {} over Tor'.format(self.kind) + else: + how = 'via {} at {}'.format(self.kind, + self.peer_addr(anon=False)) + status = 'verified' if is_good else 'failed to verify' + elapsed = time.time() - self.peer.last_try + self.log_info('{} {} in {:.1f}s'.format(status, how, elapsed)) + self.close_connection() class PeerManager(util.LoggedClass): From 594b66236ffaf878a315bc0d0cc47e716e9e042e Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 28 Mar 2017 12:30:06 +0900 Subject: [PATCH 039/117] Prepare 1.0.5 --- README.rst | 7 +++++++ server/env.py | 6 ++---- server/version.py | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index d2a9b68..0e144d6 100644 --- a/README.rst +++ b/README.rst @@ -127,6 +127,12 @@ Roadmap ChangeLog ========= +Version 1.0.5 +------------- + +* the peer looping was actually just looping of logging output, not + connections. Hopefully fixed for good in this release. Closes `#160`_. + Version 1.0.4 ------------- @@ -226,6 +232,7 @@ documentation updates. .. _#152: https://github.com/kyuupichan/electrumx/issues/152 .. _#157: https://github.com/kyuupichan/electrumx/issues/157 .. _#158: https://github.com/kyuupichan/electrumx/issues/158 +.. _#160: https://github.com/kyuupichan/electrumx/issues/160 .. _docs/HOWTO.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/HOWTO.rst .. _docs/ENVIRONMENT.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/ENVIRONMENT.rst .. _docs/PEER_DISCOVERY.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/PEER_DISCOVERY.rst diff --git a/server/env.py b/server/env.py index dbbc78d..e27d1c6 100644 --- a/server/env.py +++ b/server/env.py @@ -78,8 +78,6 @@ class Env(LoggedClass): self.integer('REPORT_SSL_PORT', self.ssl_port) or None, '' ) - if not main_identity.host.strip(): - raise self.Error('REPORT_HOST host is empty') if main_identity.tcp_port == main_identity.ssl_port: raise self.Error('REPORT_TCP_PORT and REPORT_SSL_PORT are both {}' .format(main_identity.tcp_port)) @@ -137,11 +135,11 @@ class Env(LoggedClass): try: ip = ip_address(host) except ValueError: - bad = not bool(host) + bad = not bool(host.strip()) else: bad = ip.is_multicast or ip.is_unspecified if bad: - raise self.Error('{} is not a valid REPORT_HOST'.format(host)) + raise self.Error('"{}" is not a valid REPORT_HOST'.format(host)) def obsolete(self, envvars): bad = [envvar for envvar in envvars if environ.get(envvar)] diff --git a/server/version.py b/server/version.py index 855964b..38899b1 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.4' +VERSION = 'ElectrumX 1.0.5' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From 9f12379181091b9ef66c9cb4694118d1d952fdcc Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 29 Mar 2017 17:01:40 +0900 Subject: [PATCH 040/117] Update notes about rocksdb in Python --- docs/HOWTO.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/HOWTO.rst b/docs/HOWTO.rst index dde2f81..94747c5 100644 --- a/docs/HOWTO.rst +++ b/docs/HOWTO.rst @@ -66,7 +66,8 @@ was much worse. You will need to install one of: + `plyvel `_ for LevelDB -+ `pyrocksdb `_ for RocksDB ++ `python-rocksdb `_ for RocksDB (`pip3 install python-rocksdb`) ++ `pyrocksdb `_ for an unmaintained version that doesn't work with recent releases of RocksDB Running ======= @@ -210,10 +211,10 @@ to update Python 3.5.2 to 3.5.3. See `contrib/python3.6/python-3.6.sh`_. Installing on Raspberry Pi 3 ---------------------------- -To install on the Raspberry Pi 3 you will need to update to the "stretch" distribution. +To install on the Raspberry Pi 3 you will need to update to the "stretch" distribution. See the full procedure in `contrib/raspberrypi3/install_electrumx.sh`_. -See also `contrib/raspberrypi3/run_electrumx.sh`_ for an easy way to configure and +See also `contrib/raspberrypi3/run_electrumx.sh`_ for an easy way to configure and launch electrumx. @@ -405,4 +406,3 @@ copy of your certificate and key in case you need to restore them. .. _`contrib/python3.6/python-3.6.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/contrib/python3.6/python-3.6.sh .. _`contrib/raspberrypi3/install_electrumx.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/contrib/raspberrypi3/install_electrumx.sh .. _`contrib/raspberrypi3/run_electrumx.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/contrib/raspberrypi3/run_electrumx.sh - From 7e8141c62dfd87e629f1ef2431c53e16cbb32a4e Mon Sep 17 00:00:00 2001 From: romanornr Date: Fri, 31 Mar 2017 00:37:43 +0200 Subject: [PATCH 041/117] Add Viacoin - Segwit Auxpow --- lib/coins.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/lib/coins.py b/lib/coins.py index 330463c..d99230b 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -467,6 +467,57 @@ class LitecoinTestnetSegWit(LitecoinTestnet): return DeserializerSegWit +class Viacoin(CoinAuxPow): + NAME="Viacoin" + SHORTNAME = "VIA" + NET = "mainnet" + XPUB_VERBYTES = bytes.fromhex("0488B21E") + XPRV_VERBYTES = bytes.fromhex("0488ADE4") + P2PKH_VERBYTE = bytes.fromhex("47") + P2SH_VERBYTE = bytes.fromhex("21") + WIF_BYTE = bytes.fromhex("c7") + GENESIS_HASH = ('4e9b54001f9976049830128ec0331515' + 'eaabe35a70970d79971da1539a400ba1') + TX_COUNT = 113638 + TX_COUNT_HEIGHT = 3473674 + TX_PER_BLOCK = 30 + IRC_PREFIX = "E_" + IRC_CHANNEL="#vialectrum" + RPC_PORT = 5222 + REORG_LIMIT = 5000 + PEERS = [ + 'vialectrum.bitops.me s t', + 'server.vialectrum.org s t', + 'vialectrum.viacoin.net s t', + 'viax1.bitops.me s t', + ] + + +class ViacoinTestnet(Viacoin): + SHORTNAME = "TVI" + NET = "testnet" + XPUB_VERBYTES = bytes.fromhex("043587CF") + XPRV_VERBYTES = bytes.fromhex("04358394") + P2PKH_VERBYTE = bytes.fromhex("7f") + P2SH_VERBYTE = bytes.fromhex("c4") + WIF_BYTE = bytes.fromhex("ff") + GENESIS_HASH = ('00000007199508e34a9ff81e6ec0c477' + 'a4cccff2a4767a8eee39c11db367b008') + RPC_PORT = 25222 + REORG_LIMIT = 2500 + PEER_DEFAULT_PORTS = {'t': '51001', 's': '51002'} + PEERS = [ + 'vialectrum.bysh.me s t', + ] + +class ViacoinTestnetSegWit(ViacoinTestnet): + NET = "testnet-segwit" + + @classmethod + def deserializer(cls): + return DeserializerSegWit + + # Source: namecoin.org class Namecoin(CoinAuxPow): NAME = "Namecoin" From 55cedfea9c53a40fc89450ae5d4070cae4edd562 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 1 Apr 2017 10:38:03 +0900 Subject: [PATCH 042/117] Have Daemon work with aiohttp 1 and 2 Fixes #163 --- server/daemon.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/server/daemon.py b/server/daemon.py index dccca40..39109cc 100644 --- a/server/daemon.py +++ b/server/daemon.py @@ -41,6 +41,13 @@ class Daemon(util.LoggedClass): self.workqueue_semaphore = asyncio.Semaphore(value=10) self.down = False self.last_error_time = 0 + # assignment of asyncio.TimeoutError are essentially ignored + if aiohttp.__version__.startswith('1.'): + self.ClientHttpProcessingError = aiohttp.ClientHttpProcessingError + self.ClientPayloadError = asyncio.TimeoutError + else: + self.ClientHttpProcessingError = asyncio.TimeoutError + self.ClientPayloadError = aiohttp.ClientPayloadError def set_urls(self, urls): '''Set the URLS to the given list, and switch to the first one.''' @@ -114,10 +121,12 @@ class Daemon(util.LoggedClass): .format(result[0], result[1])) except asyncio.TimeoutError: log_error('timeout error.') - except aiohttp.ClientHttpProcessingError: - log_error('HTTP error.') except aiohttp.ServerDisconnectedError: log_error('disconnected.') + except self.ClientHttpProcessingError: + log_error('HTTP error.') + except self.ClientPayloadError: + log_error('payload encoding error.') except aiohttp.ClientConnectionError: log_error('connection problem - is your daemon running?') except self.DaemonWarmingUpError: From abba36ac6c3c56a235f9513eb75464c5abb1a585 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 1 Apr 2017 11:17:57 +0900 Subject: [PATCH 043/117] Relax the get_chunk restriction based on client Closes #162 --- server/session.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/server/session.py b/server/session.py index b7e4ddb..5236001 100644 --- a/server/session.py +++ b/server/session.py @@ -34,6 +34,7 @@ class SessionBase(JSONSession): self.env = controller.env self.daemon = self.bp.daemon self.client = 'unknown' + self.client_version = (1) self.protocol_version = '1.0' self.anon_logs = self.env.anon_logs self.last_delay = 0 @@ -262,14 +263,15 @@ class ElectrumX(SessionBase): index: the chunk index''' index = self.controller.non_negative_integer(index) - self.chunk_indices.append(index) - self.chunk_indices = self.chunk_indices[-5:] - # -2 allows backing up a single chunk but no more. - if index <= max(self.chunk_indices[:-2], default=-1): - msg = ('chunk indices not advancing (wrong network?): {}' - .format(self.chunk_indices)) - # INVALID_REQUEST triggers a disconnect - raise RPCError(msg, JSONRPC.INVALID_REQUEST) + if self.client_version < (2, 8, 3): + self.chunk_indices.append(index) + self.chunk_indices = self.chunk_indices[-5:] + # -2 allows backing up a single chunk but no more. + if index <= max(self.chunk_indices[:-2], default=-1): + msg = ('chunk indices not advancing (wrong network?): {}' + .format(self.chunk_indices)) + # use INVALID_REQUEST to trigger a disconnect + raise RPCError(msg, JSONRPC.INVALID_REQUEST) return self.controller.get_chunk(index) def is_tor(self): @@ -323,6 +325,11 @@ class ElectrumX(SessionBase): ''' if client_name: self.client = str(client_name)[:17] + try: + self.client_version = tuple(int(part) for part + in self.client.split('.')) + except Exception: + pass if protocol_version is not None: self.protocol_version = protocol_version return version.VERSION From 9abc1dc11efa14552dada601ac40ce207d2d3dd3 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 1 Apr 2017 11:59:54 +0900 Subject: [PATCH 044/117] REPORT_HOST no longer defaults to HOST Cleanup of identity handling. It is now possible to specify a Tor identity and no clearnet identity. --- docs/ENVIRONMENT.rst | 67 ++++++++++++++++++++-------------- server/env.py | 87 +++++++++++++++++++++++++++----------------- 2 files changed, 93 insertions(+), 61 deletions(-) diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index b7f3612..f6d5517 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -215,6 +215,7 @@ raise them. functioning Electrum clients by default will send pings roughly every 60 seconds. + Peer Discovery -------------- @@ -264,59 +265,69 @@ some of this. 9150 (Tor browser bundle) and 1080 (socks). -IRC ---- +Server Advertising +------------------ -Use the following environment variables if you want to advertise -connectivity on IRC: - -* **IRC** - - Set to anything non-empty to advertise on IRC - -* **IRC_NICK** - - The nick to use when connecting to IRC. The default is a hash of - **REPORT_HOST**. Either way a prefix will be prepended depending on - **COIN** and **NET**. +These environment variables affect how your server is advertised, both +by peer discovery (if enabled) and IRC (if enabled). * **REPORT_HOST** - The host to advertise. Defaults to **HOST**. + The clearnet host to advertise. If not set, no clearnet host is + advertised. * **REPORT_TCP_PORT** - The TCP port to advertise. Defaults to **TCP_PORT**. '0' disables - publishing the port. + The clearnet TCP port to advertise if **REPORT_HOST** is set. + Defaults to **TCP_PORT**. '0' disables publishing a TCP port. * **REPORT_SSL_PORT** - The SSL port to advertise. Defaults to **SSL_PORT**. '0' disables - publishing the port. + The clearnet SSL port to advertise if **REPORT_HOST** is set. + Defaults to **SSL_PORT**. '0' disables publishing an SSL port. * **REPORT_HOST_TOR** - The tor address to advertise; must end with `.onion`. If set, an - additional connection to IRC happens with '_tor' appended to - **IRC_NICK**. + If you wish run a Tor service, this is the Tor host name to + advertise and must end with `.onion`. * **REPORT_TCP_PORT_TOR** - The TCP port to advertise for Tor. Defaults to **REPORT_TCP_PORT**, - unless it is '0', otherwise **TCP_PORT**. '0' disables publishing - the port. + The Tor TCP port to advertise. The default is the clearnet + **REPORT_TCP_PORT**, unless disabled or it is '0', otherwise + **TCP_PORT**. '0' disables publishing a Tor TCP port. * **REPORT_SSL_PORT_TOR** - The SSL port to advertise for Tor. Defaults to **REPORT_SSL_PORT**, - unless it is '0', otherwise **SSL_PORT**. '0' disables publishing - the port. + The Tor SSL port to advertise. The default is the clearnet + **REPORT_SSL_PORT**, unless disabled or it is '0', otherwise + **SSL_PORT**. '0' disables publishing a Tor SSL port. **NOTE**: Certificate-Authority signed certificates don't work over Tor, so you should set **REPORT_SSL_PORT_TOR** to 0 if yours is not self-signed. +IRC +--- + +Use the following environment variables if you want to advertise +connectivity on IRC: + +* **IRC** + + Set to anything non-empty to advertise on IRC + +* **IRC_NICK** + + The nick to use when connecting to IRC. The default is a hash of + **REPORT_HOST**. Either way a prefix will be prepended depending on + **COIN** and **NET**. + + If **REPORT_HOST_TOR** is set, an additional connection to IRC + happens with '_tor' appended to **IRC_NICK**. + + Cache ----- diff --git a/server/env.py b/server/env.py index e27d1c6..a2bf7b0 100644 --- a/server/env.py +++ b/server/env.py @@ -70,33 +70,11 @@ class Env(LoggedClass): self.irc_nick = self.default('IRC_NICK', None) # Identities - report_host = self.default('REPORT_HOST', self.host) - self.check_report_host(report_host) - main_identity = NetIdentity( - report_host, - self.integer('REPORT_TCP_PORT', self.tcp_port) or None, - self.integer('REPORT_SSL_PORT', self.ssl_port) or None, - '' - ) - if main_identity.tcp_port == main_identity.ssl_port: - raise self.Error('REPORT_TCP_PORT and REPORT_SSL_PORT are both {}' - .format(main_identity.tcp_port)) - - self.identities = [main_identity] - tor_host = self.default('REPORT_HOST_TOR', '') - if tor_host.endswith('.onion'): - self.identities.append(NetIdentity( - tor_host, - self.integer('REPORT_TCP_PORT_TOR', - main_identity.tcp_port - if main_identity.tcp_port else - self.tcp_port) or None, - self.integer('REPORT_SSL_PORT_TOR', - main_identity.ssl_port - if main_identity.ssl_port else - self.ssl_port) or None, - '_tor', - )) + clearnet_identity = self.clearnet_identity() + tor_identity = self.tor_identity(clearnet_identity) + self.identities = [identity + for identity in (clearnet_identity, tor_identity) + if identity is not None] def default(self, envvar, default): return environ.get(envvar, default) @@ -131,7 +109,16 @@ class Env(LoggedClass): .format(env_value, value, nofile_limit)) return value - def check_report_host(self, host): + def obsolete(self, envvars): + bad = [envvar for envvar in envvars if environ.get(envvar)] + if bad: + raise self.Error('remove obsolete environment variables {}' + .format(bad)) + + def clearnet_identity(self): + host = self.default('REPORT_HOST', None) + if host is None: + return None try: ip = ip_address(host) except ValueError: @@ -140,9 +127,43 @@ class Env(LoggedClass): bad = ip.is_multicast or ip.is_unspecified if bad: raise self.Error('"{}" is not a valid REPORT_HOST'.format(host)) + tcp_port = self.integer('REPORT_TCP_PORT', self.tcp_port) or None + ssl_port = self.integer('REPORT_SSL_PORT', self.ssl_port) or None + if tcp_port == ssl_port: + raise self.Error('REPORT_TCP_PORT and REPORT_SSL_PORT ' + 'both resolve to {}'.format(tcp_port)) + return NetIdentity( + host, + tcp_port, + ssl_port, + '' + ) - def obsolete(self, envvars): - bad = [envvar for envvar in envvars if environ.get(envvar)] - if bad: - raise self.Error('remove obsolete environment variables {}' - .format(bad)) + def tor_identity(self, clearnet): + host = self.default('REPORT_HOST_TOR', None) + if host is None: + return None + if not tor_host.endswith('.onion'): + raise self.Error('tor host "{}" must end with ".onion"' + .format(host)) + + def port(port_kind): + '''Returns the clearnet identity port, if any and not zero, + otherwise the listening port.''' + result = 0 + if clearnet: + result = getattr(clearnet, port_kind) + return result or getattr(self, port_kind) + + tcp_port = self.integer('REPORT_TCP_PORT_TOR', port('tcp_port')) or None + ssl_port = self.integer('REPORT_SSL_PORT_TOR', port('ssl_port')) or None + if tcp_port == ssl_port: + raise self.Error('REPORT_TCP_PORT_TOR and REPORT_SSL_PORT_TOR ' + 'both resolve to {}'.format(tcp_port)) + + return NetIdentity( + host, + tcp_port, + ssl_port, + '_tor', + ) From d1894356d0c806eaa8a81b223067c496e7a0c966 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 1 Apr 2017 12:10:18 +0900 Subject: [PATCH 045/117] Prepare 1.0.6 --- README.rst | 14 ++++++++++++++ server/version.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 0e144d6..7761e23 100644 --- a/README.rst +++ b/README.rst @@ -127,6 +127,18 @@ Roadmap ChangeLog ========= +Version 1.0.6 +------------- + +* updated to handle incompatibilities between aiohttp 1.0 and 2.0. + ElexctrumX should work with either for now; I will drop support for + 1.0 in a few months. Fixes `#163`_. +* relax get_chunk restrictions for clients 1.8.3 and higher. Closes + `#162`_. +* **REPORT_HOST** no longer defaults to **HOST**. If not set, no + clearnet identity will be advertised. +* Add Viacoin support (romanornr) + Version 1.0.5 ------------- @@ -233,6 +245,8 @@ documentation updates. .. _#157: https://github.com/kyuupichan/electrumx/issues/157 .. _#158: https://github.com/kyuupichan/electrumx/issues/158 .. _#160: https://github.com/kyuupichan/electrumx/issues/160 +.. _#162: https://github.com/kyuupichan/electrumx/issues/162 +.. _#163: https://github.com/kyuupichan/electrumx/issues/163 .. _docs/HOWTO.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/HOWTO.rst .. _docs/ENVIRONMENT.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/ENVIRONMENT.rst .. _docs/PEER_DISCOVERY.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/PEER_DISCOVERY.rst diff --git a/server/version.py b/server/version.py index 38899b1..4243935 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.5' +VERSION = 'ElectrumX 1.0.6' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From 321315ace07f569263711de207b19a0926492962 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 1 Apr 2017 15:17:17 +0900 Subject: [PATCH 046/117] Fix typo --- server/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/env.py b/server/env.py index a2bf7b0..437ceba 100644 --- a/server/env.py +++ b/server/env.py @@ -143,7 +143,7 @@ class Env(LoggedClass): host = self.default('REPORT_HOST_TOR', None) if host is None: return None - if not tor_host.endswith('.onion'): + if not host.endswith('.onion'): raise self.Error('tor host "{}" must end with ".onion"' .format(host)) From 81e6577838067fc0cbe5813ed30df0f850da4a52 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 1 Apr 2017 22:55:16 +0900 Subject: [PATCH 047/117] Catch address resolution failure exceptions --- server/peers.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/server/peers.py b/server/peers.py index abe5e9e..91eee89 100644 --- a/server/peers.py +++ b/server/peers.py @@ -10,11 +10,11 @@ import ast import asyncio import random +import socket import ssl import time from collections import defaultdict, Counter from functools import partial -from socket import SOCK_STREAM from lib.jsonrpc import JSONSession from lib.peer import Peer @@ -337,9 +337,15 @@ class PeerManager(util.LoggedClass): permit = self.permit_new_onion_peer() reason = 'rate limiting' else: - infos = await self.loop.getaddrinfo(host, 80, type=SOCK_STREAM) - permit = any(source == info[-1][0] for info in infos) - reason = 'source-destination mismatch' + try: + infos = await self.loop.getaddrinfo(host, 80, + type=socket.SOCK_STREAM) + except socket.gaierror: + permit = False + reason = 'address resolution failure' + else: + permit = any(source == info[-1][0] for info in infos) + reason = 'source-destination mismatch' if permit: self.log_info('accepted add_peer request from {} for {}' From 178de6c396bea5b49d9c671d5c9f34a11926ff0d Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 2 Apr 2017 10:08:03 +0900 Subject: [PATCH 048/117] Tighten restrictions on HOST - private IP not allowed if intending for public use - localhost not allowed --- server/env.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/env.py b/server/env.py index 437ceba..64f9b2c 100644 --- a/server/env.py +++ b/server/env.py @@ -122,9 +122,10 @@ class Env(LoggedClass): try: ip = ip_address(host) except ValueError: - bad = not bool(host.strip()) + bad = host.lower().strip() in ('', 'localhost') else: - bad = ip.is_multicast or ip.is_unspecified + bad = (ip.is_multicast or ip.is_unspecified + or (ip.is_private and (self.irc or self.peer_announce))) if bad: raise self.Error('"{}" is not a valid REPORT_HOST'.format(host)) tcp_port = self.integer('REPORT_TCP_PORT', self.tcp_port) or None From 0aa9195fc5bf6768548b1f801141fdae7dc6e766 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 2 Apr 2017 10:57:04 +0900 Subject: [PATCH 049/117] Remove bad onion default peer --- lib/coins.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/coins.py b/lib/coins.py index d99230b..46a20ea 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -330,7 +330,6 @@ class Bitcoin(Coin): RPC_PORT = 8332 PEERS = [ 'btc.smsys.me s995', - 'ca6ulp2j2mpsft3y.onion s t', 'electrum.be s t', 'electrum.trouth.net p10000 s t', 'electrum.vom-stausee.de s t', From 77a441ad064e7d2396394c75b5d74d623582f214 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 2 Apr 2017 14:21:52 +0900 Subject: [PATCH 050/117] Improve proxy handling Have a background proxy detection loop; removes need to check specific peers at startup. Consider proxy down once attempts to use it fail 3 times in a row. Regularly attempt to rediscover a proxy if it is down. --- lib/socks.py | 117 +++++++++++++++++++++++++++++++++++------------- server/peers.py | 17 ++----- 2 files changed, 91 insertions(+), 43 deletions(-) diff --git a/lib/socks.py b/lib/socks.py index 86fddb6..d31a5d1 100644 --- a/lib/socks.py +++ b/lib/socks.py @@ -137,44 +137,101 @@ class Socks(util.LoggedClass): class SocksProxy(util.LoggedClass): def __init__(self, host, port, loop=None): - '''Host can be an IPv4 address, IPv6 address, or a host name.''' + '''Host can be an IPv4 address, IPv6 address, or a host name. + Port can be None, in which case one is auto-detected.''' super().__init__() + # Host and port of the proxy self.host = host - self.port = port + self.try_ports = [port, 9050, 9150, 1080] + self.errors = 0 self.ip_addr = None + self.lost_event = asyncio.Event() self.loop = loop or asyncio.get_event_loop() - - async def create_connection(self, protocol_factory, host, port, ssl=None): - '''All arguments are as to asyncio's create_connection method.''' - if self.port is None: - proxy_ports = [9050, 9150, 1080] - else: - proxy_ports = [self.port] - - for proxy_port in proxy_ports: - address = (self.host, proxy_port) - sock = socket.socket() - sock.setblocking(False) - try: - await self.loop.sock_connect(sock, address) - except OSError as e: - if proxy_port == proxy_ports[-1]: - raise - continue - + self.set_lost() + + async def auto_detect_loop(self): + '''Try to detect a proxy at regular intervals until one is found. + If one is found, do nothing until one is lost.''' + while True: + await self.lost_event.wait() + self.lost_event.clear() + tries = 0 + while True: + tries += 1 + log_failure = tries % 10 == 1 + await self.detect_proxy(log_failure=log_failure) + if self.is_up(): + break + await asyncio.sleep(600) + + def is_up(self): + '''Returns True if we have a good proxy.''' + return self.port is not None + + def set_lost(self): + '''Called when the proxy appears lost/down.''' + self.port = None + self.lost_event.set() + + async def connect_via_proxy(self, host, port, proxy_address=None): + '''Connect to a (host, port) pair via the proxy. Returns the + connected socket on success.''' + proxy_address = proxy_address or (self.host, self.port) + sock = socket.socket() + sock.setblocking(False) + try: + await self.loop.sock_connect(sock, proxy_address) socks = Socks(self.loop, sock, host, port) + await socks.handshake() + return sock + except Exception: + sock.close() + raise + + async def detect_proxy(self, host='www.google.com', port=80, + log_failure=True): + '''Attempt to detect a proxy by establishing a connection through it + to the given target host / port pair. + ''' + if self.is_up(): + return + + sock = None + for proxy_port in self.try_ports: + if proxy_port is None: + continue + paddress = (self.host, proxy_port) try: - await socks.handshake() - if self.port is None: - self.ip_addr = sock.getpeername()[0] - self.port = proxy_port - self.logger.info('detected proxy at {} ({})' - .format(util.address_string(address), - self.ip_addr)) + sock = await self.connect_via_proxy(host, port, paddress) break except Exception as e: - sock.close() - raise + if log_failure: + self.logger.info('failed to detect proxy at {}: {}' + .format(util.address_string(paddress), e)) + + # Failed all ports? + if sock is None: + return + + peername = sock.getpeername() + sock.close() + self.ip_addr = peername[0] + self.port = proxy_port + self.errors = 0 + self.logger.info('detected proxy at {} ({})' + .format(util.address_string(paddress), self.ip_addr)) + + async def create_connection(self, protocol_factory, host, port, ssl=None): + '''All arguments are as to asyncio's create_connection method.''' + try: + sock = await self.connect_via_proxy(host, port) + self.errors = 0 + except Exception: + self.errors += 1 + # If we have 3 consecutive errors, consider the proxy undetected + if self.errors == 3: + self.set_lost() + raise hostname = host if ssl else None return await self.loop.create_connection( diff --git a/server/peers.py b/server/peers.py index 91eee89..47786c6 100644 --- a/server/peers.py +++ b/server/peers.py @@ -230,7 +230,6 @@ class PeerManager(util.LoggedClass): self.peers = set() self.onion_peers = [] self.permit_onion_peer_time = time.time() - self.last_tor_retry_time = 0 self.tor_proxy = SocksProxy(env.tor_proxy_host, env.tor_proxy_port, loop=self.loop) self.import_peers() @@ -463,6 +462,7 @@ class PeerManager(util.LoggedClass): 2) Verifying connectivity of new peers. 3) Retrying old peers at regular intervals. ''' + self.ensure_future(self.tor_proxy.auto_detect_loop()) self.connect_to_irc() if not self.env.peer_discovery: self.logger.info('peer discovery is disabled') @@ -492,10 +492,6 @@ class PeerManager(util.LoggedClass): nearly_stale_time = (now - STALE_SECS) + WAKEUP_SECS * 2 def should_retry(peer): - # Try some Tor at startup to determine the proxy so we can - # serve the right banner file - if self.tor_proxy.port is None and self.is_coin_onion_peer(peer): - return True # Retry a peer whose ports might have updated if peer.other_port_pairs: return True @@ -507,14 +503,6 @@ class PeerManager(util.LoggedClass): peers = [peer for peer in self.peers if should_retry(peer)] - # If we don't have a tor proxy drop tor peers, but retry - # occasionally - if self.tor_proxy.port is None: - if now < self.last_tor_retry_time + 3600: - peers = [peer for peer in peers if not peer.is_tor] - elif any(peer.is_tor for peer in peers): - self.last_tor_retry_time = now - for peer in peers: peer.try_count += 1 pairs = peer.connection_port_pairs() @@ -529,6 +517,9 @@ class PeerManager(util.LoggedClass): sslc = ssl.SSLContext(ssl.PROTOCOL_TLS) if kind == 'SSL' else None if peer.is_tor: + # Don't attempt an onion connection if we don't have a tor proxy + if not self.tor_proxy.is_up(): + return create_connection = self.tor_proxy.create_connection else: create_connection = self.loop.create_connection From a94d320e5daeac936b38c553af357b5c54685225 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 2 Apr 2017 14:59:45 +0900 Subject: [PATCH 051/117] New feature: force peer discovery via proxy Set FORCE_PROXY to non-empty to force peer discovery to go through the proxy. See docs/ENVIRONMENT.rst Wait for an attempt at proxy discovery to be made before beginning peer discovery. --- docs/ENVIRONMENT.rst | 12 +++++++++++- lib/socks.py | 3 +++ server/env.py | 1 + server/peers.py | 44 +++++++++++++++++++++++++------------------- server/session.py | 4 ++-- 5 files changed, 42 insertions(+), 22 deletions(-) diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index f6d5517..5c63cd5 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -251,6 +251,15 @@ some of this. peer discovery if it notices it is not present in the peer's returned list. +* **FORCE_PROXY** + + By default peer discovery happens over the clear internet. Set this + to non-empty to force peer discovery to be done via the proxy. This + might be useful if you are running a Tor service exclusively and + wish to keep your IP address private. **NOTE**: in such a case you + should leave **IRC** unset as IRC connections are *always* over the + normal internet. + * **TOR_PROXY_HOST** The host where your Tor proxy is running. Defaults to *localhost*. @@ -316,7 +325,8 @@ connectivity on IRC: * **IRC** - Set to anything non-empty to advertise on IRC + Set to anything non-empty to advertise on IRC. ElectrumX connects + to IRC over the clear internet, always. * **IRC_NICK** diff --git a/lib/socks.py b/lib/socks.py index d31a5d1..fef9498 100644 --- a/lib/socks.py +++ b/lib/socks.py @@ -146,6 +146,7 @@ class SocksProxy(util.LoggedClass): self.errors = 0 self.ip_addr = None self.lost_event = asyncio.Event() + self.tried_event = asyncio.Event() self.loop = loop or asyncio.get_event_loop() self.set_lost() @@ -209,6 +210,8 @@ class SocksProxy(util.LoggedClass): self.logger.info('failed to detect proxy at {}: {}' .format(util.address_string(paddress), e)) + self.tried_event.set() + # Failed all ports? if sock is None: return diff --git a/server/env.py b/server/env.py index 64f9b2c..bef42c8 100644 --- a/server/env.py +++ b/server/env.py @@ -53,6 +53,7 @@ class Env(LoggedClass): # Peer discovery self.peer_discovery = bool(self.default('PEER_DISCOVERY', True)) self.peer_announce = bool(self.default('PEER_ANNOUNCE', True)) + self.force_proxy = bool(self.default('FORCE_PROXY', False)) self.tor_proxy_host = self.default('TOR_PROXY_HOST', 'localhost') self.tor_proxy_port = self.integer('TOR_PROXY_PORT', None) # The electrum client takes the empty string as unspecified diff --git a/server/peers.py b/server/peers.py index 47786c6..ac1ff33 100644 --- a/server/peers.py +++ b/server/peers.py @@ -195,15 +195,7 @@ class PeerSession(JSONSession): def shutdown_connection(self): self.peer.last_connect = time.time() is_good = not (self.failed or self.bad) - self.peer_mgr.set_connection_status(self.peer, is_good) - if self.peer.is_tor: - how = 'via {} over Tor'.format(self.kind) - else: - how = 'via {} at {}'.format(self.kind, - self.peer_addr(anon=False)) - status = 'verified' if is_good else 'failed to verify' - elapsed = time.time() - self.peer.last_try - self.log_info('{} {} in {:.1f}s'.format(status, how, elapsed)) + self.peer_mgr.set_verification_status(self.peer, self.kind, is_good) self.close_connection() @@ -230,8 +222,8 @@ class PeerManager(util.LoggedClass): self.peers = set() self.onion_peers = [] self.permit_onion_peer_time = time.time() - self.tor_proxy = SocksProxy(env.tor_proxy_host, env.tor_proxy_port, - loop=self.loop) + self.proxy = SocksProxy(env.tor_proxy_host, env.tor_proxy_port, + loop=self.loop) self.import_peers() def my_clearnet_peer(self): @@ -462,13 +454,19 @@ class PeerManager(util.LoggedClass): 2) Verifying connectivity of new peers. 3) Retrying old peers at regular intervals. ''' - self.ensure_future(self.tor_proxy.auto_detect_loop()) self.connect_to_irc() if not self.env.peer_discovery: self.logger.info('peer discovery is disabled') return + # Wait a few seconds after starting the proxy detection loop + # for proxy detection to succeed + self.ensure_future(self.proxy.auto_detect_loop()) + await self.proxy.tried_event.wait() + self.logger.info('beginning peer discovery') + self.logger.info('force use of proxy: {}'.format(self.env.force_proxy)) + try: while True: timeout = self.loop.call_later(WAKEUP_SECS, @@ -516,11 +514,11 @@ class PeerManager(util.LoggedClass): kind, port = port_pairs[0] sslc = ssl.SSLContext(ssl.PROTOCOL_TLS) if kind == 'SSL' else None - if peer.is_tor: - # Don't attempt an onion connection if we don't have a tor proxy - if not self.tor_proxy.is_up(): + if self.env.force_proxy or peer.is_tor: + # Only attempt a proxy connection if the proxy is up + if not self.proxy.is_up(): return - create_connection = self.tor_proxy.create_connection + create_connection = self.proxy.create_connection else: create_connection = self.loop.create_connection @@ -546,10 +544,18 @@ class PeerManager(util.LoggedClass): if port_pairs: self.retry_peer(peer, port_pairs) else: - self.set_connection_status(peer, False) + self.maybe_forget_peer(peer) + + def set_verification_status(self, peer, kind, good): + '''Called when a verification succeeded or failed.''' + if self.env.force_proxy or peer.is_tor: + how = 'via {} over Tor'.format(kind) + else: + how = 'via {} at {}'.format(kind, peer.ip_addr) + status = 'verified' if good else 'failed to verify' + elapsed = time.time() - peer.last_try + self.log_info('{} {} in {:.1f}s'.format(status, how, elapsed)) - def set_connection_status(self, peer, good): - '''Called when a connection succeeded or failed.''' if good: peer.try_count = 0 peer.source = 'peer' diff --git a/server/session.py b/server/session.py index 5236001..b184e41 100644 --- a/server/session.py +++ b/server/session.py @@ -277,9 +277,9 @@ class ElectrumX(SessionBase): def is_tor(self): '''Try to detect if the connection is to a tor hidden service we are running.''' - tor_proxy = self.controller.peer_mgr.tor_proxy + proxy = self.controller.peer_mgr.proxy peer_info = self.peer_info() - return peer_info and peer_info[0] == tor_proxy.ip_addr + return peer_info and peer_info[0] == proxy.ip_addr async def replaced_banner(self, banner): network_info = await self.controller.daemon_request('getnetworkinfo') From e96b8f042156697366951bda846d08a20e1e0712 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 2 Apr 2017 15:12:50 +0900 Subject: [PATCH 052/117] Prepare 1.0.7 --- README.rst | 15 +++++++++++++++ server/version.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 7761e23..52cecb4 100644 --- a/README.rst +++ b/README.rst @@ -127,6 +127,21 @@ Roadmap ChangeLog ========= +Version 1.0.7 +------------- + +Improvements to proxy handling and peer discovery + +* background async proxy detection loop. Removes responsibility for + proxy detection and maintenance from the peer manager. +* peer discovery waits for an initial proxy detection attempt to complete + before starting +* new feature: flag to force peer discovery to happen via the proxy. + This might be useful for someone exlusively running a Tor service + that doesn't want to reveal its IP address. See **FORCE_PROXY** in + `docs/ENVIRONMENT.rst`_ for details and caveats. +* other minor fixes and tweaks + Version 1.0.6 ------------- diff --git a/server/version.py b/server/version.py index 4243935..a99169f 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.6' +VERSION = 'ElectrumX 1.0.7' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From 2656fd78a4550a3fa3285a9d0ad4d2c7f83bbf06 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 2 Apr 2017 21:25:02 +0900 Subject: [PATCH 053/117] Clarify that we may not have a clearnet peer --- server/peers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/peers.py b/server/peers.py index ac1ff33..db19f32 100644 --- a/server/peers.py +++ b/server/peers.py @@ -173,7 +173,7 @@ class PeerSession(JSONSession): if self.peer in self.peer_mgr.myselves: return my = self.peer_mgr.my_clearnet_peer() - if not my.is_public: + if not my or not my.is_public: return for peer in my.matches(peers): if peer.tcp_port == my.tcp_port and peer.ssl_port == my.ssl_port: @@ -227,8 +227,9 @@ class PeerManager(util.LoggedClass): self.import_peers() def my_clearnet_peer(self): - '''Returns the clearnet peer representing this server.''' - return [peer for peer in self.myselves if not peer.is_tor][0] + '''Returns the clearnet peer representing this server, if any.''' + clearnet = [peer for peer in self.myselves if not peer.is_tor] + return clearnet[0] if clearnet else None def info(self): '''The number of peers.''' From 9f27ea875c3cc2746b80cd57f86a9eb008742a9a Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 2 Apr 2017 21:25:47 +0900 Subject: [PATCH 054/117] Fix peer replacement logic - drop the IP address peer - update the remaining peer with fresh info --- lib/peer.py | 18 ++++++++++++------ server/peers.py | 15 ++++++++++----- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/lib/peer.py b/lib/peer.py index 5f94e03..d011300 100644 --- a/lib/peer.py +++ b/lib/peer.py @@ -94,12 +94,14 @@ class Peer(object): return tuple(int(part) for part in vstr.split('.')) def matches(self, peers): - '''Return peers whose host matches the given peer's host or IP - address. This results in our favouring host names over IP - addresses. + '''Return peers whose host matches our hostname or IP address. + Additionally include all peers whose IP address matches our + hostname if that is an IP address. ''' candidates = (self.host.lower(), self.ip_addr) - return [peer for peer in peers if peer.host.lower() in candidates] + return [peer for peer in peers + if peer.host.lower() in candidates + or peer.ip_addr == self.host] def __str__(self): return self.host @@ -111,9 +113,13 @@ class Peer(object): except Exception: pass else: - self.features = tmp.features + self.update_features_from_peer(tmp) + + def update_features_from_peer(self, peer): + if peer != self: + self.features = peer.features for feature in self.FEATURES: - setattr(self, feature, getattr(tmp, feature)) + setattr(self, feature, getattr(peer, feature)) def connection_port_pairs(self): '''Return a list of (kind, port) pairs to try when making a diff --git a/server/peers.py b/server/peers.py index db19f32..8fb01a8 100644 --- a/server/peers.py +++ b/server/peers.py @@ -555,15 +555,20 @@ class PeerManager(util.LoggedClass): how = 'via {} at {}'.format(kind, peer.ip_addr) status = 'verified' if good else 'failed to verify' elapsed = time.time() - peer.last_try - self.log_info('{} {} in {:.1f}s'.format(status, how, elapsed)) + self.log_info('{} {} {} in {:.1f}s'.format(status, peer, how, elapsed)) if good: peer.try_count = 0 peer.source = 'peer' - # Remove matching IP addresses - for match in peer.matches(self.peers): - if match != peer and peer.host == peer.ip_addr: - self.peers.remove(match) + # At most 2 matches if we're a host name, potentially several if + # we're an IP address (several instances can share a NAT). + matches = peer.matches(self.peers) + for match in matches: + if match.ip_address: + if len(matches) > 1: + self.peers.remove(match) + elif peer.host in match.features['hosts']: + match.update_features_from_peer(peer) else: self.maybe_forget_peer(peer) From 7b17d99c5af2b8848e44a9a198dacfc459f4ce1c Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Mon, 3 Apr 2017 20:10:42 +0900 Subject: [PATCH 055/117] Put log on one line --- server/peers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/peers.py b/server/peers.py index 8fb01a8..5b85c92 100644 --- a/server/peers.py +++ b/server/peers.py @@ -465,8 +465,8 @@ class PeerManager(util.LoggedClass): self.ensure_future(self.proxy.auto_detect_loop()) await self.proxy.tried_event.wait() - self.logger.info('beginning peer discovery') - self.logger.info('force use of proxy: {}'.format(self.env.force_proxy)) + self.logger.info('beginning peer discovery; force use of proxy: {}' + .format(self.env.force_proxy)) try: while True: From e0a79c313c31a86b32517e58d5d85294cf74c570 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Mon, 3 Apr 2017 20:13:35 +0900 Subject: [PATCH 056/117] Prepare 1.0.8 --- README.rst | 11 +++++++++++ server/version.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 52cecb4..bc2596f 100644 --- a/README.rst +++ b/README.rst @@ -127,6 +127,17 @@ Roadmap ChangeLog ========= +Version 1.0.8 +------------- + +Minor peer-discovery tweaks: + +* I intended that if a host and its IP address were both registered as + peers, that the real hostname replace the IP address. That wasn't + working properly and is fixed now. +* 1.0.6 no longer required a clearnet identity but part of the peer + discovery logic assumed one existed. That is now fixed. + Version 1.0.7 ------------- diff --git a/server/version.py b/server/version.py index a99169f..97d5497 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.7' +VERSION = 'ElectrumX 1.0.8' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From 2c43e89b05182311608b5fa3993bffe62f4c449e Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Mon, 3 Apr 2017 21:35:35 +0900 Subject: [PATCH 057/117] Only set last_good if successfully verified Rename last_connect to last_good --- lib/peer.py | 10 +++++++--- server/controller.py | 4 ++-- server/peers.py | 21 ++++++++++++--------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/lib/peer.py b/lib/peer.py index d011300..83d6334 100644 --- a/lib/peer.py +++ b/lib/peer.py @@ -38,13 +38,13 @@ class Peer(object): ATTRS = ('host', 'features', # metadata 'source', 'ip_addr', 'good_ports', - 'last_connect', 'last_try', 'try_count') + 'last_good', 'last_try', 'try_count') FEATURES = ('pruning', 'server_version', 'protocol_min', 'protocol_max') # This should be set by the application DEFAULT_PORTS = {} def __init__(self, host, features, source='unknown', ip_addr=None, - good_ports=[], last_connect=0, last_try=0, try_count=0): + good_ports=[], last_good=0, last_try=0, try_count=0): '''Create a peer given a host name (or IP address as a string), a dictionary of features, and a record of the source.''' assert isinstance(host, str) @@ -59,7 +59,11 @@ class Peer(object): self.source = source self.ip_addr = ip_addr self.good_ports = good_ports.copy() - self.last_connect = last_connect + # last_good represents the last connection that was + # successful *and* successfully verified, at which point + # try_count is set to 0. Failure to connect or failure to + # verify increment the try_count. + self.last_good = last_good self.last_try = last_try self.try_count = try_count # Transient, non-persisted metadata diff --git a/server/controller.py b/server/controller.py index 3bd8264..783353a 100644 --- a/server/controller.py +++ b/server/controller.py @@ -500,7 +500,7 @@ class Controller(util.LoggedClass): fmt = ('{:<30} {:<6} {:>5} {:>5} {:<17} {:>3} ' '{:>3} {:>8} {:>11} {:>11} {:>5} {:>20} {:<15}') yield fmt.format('Host', 'Status', 'TCP', 'SSL', 'Server', 'Min', - 'Max', 'Pruning', 'Last Conn', 'Last Try', + 'Max', 'Pruning', 'Last Good', 'Last Try', 'Tries', 'Source', 'IP Address') for item in data: features = item['features'] @@ -514,7 +514,7 @@ class Controller(util.LoggedClass): features['protocol_min'], features['protocol_max'], features['pruning'] or '', - time_fmt(item['last_connect']), + time_fmt(item['last_good']), time_fmt(item['last_try']), item['try_count'], item['source'][:20], diff --git a/server/peers.py b/server/peers.py index 5b85c92..1e95956 100644 --- a/server/peers.py +++ b/server/peers.py @@ -193,7 +193,6 @@ class PeerSession(JSONSession): self.shutdown_connection() def shutdown_connection(self): - self.peer.last_connect = time.time() is_good = not (self.failed or self.bad) self.peer_mgr.set_verification_status(self.peer, self.kind, is_good) self.close_connection() @@ -249,9 +248,9 @@ class PeerManager(util.LoggedClass): for peer in self.peers: if peer.bad: peer.status = PEER_BAD - elif peer.last_connect > cutoff: + elif peer.last_good > cutoff: peer.status = PEER_GOOD - elif peer.last_connect: + elif peer.last_good: peer.status = PEER_STALE else: peer.status = PEER_NEVER @@ -267,7 +266,7 @@ class PeerManager(util.LoggedClass): return data def peer_key(peer): - return (peer.bad, -peer.last_connect) + return (peer.bad, -peer.last_good) return [peer_data(peer) for peer in sorted(self.peers, key=peer_key)] @@ -358,13 +357,13 @@ class PeerManager(util.LoggedClass): ''' cutoff = time.time() - STALE_SECS recent = [peer for peer in self.peers - if peer.last_connect > cutoff and + if peer.last_good > cutoff and not peer.bad and peer.is_public] onion_peers = [] # Always report ourselves if valid (even if not public) peers = set(myself for myself in self.myselves - if myself.last_connect > cutoff) + if myself.last_good > cutoff) # Bucket the clearnet peers and select up to two from each buckets = defaultdict(list) @@ -409,6 +408,8 @@ class PeerManager(util.LoggedClass): if version == 1: peers = [] for item in items: + if 'last_connect' in item: + item['last_good'] = item.pop('last_connect') try: peers.append(Peer.deserialize(item)) except Exception: @@ -496,7 +497,7 @@ class PeerManager(util.LoggedClass): return True # Retry a good connection if it is about to turn stale if peer.try_count == 0: - return peer.last_connect < nearly_stale_time + return peer.last_good < nearly_stale_time # Retry a failed connection if enough time has passed return peer.last_try < now - WAKEUP_SECS * 2 ** peer.try_count @@ -549,16 +550,18 @@ class PeerManager(util.LoggedClass): def set_verification_status(self, peer, kind, good): '''Called when a verification succeeded or failed.''' + now = time.time() if self.env.force_proxy or peer.is_tor: how = 'via {} over Tor'.format(kind) else: how = 'via {} at {}'.format(kind, peer.ip_addr) status = 'verified' if good else 'failed to verify' - elapsed = time.time() - peer.last_try + elapsed = now - peer.last_try self.log_info('{} {} {} in {:.1f}s'.format(status, peer, how, elapsed)) if good: peer.try_count = 0 + peer.last_good = now peer.source = 'peer' # At most 2 matches if we're a host name, potentially several if # we're an IP address (several instances can share a NAT). @@ -574,7 +577,7 @@ class PeerManager(util.LoggedClass): def maybe_forget_peer(self, peer): '''Forget the peer if appropriate, e.g. long-term unreachable.''' - if peer.last_connect and not peer.bad: + if peer.last_good and not peer.bad: try_limit = 10 else: try_limit = 3 From 30df09534fed44f503f65214736f231761172c32 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Mon, 3 Apr 2017 21:40:02 +0900 Subject: [PATCH 058/117] Bump to 1.0.8a --- server/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/version.py b/server/version.py index 97d5497..b3d315f 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.8' +VERSION = 'ElectrumX 1.0.8a' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From 9549158115b5eedb7730d44e781f4e7971f43669 Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Wed, 5 Apr 2017 14:38:34 +0300 Subject: [PATCH 059/117] Add Einsteinium support --- lib/coins.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/coins.py b/lib/coins.py index 46a20ea..5a1c5a1 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -794,3 +794,23 @@ class Zcash(Coin): @classmethod def deserializer(cls): return DeserializerZcash + + +class Einsteinium(Coin): + NAME = "Einsteinium" + SHORTNAME = "EMC2" + NET = "mainnet" + # TODO add correct values for XPUB, XPRIV + XPUB_VERBYTES = bytes.fromhex("0488b21e") + XPRV_VERBYTES = bytes.fromhex("0488ade4") + P2PKH_VERBYTE = bytes.fromhex("21") + P2SH_VERBYTE = bytes.fromhex("05") + WIF_BYTE = bytes.fromhex("a1") + GENESIS_HASH = ('4e56204bb7b8ac06f860ff1c845f03f9' + '84303b5b97eb7b42868f714611aed94b') + TX_COUNT = 2087559 + TX_COUNT_HEIGHT = 1358517 + TX_PER_BLOCK = 2 + IRC_PREFIX = "E_" + IRC_CHANNEL = "#electrum-emc2" + RPC_PORT = 41879 From cf08903d122279de5814b99f561ea701430be277 Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Wed, 5 Apr 2017 15:31:38 +0300 Subject: [PATCH 060/117] Fix shebang for contrib scripts --- contrib/daemontools/run | 2 +- contrib/python3.6/python-3.6.sh | 6 ++++-- contrib/raspberrypi3/install_electrumx.sh | 1 + contrib/raspberrypi3/run_electrumx.sh | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/contrib/daemontools/run b/contrib/daemontools/run index 5bafc69..ccb9d3c 100644 --- a/contrib/daemontools/run +++ b/contrib/daemontools/run @@ -1,3 +1,3 @@ -j#!/bin/sh +#!/bin/sh echo "Launching ElectrumX server..." exec 2>&1 envdir ./env /bin/sh -c 'envuidgid $USERNAME python3 $ELECTRUMX' diff --git a/contrib/python3.6/python-3.6.sh b/contrib/python3.6/python-3.6.sh index d83839d..9472ce7 100644 --- a/contrib/python3.6/python-3.6.sh +++ b/contrib/python3.6/python-3.6.sh @@ -1,5 +1,7 @@ -Installation of Python 3.6 --------------------------- +#!/bin/sh +########################### +#Installation of Python 3.6 +########################### sudo add-apt-repository ppa:jonathonf/python-3.6 sudo apt-get update && sudo apt-get install python3.6 python3.6-dev diff --git a/contrib/raspberrypi3/install_electrumx.sh b/contrib/raspberrypi3/install_electrumx.sh index db64146..5643784 100644 --- a/contrib/raspberrypi3/install_electrumx.sh +++ b/contrib/raspberrypi3/install_electrumx.sh @@ -1,3 +1,4 @@ +#!/bin/sh ################### # install electrumx ################### diff --git a/contrib/raspberrypi3/run_electrumx.sh b/contrib/raspberrypi3/run_electrumx.sh index e3d633f..7b4ade4 100644 --- a/contrib/raspberrypi3/run_electrumx.sh +++ b/contrib/raspberrypi3/run_electrumx.sh @@ -1,3 +1,4 @@ +#!/bin/sh ############### # run_electrumx ############### From f3de91180e7b72cfc56a65980dbda14d0ba8448f Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 4 Apr 2017 21:43:56 +0900 Subject: [PATCH 061/117] Add tests for server/env.py --- server/env.py | 17 ++- server/peers.py | 6 +- tests/server/test_env.py | 279 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 292 insertions(+), 10 deletions(-) create mode 100644 tests/server/test_env.py diff --git a/server/env.py b/server/env.py index bef42c8..600dc9d 100644 --- a/server/env.py +++ b/server/env.py @@ -29,14 +29,14 @@ class Env(LoggedClass): def __init__(self): super().__init__() self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK']) + self.db_dir = self.required('DB_DIRECTORY') + self.daemon_url = self.required('DAEMON_URL') coin_name = self.default('COIN', 'Bitcoin') network = self.default('NET', 'mainnet') self.coin = Coin.lookup_coin_class(coin_name, network) - self.db_dir = self.required('DB_DIRECTORY') self.cache_MB = self.integer('CACHE_MB', 1200) self.host = self.default('HOST', 'localhost') self.reorg_limit = self.integer('REORG_LIMIT', self.coin.REORG_LIMIT) - self.daemon_url = self.required('DAEMON_URL') # Server stuff self.tcp_port = self.integer('TCP_PORT', None) self.ssl_port = self.integer('SSL_PORT', None) @@ -48,12 +48,12 @@ class Env(LoggedClass): self.banner_file = self.default('BANNER_FILE', None) self.tor_banner_file = self.default('TOR_BANNER_FILE', self.banner_file) - self.anon_logs = self.default('ANON_LOGS', False) + self.anon_logs = self.boolean('ANON_LOGS', False) self.log_sessions = self.integer('LOG_SESSIONS', 3600) # Peer discovery - self.peer_discovery = bool(self.default('PEER_DISCOVERY', True)) - self.peer_announce = bool(self.default('PEER_ANNOUNCE', True)) - self.force_proxy = bool(self.default('FORCE_PROXY', False)) + self.peer_discovery = self.boolean('PEER_DISCOVERY', True) + self.peer_announce = self.boolean('PEER_ANNOUNCE', True) + self.force_proxy = self.boolean('FORCE_PROXY', False) self.tor_proxy_host = self.default('TOR_PROXY_HOST', 'localhost') self.tor_proxy_port = self.integer('TOR_PROXY_PORT', None) # The electrum client takes the empty string as unspecified @@ -67,7 +67,7 @@ class Env(LoggedClass): self.bandwidth_limit = self.integer('BANDWIDTH_LIMIT', 2000000) self.session_timeout = self.integer('SESSION_TIMEOUT', 600) # IRC - self.irc = self.default('IRC', False) + self.irc = self.boolean('IRC', False) self.irc_nick = self.default('IRC_NICK', None) # Identities @@ -80,6 +80,9 @@ class Env(LoggedClass): def default(self, envvar, default): return environ.get(envvar, default) + def boolean(self, envvar, default): + return bool(self.default(envvar, default)) + def required(self, envvar): value = environ.get(envvar) if value is None: diff --git a/server/peers.py b/server/peers.py index 1e95956..46c34f2 100644 --- a/server/peers.py +++ b/server/peers.py @@ -294,9 +294,9 @@ class PeerManager(util.LoggedClass): use_peers = new_peers[:limit] else: use_peers = new_peers - self.logger.info('accepted {:d}/{:d} new peers of {:d} from {}' - .format(len(use_peers), len(new_peers), - len(peers), source)) + for n, peer in enumerate(use_peers): + self.logger.info('accepted new peer {:d}/{:d} {} from {} ' + .format(n + 1, len(use_peers), peer, source)) self.peers.update(use_peers) if retry: diff --git a/tests/server/test_env.py b/tests/server/test_env.py new file mode 100644 index 0000000..088fa54 --- /dev/null +++ b/tests/server/test_env.py @@ -0,0 +1,279 @@ +# Tests of server/env.py + +import os +import random + +import pytest + +from server.env import Env, NetIdentity +import lib.coins as lib_coins + + +BASE_DAEMON_URL = 'http://username:password@hostname:321/' +BASE_DB_DIR = '/some/dir' + +base_environ = { + 'DB_DIRECTORY': BASE_DB_DIR, + 'DAEMON_URL': BASE_DAEMON_URL, +} + +def setup_base_env(): + os.environ.clear() + os.environ.update(base_environ) + +def assert_required(env_var): + setup_base_env() + os.environ.pop(env_var, None) + with pytest.raises(Env.Error): + Env() + +def assert_default(env_var, attr, default): + setup_base_env() + e = Env() + assert getattr(e, attr) == default + os.environ[env_var] = 'foo' + e = Env() + assert getattr(e, attr) == 'foo' + +def assert_integer(env_var, attr, default=''): + if default != '': + e = Env() + assert getattr(e, attr) == default + value = random.randrange(5, 2000) + os.environ[env_var] = str(value) + '.1' + with pytest.raises(Env.Error): + Env() + os.environ[env_var] = str(value) + e = Env() + assert getattr(e, attr) == value + +def assert_boolean(env_var, attr, default): + e = Env() + assert getattr(e, attr) == default + os.environ[env_var] = 'foo' + e = Env() + assert getattr(e, attr) == True + os.environ[env_var] = '' + e = Env() + assert getattr(e, attr) == False + +def test_minimal(): + setup_base_env() + Env() + +def test_DB_DIRECTORY(): + assert_required('DB_DIRECTORY') + setup_base_env() + e = Env() + assert e.db_dir == BASE_DB_DIR + +def test_DAEMON_URL(): + assert_required('DAEMON_URL') + setup_base_env() + e = Env() + assert e.daemon_url == BASE_DAEMON_URL + +def test_COIN_NET(): + '''Test COIN and NET defaults and redirection.''' + setup_base_env() + e = Env() + assert e.coin == lib_coins.Bitcoin + os.environ['NET'] = 'testnet' + e = Env() + assert e.coin == lib_coins.BitcoinTestnet + os.environ.pop('NET') + os.environ['COIN'] = 'Litecoin' + e = Env() + assert e.coin == lib_coins.Litecoin + os.environ['NET'] = 'testnet' + e = Env() + assert e.coin == lib_coins.LitecoinTestnet + +def test_CACHE_MB(): + assert_integer('CACHE_MB', 'cache_MB', 1200) + +def test_HOST(): + assert_default('HOST', 'host', 'localhost') + +def test_REORG_LIMIT(): + assert_integer('REORG_LIMIT', 'reorg_limit', lib_coins.Bitcoin.REORG_LIMIT) + +def test_TCP_PORT(): + assert_integer('TCP_PORT', 'tcp_port', None) + +def test_SSL_PORT(): + # Requires both SSL_CERTFILE and SSL_KEYFILE to be set + os.environ['SSL_PORT'] = '50002' + os.environ['SSL_CERTFILE'] = 'certfile' + with pytest.raises(Env.Error): + Env() + os.environ.pop('SSL_CERTFILE') + os.environ['SSL_KEYFILE'] = 'keyfile' + with pytest.raises(Env.Error): + Env() + os.environ['SSL_CERTFILE'] = 'certfile' + os.environ.pop('SSL_PORT') + assert_integer('SSL_PORT', 'ssl_port', None) + +def test_RPC_PORT(): + assert_integer('RPC_PORT', 'rpc_port', 8000) + +def test_MAX_SUBSCRIPTIONS(): + assert_integer('MAX_SUBSCRIPTIONS', 'max_subscriptions', 10000) + +def test_LOG_SESSIONS(): + assert_integer('LOG_SESSIONS', 'log_sessions', 3600) + +def test_DONATION_ADDRESS(): + assert_default('DONATION_ADDRESS', 'donation_address', '') + +def test_DB_ENGINE(): + assert_default('DB_ENGINE', 'db_engine', 'leveldb') + +def test_MAX_SEND(): + assert_integer('MAX_SEND', 'max_send', 1000000) + +def test_MAX_SUBS(): + assert_integer('MAX_SUBS', 'max_subs', 250000) + +def test_MAX_SESSION_SUBS(): + assert_integer('MAX_SESSION_SUBS', 'max_session_subs', 50000) + +def test_BANDWIDTH_LIMIT(): + assert_integer('BANDWIDTH_LIMIT', 'bandwidth_limit', 2000000) + +def test_SESSION_TIMEOUT(): + assert_integer('SESSION_TIMEOUT', 'session_timeout', 600) + +def test_BANNER_FILE(): + e = Env() + assert e.banner_file is None + assert e.tor_banner_file is None + os.environ['BANNER_FILE'] = 'banner_file' + e = Env() + assert e.banner_file == 'banner_file' + assert e.tor_banner_file == 'banner_file' + os.environ['TOR_BANNER_FILE'] = 'tor_banner_file' + e = Env() + assert e.banner_file == 'banner_file' + assert e.tor_banner_file == 'tor_banner_file' + +def test_ANON_LOGS(): + assert_boolean('ANON_LOGS', 'anon_logs', False) + +def test_PEER_DISCOVERY(): + assert_boolean('PEER_DISCOVERY', 'peer_discovery', True) + +def test_PEER_ANNOUNCE(): + assert_boolean('PEER_ANNOUNCE', 'peer_announce', True) + +def test_FORCE_PROXY(): + assert_boolean('FORCE_PROXY', 'force_proxy', False) + +def test_TOR_PROXY_HOST(): + assert_default('TOR_PROXY_HOST', 'tor_proxy_host', 'localhost') + +def test_TOR_PROXY_PORT(): + assert_integer('TOR_PROXY_PORT', 'tor_proxy_port', None) + +def test_IRC(): + assert_boolean('IRC', 'irc', False) + +def test_IRC_NICK(): + assert_default('IRC_NICK', 'irc_nick', None) + +def test_clearnet_identity(): + os.environ['REPORT_TCP_PORT'] = '456' + e = Env() + assert len(e.identities) == 0 + os.environ['REPORT_HOST'] = '8.8.8.8' + e = Env() + assert len(e.identities) == 1 + assert e.identities[0].host == '8.8.8.8' + os.environ['REPORT_HOST'] = 'localhost' + with pytest.raises(Env.Error): + Env() + os.environ['REPORT_HOST'] = '' + with pytest.raises(Env.Error): + Env() + os.environ['REPORT_HOST'] = '127.0.0.1' + with pytest.raises(Env.Error): + Env() + os.environ['REPORT_HOST'] = '0.0.0.0' + with pytest.raises(Env.Error): + Env() + os.environ['REPORT_HOST'] = '224.0.0.2' + with pytest.raises(Env.Error): + Env() + # Accept private IP, unless IRC or PEER_ANNOUNCE + os.environ.pop('IRC', None) + os.environ['PEER_ANNOUNCE'] = '' + os.environ['REPORT_HOST'] = '192.168.0.1' + os.environ['SSL_CERTFILE'] = 'certfile' + os.environ['SSL_KEYFILE'] = 'keyfile' + Env() + os.environ['IRC'] = 'OK' + with pytest.raises(Env.Error): + Env() + os.environ.pop('IRC', None) + os.environ['PEER_ANNOUNCE'] = 'OK' + with pytest.raises(Env.Error): + Env() + os.environ['REPORT_SSL_PORT'] = os.environ['REPORT_TCP_PORT'] + with pytest.raises(Env.Error): + Env() + os.environ['REPORT_SSL_PORT'] = '457' + os.environ['REPORT_HOST'] = 'foo.com' + e = Env() + assert len(e.identities) == 1 + ident = e.identities[0] + assert ident.host == 'foo.com' + assert ident.tcp_port == 456 + assert ident.ssl_port == 457 + assert ident.nick_suffix == '' + +def test_tor_identity(): + tor_host = 'something.onion' + os.environ.pop('REPORT_HOST', None) + os.environ.pop('REPORT_HOST_TOR', None) + e = Env() + assert len(e.identities) == 0 + os.environ['REPORT_HOST_TOR'] = 'foo' + os.environ['REPORT_SSL_PORT_TOR'] = '123' + os.environ['TCP_PORT'] = '456' + with pytest.raises(Env.Error): + Env() + os.environ['REPORT_HOST_TOR'] = tor_host + e = Env() + assert len(e.identities) == 1 + ident = e.identities[0] + assert ident.host == tor_host + assert ident.tcp_port == 456 + assert ident.ssl_port == 123 + assert ident.nick_suffix == '_tor' + os.environ['REPORT_TCP_PORT_TOR'] = os.environ['REPORT_SSL_PORT_TOR'] + with pytest.raises(Env.Error): + Env() + os.environ['REPORT_HOST'] = 'foo.com' + os.environ['TCP_PORT'] = '456' + os.environ['SSL_PORT'] = '789' + os.environ['REPORT_TCP_PORT'] = '654' + os.environ['REPORT_SSL_PORT'] = '987' + os.environ['SSL_CERTFILE'] = 'certfile' + os.environ['SSL_KEYFILE'] = 'keyfile' + os.environ.pop('REPORT_TCP_PORT_TOR', None) + os.environ.pop('REPORT_SSL_PORT_TOR', None) + e = Env() + assert len(e.identities) == 2 + ident = e.identities[1] + assert ident.host == tor_host + assert ident.tcp_port == 654 + assert ident.ssl_port == 987 + os.environ['REPORT_TCP_PORT_TOR'] = '234' + os.environ['REPORT_SSL_PORT_TOR'] = '432' + e = Env() + assert len(e.identities) == 2 + ident = e.identities[1] + assert ident.host == tor_host + assert ident.tcp_port == 234 + assert ident.ssl_port == 432 From fde47d57396c4d35dfe467e91215fbec5bb2f69a Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Thu, 6 Apr 2017 07:32:56 +0900 Subject: [PATCH 062/117] Organise tests --- .gitignore | 2 ++ tests/{ => lib}/test_addresses.py | 0 tests/{ => lib}/test_util.py | 0 tests/{ => server}/test_storage.py | 0 4 files changed, 2 insertions(+) rename tests/{ => lib}/test_addresses.py (100%) rename tests/{ => lib}/test_util.py (100%) rename tests/{ => server}/test_storage.py (100%) diff --git a/.gitignore b/.gitignore index 7bacdd8..d5727dd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ */__pycache__/ +.cache/ +tests/*/__pycache__/ */*~ *.#* *# diff --git a/tests/test_addresses.py b/tests/lib/test_addresses.py similarity index 100% rename from tests/test_addresses.py rename to tests/lib/test_addresses.py diff --git a/tests/test_util.py b/tests/lib/test_util.py similarity index 100% rename from tests/test_util.py rename to tests/lib/test_util.py diff --git a/tests/test_storage.py b/tests/server/test_storage.py similarity index 100% rename from tests/test_storage.py rename to tests/server/test_storage.py From c0ff2c0c20c1afd4ee0df55f972fd66d0ae69868 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Thu, 6 Apr 2017 07:36:04 +0900 Subject: [PATCH 063/117] Bump to 1.0.8b --- server/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/version.py b/server/version.py index b3d315f..f6327f9 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.8a' +VERSION = 'ElectrumX 1.0.8b' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From b6d8b86dd6fc12835cd40964650b88caa8cd79e8 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 9 Apr 2017 14:02:32 +0900 Subject: [PATCH 064/117] Ignore hosts not appearing in their own features --- server/peers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/peers.py b/server/peers.py index 46c34f2..9f5e651 100644 --- a/server/peers.py +++ b/server/peers.py @@ -114,8 +114,11 @@ class PeerSession(JSONSession): if our_hash != features.get('genesis_hash'): self.bad = True self.log_warning('incorrect genesis hash') - else: + elif self.peer.host in features.get('hosts', {}): self.peer.update_features(features) + else: + self.bad = True + self.log_warning('marking DNS alias bad') self.close_if_done() def on_headers(self, result, error): From 8a2821d542a13e5987eefb055d6f643dc58ea6c7 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 9 Apr 2017 14:15:10 +0900 Subject: [PATCH 065/117] Reject invalid hostnames in Env --- lib/util.py | 2 +- server/env.py | 7 ++++--- server/peers.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/util.py b/lib/util.py index e6fdd9a..78006b9 100644 --- a/lib/util.py +++ b/lib/util.py @@ -249,6 +249,6 @@ def is_valid_hostname(hostname): if len(hostname) > 255: return False # strip exactly one dot from the right, if present - if hostname[-1] == ".": + if hostname and hostname[-1] == ".": hostname = hostname[:-1] return all(SEGMENT_REGEX.match(x) for x in hostname.split(".")) diff --git a/server/env.py b/server/env.py index 600dc9d..c9f0b56 100644 --- a/server/env.py +++ b/server/env.py @@ -14,13 +14,13 @@ from ipaddress import ip_address from os import environ from lib.coins import Coin -from lib.util import LoggedClass +import lib.util as lib_util NetIdentity = namedtuple('NetIdentity', 'host tcp_port ssl_port nick_suffix') -class Env(LoggedClass): +class Env(lib_util.LoggedClass): '''Wraps environment configuration.''' class Error(Exception): @@ -126,7 +126,8 @@ class Env(LoggedClass): try: ip = ip_address(host) except ValueError: - bad = host.lower().strip() in ('', 'localhost') + bad = (not lib_util.is_valid_hostname(host) + or hostname.lower() == 'localhost') else: bad = (ip.is_multicast or ip.is_unspecified or (ip.is_private and (self.irc or self.peer_announce))) diff --git a/server/peers.py b/server/peers.py index 9f5e651..51d6674 100644 --- a/server/peers.py +++ b/server/peers.py @@ -118,7 +118,7 @@ class PeerSession(JSONSession): self.peer.update_features(features) else: self.bad = True - self.log_warning('marking DNS alias bad') + self.log_warning('ignoring - not listed in features') self.close_if_done() def on_headers(self, result, error): From d216d5111b0150610c9fa36f41b05ab17453e347 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 9 Apr 2017 14:19:27 +0900 Subject: [PATCH 066/117] Prepare 1.0.9 --- README.rst | 9 +++++++++ server/env.py | 2 +- server/version.py | 2 +- tests/server/test_env.py | 3 +++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index bc2596f..3809828 100644 --- a/README.rst +++ b/README.rst @@ -127,6 +127,15 @@ Roadmap ChangeLog ========= +Version 1.0.9 +------------- + +- ignore peers not appearing in their features list +- validate hostnames in Env object +- added tests for env.py +- Einsteinium support and contrib script shebang fix (erasmospunk) +- set last_good only if successfully verified + Version 1.0.8 ------------- diff --git a/server/env.py b/server/env.py index c9f0b56..94072ec 100644 --- a/server/env.py +++ b/server/env.py @@ -127,7 +127,7 @@ class Env(lib_util.LoggedClass): ip = ip_address(host) except ValueError: bad = (not lib_util.is_valid_hostname(host) - or hostname.lower() == 'localhost') + or host.lower() == 'localhost') else: bad = (ip.is_multicast or ip.is_unspecified or (ip.is_private and (self.irc or self.peer_announce))) diff --git a/server/version.py b/server/version.py index f6327f9..fc5a6f8 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.8b' +VERSION = 'ElectrumX 1.0.9' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' diff --git a/tests/server/test_env.py b/tests/server/test_env.py index 088fa54..4a7f944 100644 --- a/tests/server/test_env.py +++ b/tests/server/test_env.py @@ -203,6 +203,9 @@ def test_clearnet_identity(): with pytest.raises(Env.Error): Env() os.environ['REPORT_HOST'] = '224.0.0.2' + with pytest.raises(Env.Error): + Env() + os.environ['REPORT_HOST'] = '$HOST' with pytest.raises(Env.Error): Env() # Accept private IP, unless IRC or PEER_ANNOUNCE From e9acb685ab92d26396057248e74f8931b2eb691b Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 9 Apr 2017 14:28:15 +0900 Subject: [PATCH 067/117] Display hosts in diagnostic --- server/peers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/peers.py b/server/peers.py index 51d6674..554ed0f 100644 --- a/server/peers.py +++ b/server/peers.py @@ -110,15 +110,17 @@ class PeerSession(JSONSession): # Several peers don't implement this. If they do, check they are # the same network with the genesis hash. if not error and isinstance(features, dict): + hosts = [host.lower() for host in features.get('hosts', {})] our_hash = self.peer_mgr.env.coin.GENESIS_HASH if our_hash != features.get('genesis_hash'): self.bad = True self.log_warning('incorrect genesis hash') - elif self.peer.host in features.get('hosts', {}): + elif self.peer.host.lower() in hosts: self.peer.update_features(features) else: self.bad = True - self.log_warning('ignoring - not listed in features') + self.log_warning('ignoring - not listed in host list {}' + .format(hosts)) self.close_if_done() def on_headers(self, result, error): From 92584cc3c60dcff9ddf219d9be18a16bb86bd985 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Fri, 28 Apr 2017 20:06:00 +0900 Subject: [PATCH 068/117] Update Litecoin entries as Segwit has activated. --- lib/coins.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index 5a1c5a1..d4a0c24 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -420,8 +420,8 @@ class Litecoin(Coin): TX_COUNT = 8908766 TX_COUNT_HEIGHT = 1105256 TX_PER_BLOCK = 10 - IRC_PREFIX = "EL_" - IRC_CHANNEL = "#electrum-ltc" + IRC_CHANNEL = "#electrum-ltc" # obsolete + IRC_PREFIX = None RPC_PORT = 9332 REORG_LIMIT = 800 PEERS = [ @@ -431,14 +431,16 @@ class Litecoin(Coin): 'electrum.cryptomachine.com p1000 s t', 'electrum.ltc.xurious.com s t', 'eywr5eubdbbe2laq.onion s50008 t50007', - 'us11.einfachmalnettsein.de s50008 t50007', ] + @classmethod + def deserializer(cls): + return DeserializerSegWit + class LitecoinTestnet(Litecoin): SHORTNAME = "XLT" NET = "testnet" - IRC_PREFIX = None XPUB_VERBYTES = bytes.fromhex("043587cf") XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("6f") @@ -458,14 +460,6 @@ class LitecoinTestnet(Litecoin): ] -class LitecoinTestnetSegWit(LitecoinTestnet): - NET = "testnet-segwit" - - @classmethod - def deserializer(cls): - return DeserializerSegWit - - class Viacoin(CoinAuxPow): NAME="Viacoin" SHORTNAME = "VIA" From 9af037b4bec2f4bab7da79abc84e8051697ef784 Mon Sep 17 00:00:00 2001 From: Johann Bauer Date: Sun, 9 Apr 2017 21:48:24 +0200 Subject: [PATCH 069/117] Add installer to README --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 3809828..5509e36 100644 --- a/README.rst +++ b/README.rst @@ -17,6 +17,9 @@ Getting Started =============== See `docs/HOWTO.rst`_. +There is also an `installer`_ available that simplifies the installation on various Linux-based distributions. + +.. _installer: https://github.com/bauerj/electrumx-installer Features ======== From 2fe67932c5feb12ae68a2a7b9c3b81e301fdcf47 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Fri, 28 Apr 2017 22:44:48 +0900 Subject: [PATCH 070/117] Prepare 1.0.10 --- README.rst | 11 +++++++++++ server/version.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5509e36..fb7cac8 100644 --- a/README.rst +++ b/README.rst @@ -130,6 +130,17 @@ Roadmap ChangeLog ========= +Version 1.0.10 +-------------- + +- add bauerj's installer docs +- segwit has activated on Litecoin. Make segwit deserialization the + default. Also as the first Segwit block probably will break old + electrum-server implementation servers, disable IRC and make + Litecoin mainnet and testnet use the peer-discovery protocol. If + you previously used "testnet-segwit" as your NET you should instead + use "testnet". + Version 1.0.9 ------------- diff --git a/server/version.py b/server/version.py index fc5a6f8..cdab28f 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.9' +VERSION = 'ElectrumX 1.0.10' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From 961936245c38433e1bf3ae66ae44a6614e183e8a Mon Sep 17 00:00:00 2001 From: pooler Date: Sun, 30 Apr 2017 09:48:52 +0200 Subject: [PATCH 071/117] Allow multiple P2SH address versions --- lib/coins.py | 42 ++++++++++++++++++------------------- tests/lib/test_addresses.py | 4 ++-- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index d4a0c24..e81ffd4 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -170,7 +170,7 @@ class Coin(object): def P2SH_address_from_hash160(cls, hash160): '''Return a coin address given a hash160.''' assert len(hash160) == 20 - return Base58.encode_check(cls.P2SH_VERBYTE + hash160) + return Base58.encode_check(cls.P2SH_VERBYTES[0] + hash160) @classmethod def multisig_address(cls, m, pubkeys): @@ -213,7 +213,7 @@ class Coin(object): if verbyte == cls.P2PKH_VERBYTE: return ScriptPubKey.P2PKH_script(hash_bytes) - if verbyte == cls.P2SH_VERBYTE: + if verbyte in cls.P2SH_VERBYTES: return ScriptPubKey.P2SH_script(hash_bytes) raise CoinError('invalid address: {}'.format(address)) @@ -318,7 +318,7 @@ class Bitcoin(Coin): XPUB_VERBYTES = bytes.fromhex("0488b21e") XPRV_VERBYTES = bytes.fromhex("0488ade4") P2PKH_VERBYTE = bytes.fromhex("00") - P2SH_VERBYTE = bytes.fromhex("05") + P2SH_VERBYTES = [bytes.fromhex("05")] WIF_BYTE = bytes.fromhex("80") GENESIS_HASH = ('000000000019d6689c085ae165831e93' '4ff763ae46a2a6c172b3f1b60a8ce26f') @@ -353,7 +353,7 @@ class BitcoinTestnet(Bitcoin): XPUB_VERBYTES = bytes.fromhex("043587cf") XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("6f") - P2SH_VERBYTE = bytes.fromhex("c4") + P2SH_VERBYTES = [bytes.fromhex("c4")] WIF_BYTE = bytes.fromhex("ef") GENESIS_HASH = ('000000000933ea01ad0ee984209779ba' 'aec3ced90fa3f408719526f8d77f4943') @@ -413,7 +413,7 @@ class Litecoin(Coin): XPUB_VERBYTES = bytes.fromhex("0488b21e") XPRV_VERBYTES = bytes.fromhex("0488ade4") P2PKH_VERBYTE = bytes.fromhex("30") - P2SH_VERBYTE = bytes.fromhex("05") + P2SH_VERBYTES = [bytes.fromhex("32"), bytes.fromhex("05")] WIF_BYTE = bytes.fromhex("b0") GENESIS_HASH = ('12a765e31ffd4059bada1e25190f6e98' 'c99d9714d334efa41a195a7e7e04bfe2') @@ -444,7 +444,7 @@ class LitecoinTestnet(Litecoin): XPUB_VERBYTES = bytes.fromhex("043587cf") XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("6f") - P2SH_VERBYTE = bytes.fromhex("c4") + P2SH_VERBYTES = [bytes.fromhex("3a"), bytes.fromhex("c4")] WIF_BYTE = bytes.fromhex("ef") GENESIS_HASH = ('4966625a4b2851d9fdee139e56211a0d' '88575f59ed816ff5e6a63deb4e3e29a0') @@ -467,7 +467,7 @@ class Viacoin(CoinAuxPow): XPUB_VERBYTES = bytes.fromhex("0488B21E") XPRV_VERBYTES = bytes.fromhex("0488ADE4") P2PKH_VERBYTE = bytes.fromhex("47") - P2SH_VERBYTE = bytes.fromhex("21") + P2SH_VERBYTES = [bytes.fromhex("21")] WIF_BYTE = bytes.fromhex("c7") GENESIS_HASH = ('4e9b54001f9976049830128ec0331515' 'eaabe35a70970d79971da1539a400ba1') @@ -492,7 +492,7 @@ class ViacoinTestnet(Viacoin): XPUB_VERBYTES = bytes.fromhex("043587CF") XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("7f") - P2SH_VERBYTE = bytes.fromhex("c4") + P2SH_VERBYTES = [bytes.fromhex("c4")] WIF_BYTE = bytes.fromhex("ff") GENESIS_HASH = ('00000007199508e34a9ff81e6ec0c477' 'a4cccff2a4767a8eee39c11db367b008') @@ -519,7 +519,7 @@ class Namecoin(CoinAuxPow): XPUB_VERBYTES = bytes.fromhex("d7dd6370") XPRV_VERBYTES = bytes.fromhex("d7dc6e31") P2PKH_VERBYTE = bytes.fromhex("34") - P2SH_VERBYTE = bytes.fromhex("0d") + P2SH_VERBYTES = [bytes.fromhex("0d")] WIF_BYTE = bytes.fromhex("e4") GENESIS_HASH = ('000000000062b72c5e2ceb45fbc8587e' '807c155b0da735e6483dfba2f0a9c770') @@ -537,7 +537,7 @@ class NamecoinTestnet(Namecoin): XPUB_VERBYTES = bytes.fromhex("043587cf") XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("6f") - P2SH_VERBYTE = bytes.fromhex("c4") + P2SH_VERBYTES = [bytes.fromhex("c4")] WIF_BYTE = bytes.fromhex("ef") GENESIS_HASH = ('00000007199508e34a9ff81e6ec0c477' 'a4cccff2a4767a8eee39c11db367b008') @@ -550,7 +550,7 @@ class Dogecoin(CoinAuxPow): XPUB_VERBYTES = bytes.fromhex("02facafd") XPRV_VERBYTES = bytes.fromhex("02fac398") P2PKH_VERBYTE = bytes.fromhex("1e") - P2SH_VERBYTE = bytes.fromhex("16") + P2SH_VERBYTES = [bytes.fromhex("16")] WIF_BYTE = bytes.fromhex("9e") GENESIS_HASH = ('1a91e3dace36e2be3bf030a65679fe82' '1aa1d6ef92e7c9902eb318182c355691') @@ -569,7 +569,7 @@ class DogecoinTestnet(Dogecoin): XPUB_VERBYTES = bytes.fromhex("043587cf") XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("71") - P2SH_VERBYTE = bytes.fromhex("c4") + P2SH_VERBYTES = [bytes.fromhex("c4")] WIF_BYTE = bytes.fromhex("f1") GENESIS_HASH = ('bb0a78264637406b6360aad926284d54' '4d7049f45189db5664f3c4d07350559e') @@ -585,7 +585,7 @@ class Dash(Coin): GENESIS_HASH = ('00000ffd590b1485b3caadc19b22e637' '9c733355108f107a430458cdf3407ab6') P2PKH_VERBYTE = bytes.fromhex("4c") - P2SH_VERBYTE = bytes.fromhex("10") + P2SH_VERBYTES = [bytes.fromhex("10")] WIF_BYTE = bytes.fromhex("cc") TX_COUNT_HEIGHT = 569399 TX_COUNT = 2157510 @@ -617,7 +617,7 @@ class DashTestnet(Dash): GENESIS_HASH = ('00000bafbc94add76cb75e2ec9289483' '7288a481e5c005f6563d91623bf8bc2c') P2PKH_VERBYTE = bytes.fromhex("8c") - P2SH_VERBYTE = bytes.fromhex("13") + P2SH_VERBYTES = [bytes.fromhex("13")] WIF_BYTE = bytes.fromhex("ef") TX_COUNT_HEIGHT = 101619 TX_COUNT = 132681 @@ -637,7 +637,7 @@ class Argentum(CoinAuxPow): XPUB_VERBYTES = bytes.fromhex("0488b21e") XPRV_VERBYTES = bytes.fromhex("0488ade4") P2PKH_VERBYTE = bytes.fromhex("17") - P2SH_VERBYTE = bytes.fromhex("05") + P2SH_VERBYTES = [bytes.fromhex("05")] WIF_BYTE = bytes.fromhex("97") GENESIS_HASH = ('88c667bc63167685e4e4da058fffdfe8' 'e007e5abffd6855de52ad59df7bb0bb2') @@ -655,7 +655,7 @@ class ArgentumTestnet(Argentum): XPUB_VERBYTES = bytes.fromhex("043587cf") XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("6f") - P2SH_VERBYTE = bytes.fromhex("c4") + P2SH_VERBYTES = [bytes.fromhex("c4")] WIF_BYTE = bytes.fromhex("ef") REORG_LIMIT = 2000 @@ -667,7 +667,7 @@ class DigiByte(Coin): XPUB_VERBYTES = bytes.fromhex("0488b21e") XPRV_VERBYTES = bytes.fromhex("0488ade4") P2PKH_VERBYTE = bytes.fromhex("1E") - P2SH_VERBYTE = bytes.fromhex("05") + P2SH_VERBYTES = [bytes.fromhex("05")] WIF_BYTE = bytes.fromhex("80") GENESIS_HASH = ('7497ea1b465eb39f1c8f507bc877078f' 'e016d6fcb6dfad3a64c98dcc6e1e8496') @@ -684,7 +684,7 @@ class DigiByteTestnet(DigiByte): XPUB_VERBYTES = bytes.fromhex("043587cf") XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("6f") - P2SH_VERBYTE = bytes.fromhex("c4") + P2SH_VERBYTES = [bytes.fromhex("c4")] WIF_BYTE = bytes.fromhex("ef") GENESIS_HASH = ('b5dca8039e300198e5fe7cd23bdd1728' 'e2a444af34c447dbd0916fa3430a68c2') @@ -701,7 +701,7 @@ class FairCoin(Coin): XPUB_VERBYTES = bytes.fromhex("0488b21e") XPRV_VERBYTES = bytes.fromhex("0488ade4") P2PKH_VERBYTE = bytes.fromhex("5f") - P2SH_VERBYTE = bytes.fromhex("24") + P2SH_VERBYTES = [bytes.fromhex("24")] WIF_BYTE = bytes.fromhex("df") GENESIS_HASH = ('1f701f2b8de1339dc0ec908f3fb6e9b0' 'b870b6f20ba893e120427e42bbc048d7') @@ -750,7 +750,7 @@ class Zcash(Coin): XPUB_VERBYTES = bytes.fromhex("0488b21e") XPRV_VERBYTES = bytes.fromhex("0488ade4") P2PKH_VERBYTE = bytes.fromhex("1CB8") - P2SH_VERBYTE = bytes.fromhex("1CBD") + P2SH_VERBYTES = [bytes.fromhex("1CBD")] WIF_BYTE = bytes.fromhex("80") GENESIS_HASH = ('00040fe8ec8471911baa1db1266ea15d' 'd06b4a8a5c453883c000b031973dce08') @@ -798,7 +798,7 @@ class Einsteinium(Coin): XPUB_VERBYTES = bytes.fromhex("0488b21e") XPRV_VERBYTES = bytes.fromhex("0488ade4") P2PKH_VERBYTE = bytes.fromhex("21") - P2SH_VERBYTE = bytes.fromhex("05") + P2SH_VERBYTES = [bytes.fromhex("05")] WIF_BYTE = bytes.fromhex("a1") GENESIS_HASH = ('4e56204bb7b8ac06f860ff1c845f03f9' '84303b5b97eb7b42868f714611aed94b') diff --git a/tests/lib/test_addresses.py b/tests/lib/test_addresses.py index be560d0..1e4ecce 100644 --- a/tests/lib/test_addresses.py +++ b/tests/lib/test_addresses.py @@ -36,7 +36,7 @@ addresses = [ "a773db925b09add367dcc253c1f9bbc1d11ec6fd", "062d8515e50cb92b8a3a73"), (Litecoin, "LNBAaWuZmipg29WXfz5dtAm1pjo8FEH8yg", "206168f5322583ff37f8e55665a4789ae8963532", "b8cb80b26e8932f5b12a7e"), - (Litecoin, "3GxRZWkJufR5XA8hnNJgQ2gkASSheoBcmW", + (Litecoin, "MPAZsQAGrnGWKfQbtFJ2Dfw9V939e7D3E2", "a773db925b09add367dcc253c1f9bbc1d11ec6fd", "062d8515e50cb92b8a3a73"), (Zcash, "t1LppKe1sfPNDMysGSGuTjxoAsBcvvSYv5j", "206168f5322583ff37f8e55665a4789ae8963532", "b8cb80b26e8932f5b12a7e"), @@ -64,7 +64,7 @@ def test_address_from_hash160(address): verbyte, hash_bytes = raw[:verlen], raw[verlen:] if coin.P2PKH_VERBYTE == verbyte: assert coin.P2PKH_address_from_hash160(bytes.fromhex(hash)) == addr - elif coin.P2SH_VERBYTE == verbyte: + elif verbyte in coin.P2SH_VERBYTES: assert coin.P2SH_address_from_hash160(bytes.fromhex(hash)) == addr else: raise Exception("Unknown version byte") From b52628143b1bebb03f1c7521d9bc0b7678615445 Mon Sep 17 00:00:00 2001 From: SuBPaR42 Date: Sun, 30 Apr 2017 08:21:42 -0500 Subject: [PATCH 072/117] Update coins.py Updated to more recent block height and TX count --- lib/coins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index e81ffd4..77cd9de 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -322,8 +322,8 @@ class Bitcoin(Coin): WIF_BYTE = bytes.fromhex("80") GENESIS_HASH = ('000000000019d6689c085ae165831e93' '4ff763ae46a2a6c172b3f1b60a8ce26f') - TX_COUNT = 156335304 - TX_COUNT_HEIGHT = 429972 + TX_COUNT = 217380620 + TX_COUNT_HEIGHT = 464000 TX_PER_BLOCK = 1800 IRC_PREFIX = "E_" IRC_CHANNEL = "#electrum" From 400388336ac1e7b510bf589c08a23e876d0b52a2 Mon Sep 17 00:00:00 2001 From: SuBPaR42 Date: Mon, 1 May 2017 08:35:09 -0500 Subject: [PATCH 073/117] Change to default server peers electrum.trouth.net is no longer active. Replaced with another E-S server ;-) --- lib/coins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coins.py b/lib/coins.py index 77cd9de..48b79e5 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -331,7 +331,7 @@ class Bitcoin(Coin): PEERS = [ 'btc.smsys.me s995', 'electrum.be s t', - 'electrum.trouth.net p10000 s t', + 'ELECTRUM.not.fyi p1000 s t', 'electrum.vom-stausee.de s t', 'electrum3.hachre.de p10000 s t', 'electrum.hsmiths.com s t', From 5e92feb8a67b890ad2995ca2662a4d1b1e671d59 Mon Sep 17 00:00:00 2001 From: LaoDC Date: Wed, 3 May 2017 17:09:52 +0700 Subject: [PATCH 074/117] Add new variables to BANNER(_TOR) $SERVER_VER will return the version number (eg: 1.0.10) $SERVER_SUBVERSION will return the full version string (eg: ElectrumX 1.0.10) $VERSION is kept for legacy which is the same as $SERVER_SUBVERSION --- server/session.py | 4 +++- server/version.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) mode change 100644 => 100755 server/session.py mode change 100644 => 100755 server/version.py diff --git a/server/session.py b/server/session.py old mode 100644 new mode 100755 index b184e41..1a3355e --- a/server/session.py +++ b/server/session.py @@ -289,7 +289,9 @@ class ElectrumX(SessionBase): revision //= 100 daemon_version = '{:d}.{:d}.{:d}'.format(major, minor, revision) for pair in [ - ('$VERSION', version.VERSION), + ('$VERSION', version.SUB_VERSION), # legacy + ('$SERVER_VERSION', version.VERSION), + ('$SERVER_SUBVERSION', version.SUB_VERSION), ('$DAEMON_VERSION', daemon_version), ('$DAEMON_SUBVERSION', network_info['subversion']), ('$DONATION_ADDRESS', self.env.donation_address), diff --git a/server/version.py b/server/version.py old mode 100644 new mode 100755 index cdab28f..bcce393 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,6 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.10' +VERSION = '1.0.10' +SUB_VERSION = 'ElectrumX {}'.format(VERSION) PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From 9dceeb914cad8cdfd606646ae7427576f0c77fe2 Mon Sep 17 00:00:00 2001 From: LaoDC Date: Wed, 3 May 2017 17:10:52 +0700 Subject: [PATCH 075/117] Add new variables to BANNER(_TOR) $SERVER_VER will return the version number (eg: 1.0.10) $SERVER_SUBVERSION will return the full version string (eg: ElectrumX 1.0.10) $VERSION is kept for legacy which is the same as $SERVER_SUBVERSION --- server/session.py | 0 server/version.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 server/session.py mode change 100755 => 100644 server/version.py diff --git a/server/session.py b/server/session.py old mode 100755 new mode 100644 diff --git a/server/version.py b/server/version.py old mode 100755 new mode 100644 From 9e34bf858320d9b98aff687509451df24450251c Mon Sep 17 00:00:00 2001 From: LaoDC Date: Wed, 3 May 2017 17:43:47 +0700 Subject: [PATCH 076/117] reverted and made the version split isolated within the banner logic only as not to affect other parts of the code. --- server/session.py | 7 ++++--- server/version.py | 3 +-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/session.py b/server/session.py index 1a3355e..a59c9a8 100644 --- a/server/session.py +++ b/server/session.py @@ -288,10 +288,11 @@ class ElectrumX(SessionBase): minor, revision = divmod(minor, 10000) revision //= 100 daemon_version = '{:d}.{:d}.{:d}'.format(major, minor, revision) + server_version = version.VERSION.split()[-1] for pair in [ - ('$VERSION', version.SUB_VERSION), # legacy - ('$SERVER_VERSION', version.VERSION), - ('$SERVER_SUBVERSION', version.SUB_VERSION), + ('$VERSION', version.VERSION), # legacy + ('$SERVER_VERSION', server_version), + ('$SERVER_SUBVERSION', version.VERSION), ('$DAEMON_VERSION', daemon_version), ('$DAEMON_SUBVERSION', network_info['subversion']), ('$DONATION_ADDRESS', self.env.donation_address), diff --git a/server/version.py b/server/version.py index bcce393..cdab28f 100644 --- a/server/version.py +++ b/server/version.py @@ -1,6 +1,5 @@ # Server name and protocol versions -VERSION = '1.0.10' -SUB_VERSION = 'ElectrumX {}'.format(VERSION) +VERSION = 'ElectrumX 1.0.10' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From c5c75c30defc17e82d0a13ea2b2cce64e67fb8e3 Mon Sep 17 00:00:00 2001 From: LaoDC Date: Wed, 3 May 2017 19:25:14 +0700 Subject: [PATCH 077/117] Updated docs/ENVIRONMENT.rst to reflect changes in variables --- docs/ENVIRONMENT.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index 5c63cd5..31787b8 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -116,8 +116,10 @@ These environment variables are optional: You can place several meta-variables in your banner file, which will be replaced before serving to a client. - + **$VERSION** is replaced with the ElectrumX version you are - runnning, such as *ElectrumX 0.9.22*. + + **$SERVER_VERSION** is replaced with the ElectrumX version you are + runnning, such as *1.0.10*. + + **$_SERVER_SUBVERSION** is replace with the ElectrumX user agent + string. For example, `ElectrumX 1.0.10`. + **$DAEMON_VERSION** is replaced with the daemon's version as a dot-separated string. For example *0.12.1*. + **$DAEMON_SUBVERSION** is replaced with the daemon's user agent From 70c6c87852a3ce0f68d775ebc5b11d529f759b06 Mon Sep 17 00:00:00 2001 From: LaoDC Date: Wed, 3 May 2017 19:26:49 +0700 Subject: [PATCH 078/117] fixed typo. --- docs/ENVIRONMENT.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 docs/ENVIRONMENT.rst diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst old mode 100644 new mode 100755 index 31787b8..7eb3592 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -118,7 +118,7 @@ These environment variables are optional: + **$SERVER_VERSION** is replaced with the ElectrumX version you are runnning, such as *1.0.10*. - + **$_SERVER_SUBVERSION** is replace with the ElectrumX user agent + + **$SERVER_SUBVERSION** is replace with the ElectrumX user agent string. For example, `ElectrumX 1.0.10`. + **$DAEMON_VERSION** is replaced with the daemon's version as a dot-separated string. For example *0.12.1*. From 3c8ab998e43a57e9014c6d17f7dba4fafbaa0631 Mon Sep 17 00:00:00 2001 From: LaoDC Date: Wed, 3 May 2017 19:27:00 +0700 Subject: [PATCH 079/117] sorry about permissions --- docs/ENVIRONMENT.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 docs/ENVIRONMENT.rst diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst old mode 100755 new mode 100644 From 37a9b276463b3fa3301d1b3fb79e2044a5e6655b Mon Sep 17 00:00:00 2001 From: Neil Date: Thu, 4 May 2017 00:22:53 +0900 Subject: [PATCH 080/117] Update ENVIRONMENT.rst --- docs/ENVIRONMENT.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ENVIRONMENT.rst b/docs/ENVIRONMENT.rst index 7eb3592..9a52245 100644 --- a/docs/ENVIRONMENT.rst +++ b/docs/ENVIRONMENT.rst @@ -118,7 +118,7 @@ These environment variables are optional: + **$SERVER_VERSION** is replaced with the ElectrumX version you are runnning, such as *1.0.10*. - + **$SERVER_SUBVERSION** is replace with the ElectrumX user agent + + **$SERVER_SUBVERSION** is replaced with the ElectrumX user agent string. For example, `ElectrumX 1.0.10`. + **$DAEMON_VERSION** is replaced with the daemon's version as a dot-separated string. For example *0.12.1*. From 8fb0faac2dd671a3a51698f9cb964468222f4302 Mon Sep 17 00:00:00 2001 From: "Andres G. Aragoneses" Date: Thu, 4 May 2017 15:11:32 +0800 Subject: [PATCH 081/117] HOWTO.rst: fix typo --- docs/HOWTO.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/HOWTO.rst b/docs/HOWTO.rst index 94747c5..7a58f9b 100644 --- a/docs/HOWTO.rst +++ b/docs/HOWTO.rst @@ -46,7 +46,7 @@ recommend you install one of these and familiarise yourself with them. The instructions below and sample run scripts assume `daemontools`; adapting to `runit` should be trivial for someone used to either. -When building the database form the genesis block, ElectrumX has to +When building the database from the genesis block, ElectrumX has to flush large quantities of data to disk and its DB. You will have a better experience if the database directory is on an SSD than on an HDD. Currently to around height 447,100 of the Bitcoin blockchain the From a4e4f80ad7c3d0b8ead1a4a9296c7604f5f6c3b5 Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Mon, 1 May 2017 15:46:22 +0200 Subject: [PATCH 082/117] Allow custom Daemon and BlockProcessor classes --- lib/coins.py | 38 ++++++++++++++------------------------ server/controller.py | 14 +++++--------- server/mempool.py | 4 ++-- 3 files changed, 21 insertions(+), 35 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index 48b79e5..e23efa8 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -40,6 +40,8 @@ import lib.util as util from lib.hash import Base58, hash160, double_sha256, hash_to_str from lib.script import ScriptPubKey from lib.tx import Deserializer, DeserializerSegWit, DeserializerAuxPow, DeserializerZcash +from server.block_processor import BlockProcessor +from server.daemon import Daemon Block = namedtuple("Block", "header transactions") @@ -56,12 +58,15 @@ class Coin(object): RPC_URL_REGEX = re.compile('.+@(\[[0-9a-fA-F:]+\]|[^:]+)(:[0-9]+)?') VALUE_PER_COIN = 100000000 CHUNK_SIZE = 2016 + HASHX_LEN = 11 BASIC_HEADER_SIZE = 80 STATIC_BLOCK_HEADERS = True + DESERIALIZER = Deserializer + DAEMON = Daemon + BLOCK_PROCESSOR = BlockProcessor IRC_PREFIX = None IRC_SERVER = "irc.freenode.net" IRC_PORT = 6667 - HASHX_LEN = 11 # Peer discovery PEER_DEFAULT_PORTS = {'t': '50001', 's': '50002'} PEERS = [] @@ -261,8 +266,7 @@ class Coin(object): '''Returns (header, [(deserialized_tx, tx_hash), ...]) given a block and its height.''' header = cls.block_header(block, height) - deserializer = cls.deserializer() - txs = deserializer(block[len(header):]).read_tx_block() + txs = cls.DESERIALIZER(block[len(header):]).read_tx_block() return Block(header, txs) @classmethod @@ -289,15 +293,13 @@ class Coin(object): 'nonce': nonce, } - @classmethod - def deserializer(cls): - return Deserializer class CoinAuxPow(Coin): # Set NAME and NET to avoid exception in Coin::lookup_coin_class NAME = '' NET = '' STATIC_BLOCK_HEADERS = False + DESERIALIZER = DeserializerAuxPow @classmethod def header_hash(cls, header): @@ -307,7 +309,7 @@ class CoinAuxPow(Coin): @classmethod def block_header(cls, block, height): '''Return the AuxPow block header bytes''' - block = DeserializerAuxPow(block) + block = cls.DESERIALIZER(block) return block.read_header(height, cls.BASIC_HEADER_SIZE) @@ -382,10 +384,7 @@ class BitcoinTestnetSegWit(BitcoinTestnet): bitcoind on testnet, you must use this class as your "COIN". ''' NET = "testnet-segwit" - - @classmethod - def deserializer(cls): - return DeserializerSegWit + DESERIALIZER = DeserializerSegWit class BitcoinNolnet(Bitcoin): @@ -417,6 +416,7 @@ class Litecoin(Coin): WIF_BYTE = bytes.fromhex("b0") GENESIS_HASH = ('12a765e31ffd4059bada1e25190f6e98' 'c99d9714d334efa41a195a7e7e04bfe2') + DESERIALIZER = DeserializerSegWit TX_COUNT = 8908766 TX_COUNT_HEIGHT = 1105256 TX_PER_BLOCK = 10 @@ -433,10 +433,6 @@ class Litecoin(Coin): 'eywr5eubdbbe2laq.onion s50008 t50007', ] - @classmethod - def deserializer(cls): - return DeserializerSegWit - class LitecoinTestnet(Litecoin): SHORTNAME = "XLT" @@ -505,10 +501,7 @@ class ViacoinTestnet(Viacoin): class ViacoinTestnetSegWit(ViacoinTestnet): NET = "testnet-segwit" - - @classmethod - def deserializer(cls): - return DeserializerSegWit + DESERIALIZER = DeserializerSegWit # Source: namecoin.org @@ -756,6 +749,7 @@ class Zcash(Coin): 'd06b4a8a5c453883c000b031973dce08') STATIC_BLOCK_HEADERS = False BASIC_HEADER_SIZE = 140 # Excluding Equihash solution + DESERIALIZER = DeserializerZcash TX_COUNT = 329196 TX_COUNT_HEIGHT = 68379 TX_PER_BLOCK = 5 @@ -782,13 +776,9 @@ class Zcash(Coin): @classmethod def block_header(cls, block, height): '''Return the block header bytes''' - block = DeserializerZcash(block) + block = cls.DESERIALIZER(block) return block.read_header(height, cls.BASIC_HEADER_SIZE) - @classmethod - def deserializer(cls): - return DeserializerZcash - class Einsteinium(Coin): NAME = "Einsteinium" diff --git a/server/controller.py b/server/controller.py index 783353a..ad0b46c 100644 --- a/server/controller.py +++ b/server/controller.py @@ -9,10 +9,8 @@ import asyncio import json import os import ssl -import random import time import traceback -import warnings from bisect import bisect_left from collections import defaultdict from concurrent.futures import ThreadPoolExecutor @@ -20,12 +18,11 @@ from functools import partial import pylru -from lib.jsonrpc import JSONRPC, JSONSessionBase, RPCError +from lib.jsonrpc import JSONSessionBase, RPCError from lib.hash import double_sha256, hash_to_str, hex_str_to_hash from lib.peer import Peer import lib.util as util -from server.block_processor import BlockProcessor -from server.daemon import Daemon, DaemonError +from server.daemon import DaemonError from server.mempool import MemPool from server.peers import PeerManager from server.session import LocalRPC, ElectrumX @@ -50,8 +47,8 @@ class Controller(util.LoggedClass): self.loop.set_default_executor(self.executor) self.start_time = time.time() self.coin = env.coin - self.daemon = Daemon(env.coin.daemon_urls(env.daemon_url)) - self.bp = BlockProcessor(env, self, self.daemon) + self.daemon = self.coin.DAEMON(env.coin.daemon_urls(env.daemon_url)) + self.bp = self.coin.BLOCK_PROCESSOR(env, self, self.daemon) self.mempool = MemPool(self.bp, self) self.peer_mgr = PeerManager(env, self) self.env = env @@ -864,8 +861,7 @@ class Controller(util.LoggedClass): if not raw_tx: return None raw_tx = bytes.fromhex(raw_tx) - deserializer = self.coin.deserializer() - tx, tx_hash = deserializer(raw_tx).read_tx() + tx, tx_hash = self.coin.DESERIALIZER(raw_tx).read_tx() if index >= len(tx.outputs): return None return self.coin.address_from_script(tx.outputs[index].pk_script) diff --git a/server/mempool.py b/server/mempool.py index c7843ba..0a6c27b 100644 --- a/server/mempool.py +++ b/server/mempool.py @@ -198,7 +198,7 @@ class MemPool(util.LoggedClass): not depend on the result remaining the same are fine. ''' script_hashX = self.coin.hashX_from_script - deserializer = self.coin.deserializer() + deserializer = self.coin.DESERIALIZER db_utxo_lookup = self.db.db_utxo_lookup txs = self.txs @@ -270,7 +270,7 @@ class MemPool(util.LoggedClass): if hashX not in self.hashXs: return [] - deserializer = self.coin.deserializer() + deserializer = self.coin.DESERIALIZER hex_hashes = self.hashXs[hashX] raw_txs = await self.daemon.getrawtransactions(hex_hashes) result = [] From b0e23e903dc7348e53549c4ca3a65989f5f2751b Mon Sep 17 00:00:00 2001 From: TheLazieR Yip Date: Sun, 14 May 2017 16:26:17 +0000 Subject: [PATCH 083/117] Allow custom ElectrumX class --- lib/coins.py | 2 ++ server/controller.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index e23efa8..4b0551b 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -42,6 +42,7 @@ from lib.script import ScriptPubKey from lib.tx import Deserializer, DeserializerSegWit, DeserializerAuxPow, DeserializerZcash from server.block_processor import BlockProcessor from server.daemon import Daemon +from server.session import ElectrumX Block = namedtuple("Block", "header transactions") @@ -61,6 +62,7 @@ class Coin(object): HASHX_LEN = 11 BASIC_HEADER_SIZE = 80 STATIC_BLOCK_HEADERS = True + SESSIONCLS = ElectrumX DESERIALIZER = Deserializer DAEMON = Daemon BLOCK_PROCESSOR = BlockProcessor diff --git a/server/controller.py b/server/controller.py index ad0b46c..ec536aa 100644 --- a/server/controller.py +++ b/server/controller.py @@ -25,7 +25,7 @@ import lib.util as util from server.daemon import DaemonError from server.mempool import MemPool from server.peers import PeerManager -from server.session import LocalRPC, ElectrumX +from server.session import LocalRPC class Controller(util.LoggedClass): @@ -248,7 +248,7 @@ class Controller(util.LoggedClass): server.close() async def start_server(self, kind, *args, **kw_args): - protocol_class = LocalRPC if kind == 'RPC' else ElectrumX + protocol_class = LocalRPC if kind == 'RPC' else self.coin.SESSIONCLS protocol_factory = partial(protocol_class, self, kind) server = self.loop.create_server(protocol_factory, *args, **kw_args) @@ -309,7 +309,7 @@ class Controller(util.LoggedClass): self.header_cache.clear() # Make a copy; self.sessions can change whilst await-ing - sessions = [s for s in self.sessions if isinstance(s, ElectrumX)] + sessions = [s for s in self.sessions if isinstance(s, self.coin.SESSIONCLS)] for session in sessions: await session.notify(self.bp.db_height, touched) From f179c679353cb38636f48cccd83d8e8afc3b36d5 Mon Sep 17 00:00:00 2001 From: TheLazieR Yip Date: Sun, 14 May 2017 14:59:51 +0000 Subject: [PATCH 084/117] Add support for Dash Masternode methods + Add DashDaemon class + Add DashElectrumX class + Update coin configurations for Dash --- lib/coins.py | 4 ++++ server/daemon.py | 9 ++++++++ server/session.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) diff --git a/lib/coins.py b/lib/coins.py index 4b0551b..f66d148 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -596,6 +596,10 @@ class Dash(Coin): 'electrum.dash.siampm.com s t', 'wl4sfwq2hwxnodof.onion s t', ] + from server.session import DashElectrumX + SESSIONCLS = DashElectrumX + from server.daemon import DashDaemon + DAEMON = DashDaemon @classmethod def header_hash(cls, header): diff --git a/server/daemon.py b/server/daemon.py index 39109cc..36f6b14 100644 --- a/server/daemon.py +++ b/server/daemon.py @@ -247,3 +247,12 @@ class Daemon(util.LoggedClass): If the daemon has not been queried yet this returns None.''' return self._height + +class DashDaemon(Daemon): + async def masternode_broadcast(self, params): + '''Broadcast a transaction to the network.''' + return await self._send_single('masternodebroadcast', params) + + async def masternode_list(self, params ): + '''Return the masternode status.''' + return await self._send_single('masternodelist', params) diff --git a/server/session.py b/server/session.py index a59c9a8..43cc2a3 100644 --- a/server/session.py +++ b/server/session.py @@ -387,3 +387,59 @@ class LocalRPC(SessionBase): def request_handler(self, method): '''Return the async handler for the given request method.''' return self.controller.rpc_handlers.get(method) + + +class DashElectrumX(ElectrumX): + '''A TCP server that handles incoming Electrum Dash connections.''' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.electrumx_handlers['masternode.announce.broadcast'] = self.masternode_announce_broadcast + self.electrumx_handlers['masternode.subscribe'] = self.masternode_subscribe + self.subscribe_mns = False + self.mns = set() + + async def notify(self, height, touched): + '''Notify the client about changes in masternode list.''' + + await super().notify(height, touched) + + if self.subscribe_mns: + for masternode in self.mns: + status = await self.daemon.masternode_list(['status', masternode]) + payload = { + 'id': None, + 'method': 'masternode.subscribe', + 'params': [masternode], + 'result': status.get(masternode), + } + self.send_binary(self.encode_payload(payload)) + + # Masternode command handlers + async def masternode_announce_broadcast(self, signmnb): + '''Pass through the parameters to the daemon. + + An ugly API: current Electrum clients only pass the masternode + broadcast message in hex and expect error messages to be returned in + the result field. And the server shouldn't be doing the client's + user interface job here. + ''' + try: + mnb_info = await self.daemon.masternode_broadcast(['relay',signmnb]) + return mnb_info + except DaemonError as e: + error = e.args[0] + message = error['message'] + self.log_info('masternode_broadcast: {}'.format(message)) + return ( + 'The masternode broadcast was rejected. ({})\n[{}]' + .format(message, signmnb) + ) + + async def masternode_subscribe(self, vin): + '''Returns the status of masternode.''' + result = await self.daemon.masternode_list(['status',vin]) + if result is not None: + self.mns.add(vin) + self.subscribe_mns = True + return result.get(vin) From 4da227872986bbec8f7c11e75f73764e2534fd33 Mon Sep 17 00:00:00 2001 From: TheLazieR Yip Date: Sun, 14 May 2017 17:38:55 +0000 Subject: [PATCH 085/117] Force server string response for Electrum-Dash 2.6.4 client --- server/session.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/session.py b/server/session.py index 43cc2a3..a302c58 100644 --- a/server/session.py +++ b/server/session.py @@ -415,6 +415,16 @@ class DashElectrumX(ElectrumX): } self.send_binary(self.encode_payload(payload)) + def server_version(self, client_name=None, protocol_version=None): + '''Returns the server version as a string. + ForcE version string response for Electrum-Dash 2.6.4 + ''' + + default_return = super().server_version(client_name, protocol_version) + if self.client == '2.6.4': + return '1.0' + return default_return + # Masternode command handlers async def masternode_announce_broadcast(self, signmnb): '''Pass through the parameters to the daemon. From 8d21eae2bbf3a72388af4d6969831677220d0529 Mon Sep 17 00:00:00 2001 From: TheLazieR Yip Date: Mon, 15 May 2017 04:12:15 +0000 Subject: [PATCH 086/117] Move import lines to top of class --- lib/coins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index f66d148..cff766c 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -572,6 +572,8 @@ class DogecoinTestnet(Dogecoin): # Source: https://github.com/dashpay/dash class Dash(Coin): + from server.session import DashElectrumX + from server.daemon import DashDaemon NAME = "Dash" SHORTNAME = "DASH" NET = "mainnet" @@ -596,9 +598,7 @@ class Dash(Coin): 'electrum.dash.siampm.com s t', 'wl4sfwq2hwxnodof.onion s t', ] - from server.session import DashElectrumX SESSIONCLS = DashElectrumX - from server.daemon import DashDaemon DAEMON = DashDaemon @classmethod From e99400c225e0246272c4865d5294e5b168736659 Mon Sep 17 00:00:00 2001 From: TheLazieR Yip Date: Mon, 15 May 2017 04:12:33 +0000 Subject: [PATCH 087/117] Update DashElectrumX as commented --- server/session.py | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/server/session.py b/server/session.py index a302c58..4b39098 100644 --- a/server/session.py +++ b/server/session.py @@ -396,7 +396,6 @@ class DashElectrumX(ElectrumX): super().__init__(*args, **kwargs) self.electrumx_handlers['masternode.announce.broadcast'] = self.masternode_announce_broadcast self.electrumx_handlers['masternode.subscribe'] = self.masternode_subscribe - self.subscribe_mns = False self.mns = set() async def notify(self, height, touched): @@ -404,20 +403,19 @@ class DashElectrumX(ElectrumX): await super().notify(height, touched) - if self.subscribe_mns: - for masternode in self.mns: - status = await self.daemon.masternode_list(['status', masternode]) - payload = { - 'id': None, - 'method': 'masternode.subscribe', - 'params': [masternode], - 'result': status.get(masternode), - } - self.send_binary(self.encode_payload(payload)) + for masternode in self.mns: + status = await self.daemon.masternode_list(['status', masternode]) + payload = { + 'id': None, + 'method': 'masternode.subscribe', + 'params': [masternode], + 'result': status.get(masternode), + } + self.send_binary(self.encode_payload(payload)) def server_version(self, client_name=None, protocol_version=None): '''Returns the server version as a string. - ForcE version string response for Electrum-Dash 2.6.4 + Force version string response for Electrum-Dash 2.6.4 ''' default_return = super().server_version(client_name, protocol_version) @@ -427,15 +425,10 @@ class DashElectrumX(ElectrumX): # Masternode command handlers async def masternode_announce_broadcast(self, signmnb): - '''Pass through the parameters to the daemon. + '''Pass through the masternode announce message to be broadcast by the daemon.''' - An ugly API: current Electrum clients only pass the masternode - broadcast message in hex and expect error messages to be returned in - the result field. And the server shouldn't be doing the client's - user interface job here. - ''' try: - mnb_info = await self.daemon.masternode_broadcast(['relay',signmnb]) + mnb_info = await self.daemon.masternode_broadcast(['relay', signmnb]) return mnb_info except DaemonError as e: error = e.args[0] @@ -448,8 +441,8 @@ class DashElectrumX(ElectrumX): async def masternode_subscribe(self, vin): '''Returns the status of masternode.''' - result = await self.daemon.masternode_list(['status',vin]) + result = await self.daemon.masternode_list(['status', vin]) if result is not None: self.mns.add(vin) - self.subscribe_mns = True - return result.get(vin) + return result.get(vin) + return None From 2e87d49e04b060a8e6fd9f69327d363e901e9b8c Mon Sep 17 00:00:00 2001 From: TheLazieR Yip Date: Mon, 15 May 2017 08:10:09 +0000 Subject: [PATCH 088/117] Add more comment on DashElectrumX.server_version --- server/session.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/session.py b/server/session.py index 4b39098..72fb8a3 100644 --- a/server/session.py +++ b/server/session.py @@ -415,7 +415,8 @@ class DashElectrumX(ElectrumX): def server_version(self, client_name=None, protocol_version=None): '''Returns the server version as a string. - Force version string response for Electrum-Dash 2.6.4 + Force version string response for Electrum-Dash 2.6.4 client caused by + https://github.com/dashpay/electrum-dash/commit/638cf6c0aeb7be14a85ad98f873791cb7b49ee29 ''' default_return = super().server_version(client_name, protocol_version) From 08dbbf217a0f656dd7b0a7369087aa6896078e0d Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 16 May 2017 14:55:19 +0900 Subject: [PATCH 089/117] client_version must be a tuple Fixes #180 --- server/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/session.py b/server/session.py index a59c9a8..b5ab239 100644 --- a/server/session.py +++ b/server/session.py @@ -34,7 +34,7 @@ class SessionBase(JSONSession): self.env = controller.env self.daemon = self.bp.daemon self.client = 'unknown' - self.client_version = (1) + self.client_version = (1, ) self.protocol_version = '1.0' self.anon_logs = self.env.anon_logs self.last_delay = 0 From 232d6be72c8bc40b23362ca83380b5ebffa67e6a Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Thu, 18 May 2017 15:07:34 +0200 Subject: [PATCH 090/117] Remove dead code --- lib/tx.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/lib/tx.py b/lib/tx.py index 86b207f..23aaea6 100644 --- a/lib/tx.py +++ b/lib/tx.py @@ -56,13 +56,6 @@ class TxInput(namedtuple("TxInput", "prev_hash prev_idx script sequence")): return (self.prev_hash == TxInput.ZERO and self.prev_idx == TxInput.MINUS_1) - @cachedproperty - def script_sig_info(self): - # No meaning for coinbases - if self.is_coinbase: - return None - return Script.parse_script_sig(self.script) - def __str__(self): script = self.script.hex() prev_hash = hash_to_str(self.prev_hash) @@ -71,12 +64,7 @@ class TxInput(namedtuple("TxInput", "prev_hash prev_idx script sequence")): class TxOutput(namedtuple("TxOutput", "value pk_script")): - '''Class representing a transaction output.''' - - @cachedproperty - def pay_to(self): - return Script.parse_pk_script(self.pk_script) - + pass class Deserializer(object): '''Deserializes blocks into transactions. From 9a9f78030d9b9308dac7a7032da442b51a61b657 Mon Sep 17 00:00:00 2001 From: Antti Majakivi Date: Mon, 5 Jun 2017 23:09:15 +0300 Subject: [PATCH 091/117] fix a typo Fixes a typo. --- electrumx_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electrumx_server.py b/electrumx_server.py index 695924d..946d445 100755 --- a/electrumx_server.py +++ b/electrumx_server.py @@ -33,7 +33,7 @@ def main_loop(): raise RuntimeError('Python >= 3.5.3 is required to run ElectrumX') if os.geteuid() == 0: - raise RuntimeError('DO NOT RUN AS ROOT! Create an unpriveleged user ' + raise RuntimeError('DO NOT RUN AS ROOT! Create an unprivileged user ' 'account and use that') loop = asyncio.get_event_loop() From 4cc37205894cac3245d3a639dd5c4ab13821f64e Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 7 Jun 2017 23:15:42 +0900 Subject: [PATCH 092/117] Disable IRC for bitcoin mainnet Only require IRC_CHANNEL if IRC_PREFIX is given. --- lib/coins.py | 13 +++++-------- server/peers.py | 9 +++++++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index cff766c..d2a51ab 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -78,12 +78,14 @@ class Coin(object): '''Return a coin class given name and network. Raise an exception if unrecognised.''' - req_attrs = ('TX_COUNT', 'TX_COUNT_HEIGHT', 'TX_PER_BLOCK', - 'IRC_CHANNEL') + req_attrs = ['TX_COUNT', 'TX_COUNT_HEIGHT', 'TX_PER_BLOCK'] for coin in util.subclasses(Coin): if (coin.NAME.lower() == name.lower() and coin.NET.lower() == net.lower()): - missing = [attr for attr in req_attrs + coin_req_attrs = req_attrs.copy() + if coin.IRC_PREFIX is not None: + coin_req_attrs.append('IRC_CHANNEL') + missing = [attr for attr in coin_req_attrs if not hasattr(coin, attr)] if missing: raise CoinError('coin {} missing {} attributes' @@ -329,8 +331,6 @@ class Bitcoin(Coin): TX_COUNT = 217380620 TX_COUNT_HEIGHT = 464000 TX_PER_BLOCK = 1800 - IRC_PREFIX = "E_" - IRC_CHANNEL = "#electrum" RPC_PORT = 8332 PEERS = [ 'btc.smsys.me s995', @@ -353,7 +353,6 @@ class Bitcoin(Coin): class BitcoinTestnet(Bitcoin): SHORTNAME = "XTN" NET = "testnet" - IRC_PREFIX = None XPUB_VERBYTES = bytes.fromhex("043587cf") XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("6f") @@ -422,8 +421,6 @@ class Litecoin(Coin): TX_COUNT = 8908766 TX_COUNT_HEIGHT = 1105256 TX_PER_BLOCK = 10 - IRC_CHANNEL = "#electrum-ltc" # obsolete - IRC_PREFIX = None RPC_PORT = 9332 REORG_LIMIT = 800 PEERS = [ diff --git a/server/peers.py b/server/peers.py index 554ed0f..06f46e9 100644 --- a/server/peers.py +++ b/server/peers.py @@ -216,7 +216,10 @@ class PeerManager(util.LoggedClass): self.env = env self.controller = controller self.loop = controller.loop - self.irc = IRC(env, self) + if env.irc and env.coin.IRC_PREFIX: + self.irc = IRC(env, self) + else: + self.irc = None self.myselves = peers_from_env(env) self.retry_event = asyncio.Event() # Peers have one entry per hostname. Once connected, the @@ -438,10 +441,12 @@ class PeerManager(util.LoggedClass): def connect_to_irc(self): '''Connect to IRC if not disabled.''' - if self.env.irc and self.env.coin.IRC_PREFIX: + if self.irc: pairs = [(peer.real_name(), ident.nick_suffix) for peer, ident in zip(self.myselves, self.env.identities)] self.ensure_future(self.irc.start(pairs)) + elif self.env.irc: + self.logger.info('IRC is disabled for this coin') else: self.logger.info('IRC is disabled') From 661883732c7bef75ea13adbdf5ffbc69525f7578 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 7 Jun 2017 23:28:09 +0900 Subject: [PATCH 093/117] Update nolnet parameters --- lib/coins.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index d2a51ab..133a97f 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -392,12 +392,12 @@ class BitcoinNolnet(Bitcoin): '''Bitcoin Unlimited nolimit testnet.''' NET = "nolnet" - GENESIS_HASH = ('00000000e752e935119102b142b5c27a' - '346a023532a42edcf7c8ffd0a22206e9') + GENESIS_HASH = ('0000000057e31bd2066c939a63b7b862' + '3bd0f10d8c001304bdfc1a7902ae6d35') REORG_LIMIT = 8000 - TX_COUNT = 195106 - TX_COUNT_HEIGHT = 24920 - TX_PER_BLOCK = 8 + TX_COUNT = 583589 + TX_COUNT_HEIGHT = 8617 + TX_PER_BLOCK = 50 IRC_PREFIX = "EN_" RPC_PORT = 28332 PEER_DEFAULT_PORTS = {'t': '52001', 's': '52002'} From 0cf4210a66d82377bf0d8350d84c658385db614f Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 7 Jun 2017 23:24:32 +0900 Subject: [PATCH 094/117] Prepare 1.0.11 --- README.rst | 14 ++++++++++++++ server/version.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index fb7cac8..21de0d2 100644 --- a/README.rst +++ b/README.rst @@ -130,6 +130,19 @@ Roadmap ChangeLog ========= +Version 1.0.11 +-------------- + +- disable IRC for bitcoin mainnet +- remove dead code, allow custom Daemon & BlockProcessor classes (erasmospunk) +- add SERVER_(SUB)VERSION to banner metavariables (LaoDC) +- masternode methods for Dash (TheLazier) +- allow multiple P2SH address versions, implement for Litecoin (pooler) +- update Bitcoin's TX_COUNT and block height (JWU42) +- update BU nolnet parameters +- fix diagnostic typo (anduck) +- Issues fixed: `#180`_ + Version 1.0.10 -------------- @@ -296,6 +309,7 @@ documentation updates. .. _#160: https://github.com/kyuupichan/electrumx/issues/160 .. _#162: https://github.com/kyuupichan/electrumx/issues/162 .. _#163: https://github.com/kyuupichan/electrumx/issues/163 +.. _#180: https://github.com/kyuupichan/electrumx/issues/180 .. _docs/HOWTO.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/HOWTO.rst .. _docs/ENVIRONMENT.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/ENVIRONMENT.rst .. _docs/PEER_DISCOVERY.rst: https://github.com/kyuupichan/electrumx/blob/master/docs/PEER_DISCOVERY.rst diff --git a/server/version.py b/server/version.py index cdab28f..6b84b68 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.10' +VERSION = 'ElectrumX 1.0.11' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From 1e9a65dccb33d8f115fc834cc9ab5ec7e9b81645 Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Sat, 10 Jun 2017 19:41:44 +0300 Subject: [PATCH 095/117] Handle legacy daemon RPCs Add support for daemons that don't have the new 'getblock' RPC call that returns the block in hex, the workaround is to manually recreate the block bytes. The recreated block bytes may not be the exact ones as in the underlying blockchain but it is good enough for our indexing purposes. --- lib/tx.py | 31 ++++++++++++++++++++---- lib/util.py | 15 ++++++++++++ server/daemon.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 5 deletions(-) diff --git a/lib/tx.py b/lib/tx.py index 23aaea6..7ed30ba 100644 --- a/lib/tx.py +++ b/lib/tx.py @@ -123,6 +123,11 @@ class Deserializer(object): self._read_varbytes(), # pk_script ) + def _read_byte(self): + cursor = self.cursor + self.cursor += 1 + return self.binary[cursor] + def _read_nbytes(self, n): cursor = self.cursor self.cursor = end = cursor + n @@ -182,11 +187,6 @@ class DeserializerSegWit(Deserializer): # https://bitcoincore.org/en/segwit_wallet_dev/#transaction-serialization - def _read_byte(self): - cursor = self.cursor - self.cursor += 1 - return self.binary[cursor] - def _read_witness(self, fields): read_witness_field = self._read_witness_field return [read_witness_field() for i in range(fields)] @@ -290,3 +290,24 @@ class DeserializerZcash(Deserializer): self.cursor += 32 # joinSplitPubKey self.cursor += 64 # joinSplitSig return base_tx, double_sha256(self.binary[start:self.cursor]) + + +class TxTime(namedtuple("Tx", "version time inputs outputs locktime")): + '''Class representing transaction that has a time field.''' + + @cachedproperty + def is_coinbase(self): + return self.inputs[0].is_coinbase + + +class DeserializerTxTime(Deserializer): + def read_tx(self): + start = self.cursor + + return TxTime( + self._read_le_int32(), # version + self._read_le_uint32(), # time + self._read_inputs(), # inputs + self._read_outputs(), # outputs + self._read_le_uint32(), # locktime + ), double_sha256(self.binary[start:self.cursor]) diff --git a/lib/util.py b/lib/util.py index 78006b9..2c09127 100644 --- a/lib/util.py +++ b/lib/util.py @@ -34,6 +34,7 @@ import logging import re import sys from collections import Container, Mapping +from struct import pack class LoggedClass(object): @@ -156,6 +157,20 @@ def int_to_bytes(value): return value.to_bytes((value.bit_length() + 7) // 8, 'big') +def int_to_varint(value): + '''Converts an integer to a Bitcoin-like varint bytes''' + if value < 0: + raise Exception("attempt to write size < 0") + elif value < 253: + return pack(' 0: + transactions = await self.getrawtransactions(b.get('tx'), False) + + raw_block = header + num_txs = len(transactions) + if num_txs > 0: + raw_block += util.int_to_varint(num_txs) + raw_block += b''.join(transactions) + else: + raw_block += b'\x00' + + return raw_block + + def timestamp_safe(self, t): + return t if isinstance(t, int) else timegm(strptime(t, "%Y-%m-%d %H:%M:%S %Z")) From 74f899e54469a18940822ae878ca933debffd11f Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Sat, 10 Jun 2017 19:44:30 +0300 Subject: [PATCH 096/117] Add support for Blackcoin and Peercoin --- lib/coins.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index 133a97f..60af405 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -39,9 +39,10 @@ from hashlib import sha256 import lib.util as util from lib.hash import Base58, hash160, double_sha256, hash_to_str from lib.script import ScriptPubKey -from lib.tx import Deserializer, DeserializerSegWit, DeserializerAuxPow, DeserializerZcash +from lib.tx import Deserializer, DeserializerSegWit, DeserializerAuxPow, \ + DeserializerZcash, DeserializerTxTime from server.block_processor import BlockProcessor -from server.daemon import Daemon +from server.daemon import Daemon, LegacyRPCDaemon from server.session import ElectrumX Block = namedtuple("Block", "header transactions") @@ -801,3 +802,60 @@ class Einsteinium(Coin): IRC_PREFIX = "E_" IRC_CHANNEL = "#electrum-emc2" RPC_PORT = 41879 + REORG_LIMIT = 2000 + + +class Blackcoin(Coin): + NAME = "Blackcoin" + SHORTNAME = "BLK" + NET = "mainnet" + XPUB_VERBYTES = bytes.fromhex("0488B21E") + XPRV_VERBYTES = bytes.fromhex("0488ADE4") + P2PKH_VERBYTE = bytes.fromhex("19") + P2SH_VERBYTES = [bytes.fromhex("55")] + WIF_BYTE = bytes.fromhex("99") + GENESIS_HASH = ('000001faef25dec4fbcf906e6242621d' + 'f2c183bf232f263d0ba5b101911e4563') + DESERIALIZER = DeserializerTxTime + DAEMON = LegacyRPCDaemon + TX_COUNT = 4594999 + TX_COUNT_HEIGHT = 1667070 + TX_PER_BLOCK = 3 + IRC_PREFIX = "E_" + IRC_CHANNEL = "#electrum-blk" + RPC_PORT = 15715 + REORG_LIMIT = 5000 + HEADER_HASH = None + + @classmethod + def header_hash(cls, header): + '''Given a header return the hash.''' + if cls.HEADER_HASH is None: + import scrypt + cls.HEADER_HASH = lambda x: scrypt.hash(x, x, 1024, 1, 1, 32) + + version, = struct.unpack(' 6: + return super().header_hash(header) + else: + return cls.HEADER_HASH(header); + + +class Peercoin(Coin): + NAME = "Peercoin" + SHORTNAME = "PPC" + NET = "mainnet" + P2PKH_VERBYTE = bytes.fromhex("37") + P2SH_VERBYTES = [bytes.fromhex("75")] + WIF_BYTE = bytes.fromhex("b7") + GENESIS_HASH = ('0000000032fe677166d54963b62a4677' + 'd8957e87c508eaa4fd7eb1c880cd27e3') + DESERIALIZER = DeserializerTxTime + DAEMON = LegacyRPCDaemon + TX_COUNT = 1207356 + TX_COUNT_HEIGHT = 306425 + TX_PER_BLOCK = 4 + IRC_PREFIX = "E_" + IRC_CHANNEL = "#electrum-ppc" + RPC_PORT = 9902 + REORG_LIMIT = 5000 From fe3008679596eed56382a9f0becc3ea426a99d1e Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Sat, 10 Jun 2017 19:26:06 +0300 Subject: [PATCH 097/117] Digibyte switched to SegWit --- lib/coins.py | 1 + tests/blocks/digibyte_mainnet_4394891.json | 19 +++++++++++++++++++ tests/test_blocks.py | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 tests/blocks/digibyte_mainnet_4394891.json diff --git a/lib/coins.py b/lib/coins.py index 60af405..3f45230 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -668,6 +668,7 @@ class DigiByte(Coin): WIF_BYTE = bytes.fromhex("80") GENESIS_HASH = ('7497ea1b465eb39f1c8f507bc877078f' 'e016d6fcb6dfad3a64c98dcc6e1e8496') + DESERIALIZER = DeserializerSegWit TX_COUNT = 1046018 TX_COUNT_HEIGHT = 1435000 TX_PER_BLOCK = 1000 diff --git a/tests/blocks/digibyte_mainnet_4394891.json b/tests/blocks/digibyte_mainnet_4394891.json new file mode 100644 index 0000000..7616b5c --- /dev/null +++ b/tests/blocks/digibyte_mainnet_4394891.json @@ -0,0 +1,19 @@ +{ + "hash": "00000000000000360f1e38ab606450cb321ee7ccd1749f0aa9717e81bdc88a4f", + "size": 2662, + "height": 4394891, + "merkleroot": "92cfbbc21caf5cd9158f608f6e021cf31ee02ca5d80a2ad441ad3a1968a36df4", + "tx": [ + "84013cf078d219325319ee26606f238f5d43d2a64b8b9eb0668aa36ff9f1497d", + "5a13a021223ba10d6bb45b6f0099222befae21ba5e94da45083e9a4d44208431", + "7c2de0d50ea330ea77a3ede88c1ea3ad5512a90d23c3ed440b8244d380c8f5f6", + "44c97671699cc8e33ed227dc2d083d728c21a98943e10dea0cf1fd4f39138401", + "3f5791d2a0feb15502e04ff50b357e2434ff7795e8bdf655d7bc052a31d09c6c", + "0f1420b301192ee97e27f3365db71d2e54c8bb14636a84adf7f8298b37439c3c" + ], + "time": 1493396706, + "nonce": 1152585183, + "bits": "197c6507", + "previousblockhash": "e0e151dc61399a257de7721df34af1d80caf4e2373e182ee1889356f047aef22", + "block": "0202002022ef7a046f358918ee82e173234eaf0cd8f14af31d72e77d259a3961dc51e1e0f46da368193aad41d42a0ad8a52ce01ef31c026e8f608f15d95caf1cc2bbcf92e26c035907657c19df0db34406010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff48038b0f4304e26c035904fabe6d6d7b2cb15c6afb3358bc138b2c5d3ea77029a79ba2a160eb0de64fb086dbdb86d6800000000000000087225c010d2f6e6f64655374726174756d2f00000000020000000000000000266a24aa21a9edc7679d2b2c8d87b91aa75fdc86ebad16f55cc3e7f30ba8ab3243a9aa27ae71a83d5bc6fb140000001976a914818dd7c871d1da14dff3a1ef0a1774720892d9d188ac01200000000000000000000000000000000000000000000000000000000000000000000000000100000001ce9be9710120296f8b0256ca87ad6b6da454a3f3bcd18ae2b0c04dbcafe8747a000000006a47304402206383b0eef3291a0c6721c06bc87544f9e3430f53ac7b14514491f4ca17e4e51c0220202cbdc050de073eeac995dd66222cea3c6c1ca08d17f4923f2caac547603e33012103fee47ef038c3acd7ee6ad449ce0316dcfab5bb6ec10adf89f8949f1bf3f5c117ffffffff02172e5cfc060000001976a914e2a12c65e85e07df375a4ffa01dd6f00a3e6f9b088acc6bb04f0000000001976a9142f8d7af63909d15011316795f1d760864b40590888ac8a0f43000100000001ccb42e3d95f72cb3298ff956b0b3f7e3eea1894ea341762584128506818c40b5000000006a47304402200b7bc22449dd31382af4df432a814030f0a90bc2557581d2672db577ee3b7dbd02200d794170b920cb46d757778331d6423c5ee40ddfd4ac13bfe61a7062656ddaec012103f475fe4752b8a241d346a226c186fa87861c92c1a60910d9547c32802236f233ffffffff02387a19e0070000001976a91443e4c5bcde490084b44d8b0ef8e632ab613f2f3588ac37ddfcb2020000001976a9148733c0565ff4d6d1700fb295e1e2ee54fc89a90088ac8a0f4300010000000843a4b739d09bd762bfb4db75e3302c3181233d90ecea1882d6c63ba4141960e0010000006a47304402202870f430fa9afb1cf641e03a6375fb6dc29649796112059965636ed03ba26e63022022d2f336a29d5e5883794c4dc834cd2d0cb2b0f6e7b555a635c2c1b9c1844eb90121029ecc902531c7c82051d05fb6150bda05e6d3a9ec5e5d745f2157e9a7199a5cfeffffffff889e5bd9c8546d8a7712059a49e98546f1900307ed1a4c5747c8d1ea4c613063010000006a47304402200741e47d3487e493326ef8d252b8b22ccadd3b3c8e8c6298193b490b9c157e1d02207fc176923fd23ef46adc2881565c6b59deca7b38a999bd25ad844f51ae7f87450121029ecc902531c7c82051d05fb6150bda05e6d3a9ec5e5d745f2157e9a7199a5cfeffffffffdb03f33eff3f40ec58ed398f71e9c6fbfebc5f8ce11dc0ad564c4acad260d896010000006a47304402202cdf16c328728860a3e9b9ea77f33b57cec899dfaa092c647355910cd49c5e75022073737ccda4faf99b9121c9bfc3c8519bba8a62870afaa8a7fdcd1d1233ead9c40121029ecc902531c7c82051d05fb6150bda05e6d3a9ec5e5d745f2157e9a7199a5cfeffffffff185c9f4cbd72e3393d11b38f52a4180ea9569e1e4c941735118b94a065748ee7010000006b483045022100b067da4c46680fac6e4cc5d596e9a519a10980a83fa00bca9ae700596ae02bb4022038daa9540fdba04f0d2e0e6b8ae252abb2524ce6bc35f99423db3b0fc0e3d6660121029ecc902531c7c82051d05fb6150bda05e6d3a9ec5e5d745f2157e9a7199a5cfeffffffff7ba77d5b20b9c847f36bcf91378514feccf269501bff9c03d1ee040e4576d10c010000006b48304502210082d7d3feeb0834e840b83078e6a1e877a39a4bea478b1e6a092ac258610fc075022027f02e1b4b928fc6b87d2c4cd1c38b04d4e0f8849c20cec56515f163c71e34b10121029ecc902531c7c82051d05fb6150bda05e6d3a9ec5e5d745f2157e9a7199a5cfeffffffff679991e83b2eda22969ff4e8d744d6a61a2564dfbd93e2268cebed66e3c89090010000006a47304402207464320ec4d46258311db47c7d288065e43a06f5e8ec57462d00e0107ac1f3dd02207195da6de457572ad2a859795c384fa9586a746f069b04fb2cf402e8b6761f020121029ecc902531c7c82051d05fb6150bda05e6d3a9ec5e5d745f2157e9a7199a5cfeffffffff99cceb882ff6dac25e4ed3b1544bdb1a86ea131c85a35affadac5b7d364af243010000006b483045022100c1b8bcdb3d7e1ddbc5139f0f7e3421595725341cc1c5fac100c37ff26bcc22ce0220206eb9ac6cc6f2274b73fe57a710d7c10357effa11d18ed5c1152ac0cf7883630121029ecc902531c7c82051d05fb6150bda05e6d3a9ec5e5d745f2157e9a7199a5cfefffffffffb859d8ef6107164fad5d4cf5b66b33c28fae160f5a8d0c7b581da7529c3b330010000006b483045022100810389a67e78a1ea701395cd7309d117df4da74f08494d72ed85068eb1390321022004967309f94749f7c509e58bd8e8a9a4e5e22bfe1e633c49860708ee6138a16f0121029ecc902531c7c82051d05fb6150bda05e6d3a9ec5e5d745f2157e9a7199a5cfeffffffff0222c39e2c0f0000001976a914058fe22860de1e0f920000474b75ed1f1921715488ac963677b1980000001976a91433411cfe6fd4b0711178aa01b766768f225f49e988ac8a0f430001000000027f96093f8cf13a06874e51eb746fe3abf417ae7d2112bde00500862cee7b61fd010000006b4830450221008a9b35de0e1ad798718181fc87f2e9df1e6014935184b477b039c64b202da16302200710b078f9af82730a790371e344e990808d7ad4baa2c22afee665fafbc1b48401210292f4522efa5e69119d2e730e693c044b8bba5ad903a19bc2005828e34a92fc1effffffffb96dee16abb340cbf94a82d4483fd352f07957d4a8c0af7b15152d8a9d27f5c6010000006a473044022028c09e38d44c7840cf7b27aa467172f8b1fbe563c215dbf9fa634eabf3ae7b1602202165c094826d09b5902e4e0ab1176f3842dca5df62006b5cdcd53bc181b8fac401210292f4522efa5e69119d2e730e693c044b8bba5ad903a19bc2005828e34a92fc1effffffff0316d09911140000001976a9142a33a4af7bd5ec3527d15fa2641238dc17bca9af88acb0a34d00000000001976a91499e1d69a1a492f91b5881378f12e1f04490cc38388ac266570cc130000001976a914b9dffedfa8b5de19d112062c41fe8df2f3314d3e88ac220f43000100000001620c30d8caf8a899b1cb5b8a08f68ef25900ff0457416fc9d593a1e4766e429b010000006b483045022100f217c1d780c4c38c267b715fd6605479def6f46e854dade7693ca30dde841f8e0220152434c8a50b00f0346c545551117a6edff9c1557b7fcf69216c91d795c5377f012102b11355a89c8fba57d2630ea6e83613c5a3faf19affcf66b166034969ca3520a5ffffffff026164b729010000001976a914fe59bacd86811cb31e06385d746d31b7062c1cfc88ac96bbc3a10b0000001976a91409e590d5122a22c14a737320fc7da480149b84f588ac8a0f4300" +} \ No newline at end of file diff --git a/tests/test_blocks.py b/tests/test_blocks.py index ff8b756..769a4c5 100644 --- a/tests/test_blocks.py +++ b/tests/test_blocks.py @@ -47,7 +47,7 @@ for name in os.listdir(BLOCKS_DIR): with open(os.path.join(BLOCKS_DIR, name)) as f: blocks.append((coin, json.load(f))) except Exception as e: - blocks.append(pytest.mark.skip(name)) + blocks.append(pytest.fail(name)) @pytest.fixture(params=blocks) From b48465a065ca082e940524e636d731f57ab145a9 Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Sat, 10 Jun 2017 19:30:52 +0300 Subject: [PATCH 098/117] Add Reddcoin support --- lib/coins.py | 22 +++++++++++++++++++++- lib/tx.py | 22 ++++++++++++++++++++++ tests/blocks/reddcoin_mainnet_1200000.json | 15 +++++++++++++++ tests/blocks/reddcoin_mainnet_8000.json | 20 ++++++++++++++++++++ 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 tests/blocks/reddcoin_mainnet_1200000.json create mode 100644 tests/blocks/reddcoin_mainnet_8000.json diff --git a/lib/coins.py b/lib/coins.py index 3f45230..82c48d6 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -40,7 +40,7 @@ import lib.util as util from lib.hash import Base58, hash160, double_sha256, hash_to_str from lib.script import ScriptPubKey from lib.tx import Deserializer, DeserializerSegWit, DeserializerAuxPow, \ - DeserializerZcash, DeserializerTxTime + DeserializerZcash, DeserializerTxTime, DeserializerReddcoin from server.block_processor import BlockProcessor from server.daemon import Daemon, LegacyRPCDaemon from server.session import ElectrumX @@ -860,3 +860,23 @@ class Peercoin(Coin): IRC_CHANNEL = "#electrum-ppc" RPC_PORT = 9902 REORG_LIMIT = 5000 + + +class Reddcoin(Coin): + NAME = "Reddcoin" + SHORTNAME = "RDD" + NET = "mainnet" + XPUB_VERBYTES = bytes.fromhex("0488B21E") + XPRV_VERBYTES = bytes.fromhex("0488ADE4") + P2PKH_VERBYTE = bytes.fromhex("3d") + P2SH_VERBYTES = [bytes.fromhex("05")] + WIF_BYTE = bytes.fromhex("bd") + GENESIS_HASH = ('b868e0d95a3c3c0e0dadc67ee587aaf9' + 'dc8acbf99e3b4b3110fad4eb74c1decc') + DESERIALIZER = DeserializerReddcoin + TX_COUNT = 5413508 + TX_COUNT_HEIGHT = 1717382 + TX_PER_BLOCK = 3 + IRC_PREFIX = "E_" + IRC_CHANNEL = "#electrum-rdd" + RPC_PORT = 45443 diff --git a/lib/tx.py b/lib/tx.py index 7ed30ba..554f349 100644 --- a/lib/tx.py +++ b/lib/tx.py @@ -311,3 +311,25 @@ class DeserializerTxTime(Deserializer): self._read_outputs(), # outputs self._read_le_uint32(), # locktime ), double_sha256(self.binary[start:self.cursor]) + + +class DeserializerReddcoin(Deserializer): + def read_tx(self): + start = self.cursor + + version = self._read_le_int32() + inputs = self._read_inputs() + outputs = self._read_outputs() + locktime = self._read_le_uint32() + if version > 1: + time = self._read_le_uint32() + else: + time = 0 + + return TxTime( + version, + time, + inputs, + outputs, + locktime, + ), double_sha256(self.binary[start:self.cursor]) diff --git a/tests/blocks/reddcoin_mainnet_1200000.json b/tests/blocks/reddcoin_mainnet_1200000.json new file mode 100644 index 0000000..cc7b723 --- /dev/null +++ b/tests/blocks/reddcoin_mainnet_1200000.json @@ -0,0 +1,15 @@ +{ + "hash": "bea68724bfcdc5d35bf9bbf9cb6680e196e5661afe95b2a205e74a2fe175ac79", + "size": 443, + "height": 1200000, + "merkleroot": "504b073c16d872d24f2c3de8a4c2c76d08df5056f3a4a8d0e32ff4220215a250", + "tx": [ + "6aaad9725ae7beb40d80dac9c5300a8e1cf8783adb0ea41da4988b3476bda9b8", + "4a949402995c11b3306c0c91fd85edf0d3eb8dee4bf6bd07a241fa170156cd3c" + ], + "time": 1463612841, + "nonce": 0, + "bits": "1c0a4691", + "previousblockhash": "438b564171da6fbbe6fd9d52c16ea2b1aa8c169951822225cf097d5da7cdba76", + "block": "0300000076bacda75d7d09cf2522825199168caab1a26ec1529dfde6bb6fda7141568b4350a2150222f42fe3d0a8a4f35650df086dc7c2a4e83d2c4fd272d8163c074b50a9f53c5791460a1c000000000202000000010000000000000000000000000000000000000000000000000000000000000000ffffffff020000ffffffff0100000000000000000000000000a9f53c570200000001a40cad8a9afe2888f746d762cb36649b5afd4e8ce4468fd8d08fc296d26dc4840100000048473044022036392ee6eb58c5a9a2a681692cabdc2b00166c374cfb711055bc2c4d6c61a1d40220475728eed260bf972ef44909f0d6fa282f17e92b5e57ee383c7171e8a3baee1f01ffffffff030000000000000000000056b12a38720000232102bee8ce24a99260fbb6c10f0b904498fa71ec08e51b531878d3f6568ef09acb91ac0ad6b22a38720000232102bee8ce24a99260fbb6c10f0b904498fa71ec08e51b531878d3f6568ef09acb91ac00000000a9f53c57473045022100fe801bae06c9db3076fad2f72930f76dbe1cae29a162447b13d0df749e5913df02203621013f87da4dbca08702d8c7975f702bad9df40902038b93e622a0dd9c0896" +} \ No newline at end of file diff --git a/tests/blocks/reddcoin_mainnet_8000.json b/tests/blocks/reddcoin_mainnet_8000.json new file mode 100644 index 0000000..4149476 --- /dev/null +++ b/tests/blocks/reddcoin_mainnet_8000.json @@ -0,0 +1,20 @@ +{ + "hash": "4889bb7d1ba24cc66c2d903f6643b0ade243aca5101a8aff87ab4c2ab2a15ec5", + "size": 1560, + "height": 80000, + "merkleroot": "193313cfa4d8a4bc15fb5526b69a87c922e0f6520295f66165358f0af6b5d637", + "tx": [ + "ad01e368a301b855d5f4499bc787b161428d6994c4847c0b2813950630a73950", + "1799481d7fed61c029159d314f75f3d6f69a7f8c237443470394085307802782", + "8db4b2c62fca987462c482d24ce0b78d2a3dd3928d5d99112ccad75deb6ff7de", + "ab0a1e66e54c737be6ea2be2c61cd55879d33c0fc5d35aa6389487e06c809cfc", + "1bb3854ed7fe9905b5637d405cd0715e5cb6f5fe233304a1588c53bdcf60f593", + "08d3ccf77f30e62d8773669adea730698806516239933ac7c4285bcacdb37989", + "19cbdc4acfb07dc29c73f039d8f5db967ce30c0667fda60babc700a7c53c0b5f" + ], + "time": 1396181239, + "nonce": 1368399360, + "bits": "1c0cc111", + "previousblockhash": "e34cfbf84095c64276ee098de50c3a1f863df9336968e9fb8a973bdd52e3ed04", + "block": "0200000004ede352dd3b978afbe9686933f93d861f3a0ce58d09ee7642c69540f8fb4ce337d6b5f60a8f356561f6950252f6e022c9879ab62655fb15bca4d8a4cf133319f708385311c10c1c001e90510701000000010000000000000000000000000000000000000000000000000000000000000000ffffffff2703803801062f503253482f04f608385308f800f159010000000d2f7374726174756d506f6f6c2f000000000100e63572180900001976a9148d41bc27ab2cc999338750edd4b0012bdb36f70288ac0000000001000000014b1a8085e29ca8ef2bf1de5f0637c6ef597b67223a087458e51e21168a0e44a3000000006b48304502200b781c255481e90f0e1d2fedbc1ffb42562434c324566444da8718a8a2c5182d022100f50faa7a9f7b90b4b805050c9731a79fb9c599ddfb3d84449d0cff7ee216bf59012103d7ab8ea88d09589410bdb93cd466d92f56985a3cff6d74dce3f033500135f0c5ffffffff02d72ea96581330e001976a91422758e8f790ea0e4ab87ad3990e8af86c77375c088ac1c1cab70190000001976a91434e880ed4cb32ebb1e0842b4f05efe562724f08788ac000000000100000001616c5b1a7ee823fa2d5347011b34e1ea027f9494823d37fb175eece8f852f987000000006a473044022000be9cf6677d879d170c597b8a465137577119ebc7d01773dc13df7af7e0bf1102202acfce90f478c0d179ab98d708f1e24f6dab4fe60c75893f8bad12991b30f41301210355dad820f63f1c315dc16c5afd9782e4d0b225ea29320a85576bc2c82fde6e7effffffff02ceb618fa97ac10001976a914e14548bfd2e14e0cabaf535c7c80a227238b35e188ac1c1cab70190000001976a914d2046a1ad1dbc32e69dae4da0a8730379105936e88ac000000000100000001a6b3081431b43c3247df88b3b6d123d2f2d7ba2095c6ef4f6532feb2c45f9210010000006b4830450221008fb902cc4130bae26439c47c13467a7d8a8c52ac2d88a200548f1e8f8b100b910220125b45cee0765389a59d4cca65482bdf79d3bc8fdaa5a0142e7829e4a2568124012103cdece1576249c8e05fb0aa2cbe61aa959330ff2f9e3c5cd2e5152e90650d9386ffffffff02bbba56d0d88606001976a91407499b20688a0b61b4a526681647de739dab818e88ac1c1cab70190000001976a9147085556af12556138277188e3958a869eeced02088ac000000000100000001fc9c806ce0879438a65ad3c50f3cd37958d51cc6e22beae67b734ce5661e0aab000000006c493046022100dca959b02a4dde588b3e5c3e71877797b97d7094a82cdd6b6b52c3d04a8c17c3022100938b2f70eed007d20ef9d7d055fc9b8785e71e3f0981558503fb3635b08aa6d40121039d216b71bad34246ceff262afe6df520761fc696fd9862c3f2f7e337ad93d881ffffffff0202386cc4f57e06001976a914ee343e816e6782262c3f6b1b9ec8f8c17d47a88c88acb9a1f405e30700001976a914ba81e33df7ba3d18728c6c206f8ad0b30b83b71988ac00000000010000000193f560cfbd538c58a1043323fef5b65c5e71d05c407d63b50599fed74e85b31b000000006a4730440220153f0a0a16e13943c4869e8f768c64e9f1844d14823f80878a6e44752a041c49022036ec13a307bafee74387048c3772cfb5ebdc138d70d6b4c256788a86db93ab5801210281232e155b37ebd64759ee4983962e9f8ccfd95e302d828de1406549e7c327a4ffffffff029014fad0166506001976a914b05959ea5dd831fd082488298466c9307a46f55b88ac72427cedde1900001976a914c2e3e90990f452c19ccef5df1cc3711c2e5d448288ac0000000001000000018979b3cdca5b28c4c73a9339625106886930a7de9a6673872de6307ff7ccd308000000006b483045022100ec50258bfec642e6c986192f338b7a1eec84c872d9b51ccc6f1c7329da20af77022047a6836d7c5f416c2eef6ef59fae9cc627ff80882897fe3eabd775e2a4a08533012102240bb70ae679cb25d60e2e0f90f98017eac7b6abbf1e00797ef930f02f0b98eeffffffff029e75ab66cd6306001976a9144c9ef3b178febefc62a0067e67e8434afe864a6788acf2bd5864490100001976a9143cde6d950e730b199c5857564afe7f222e139ead88ac00000000" +} \ No newline at end of file From 2940ea319982ebcfb51ea9edbbe78103b51b1a59 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 17 Jun 2017 16:33:15 +0900 Subject: [PATCH 099/117] Add new function get_history_txnums --- server/db.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/server/db.py b/server/db.py index cd42d66..4958c2b 100644 --- a/server/db.py +++ b/server/db.py @@ -471,13 +471,11 @@ class DB(util.LoggedClass): return nremoves - def get_history(self, hashX, limit=1000): - '''Generator that returns an unpruned, sorted list of (tx_hash, - height) tuples of confirmed transactions that touched the address, - earliest in the blockchain first. Includes both spending and - receiving transactions. By default yields at most 1000 entries. - Set limit to None to get them all. - ''' + def get_history_txnums(self, hashX, limit=1000): + '''Generator that returns an unpruned, sorted list of tx_nums in the + history of a hashX. Includes both spending and receiving + transactions. By default yields at most 1000 entries. Set + limit to None to get them all. ''' limit = self._resolve_limit(limit) for key, hist in self.hist_db.iterator(prefix=hashX): a = array.array('I') @@ -485,5 +483,15 @@ class DB(util.LoggedClass): for tx_num in a: if limit == 0: return - yield self.fs_tx_hash(tx_num) + yield tx_num limit -= 1 + + def get_history(self, hashX, limit=1000): + '''Generator that returns an unpruned, sorted list of (tx_hash, + height) tuples of confirmed transactions that touched the address, + earliest in the blockchain first. Includes both spending and + receiving transactions. By default yields at most 1000 entries. + Set limit to None to get them all. + ''' + for tx_num in self.get_history_txnums(hashX, limit): + yield self.fs_tx_hash(tx_num) From 2f26e8162924cce0979d18cf622adf13a82eda57 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 9 Apr 2017 21:43:36 +0900 Subject: [PATCH 100/117] Implement history compression with tests. Still to do: running compression in background when the flush count reaches a certain level --- compact_history.py | 76 ++++++++++++++ server/block_processor.py | 7 +- server/db.py | 180 +++++++++++++++++++++++++++++++- tests/server/test_compaction.py | 131 +++++++++++++++++++++++ 4 files changed, 390 insertions(+), 4 deletions(-) create mode 100755 compact_history.py create mode 100644 tests/server/test_compaction.py diff --git a/compact_history.py b/compact_history.py new file mode 100755 index 0000000..32e0cbb --- /dev/null +++ b/compact_history.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2017, Neil Booth +# +# All rights reserved. +# +# See the file "LICENCE" for information about the copyright +# and warranty status of this software. + +'''Script to compact the history database. This should save space and +will reset the flush counter to a low number, avoiding overflow when +the flush count reaches 65,536. + +This needs to lock the database so ElectrumX must not be running - +shut it down cleanly first. + +It is recommended you run this script with the same environment as +ElectrumX. However it is intended to be runnable with just +DB_DIRECTORY and COIN set (COIN defaults as for ElectrumX). + +If you use daemon tools, you might run this script like so: + + envdir /path/to/the/environment/directory ./compact_history.py + +Depending on your hardware this script may take up to 6 hours to +complete; it logs progress regularly. + +Compaction can be interrupted and restarted harmlessly and will pick +up where it left off. However, if you restart ElectrumX without +running the compaction to completion, it will not benefit and +subsequent compactions will restart from the beginning. +''' + +import logging +import sys +import traceback +from os import environ + +from server.env import Env +from server.db import DB + + +def compact_history(): + if sys.version_info < (3, 5, 3): + raise RuntimeError('Python >= 3.5.3 is required to run ElectrumX') + + environ['DAEMON_URL'] = '' # Avoid Env erroring out + env = Env() + db = DB(env) + + assert not db.first_sync + # Continue where we left off, if interrupted + if db.comp_cursor == -1: + db.comp_cursor = 0 + + db.comp_flush_count = max(db.comp_flush_count, 1) + limit = 8 * 1000 * 1000 + + while db.comp_cursor != -1: + db._compact_history(limit) + + +def main(): + logging.basicConfig(level=logging.INFO) + logging.info('Starting history compaction...') + try: + compact_history() + except Exception: + traceback.print_exc() + logging.critical('History compaction terminated abnormally') + else: + logging.info('History compaction complete') + + +if __name__ == '__main__': + main() diff --git a/server/block_processor.py b/server/block_processor.py index fe26f63..63e2b1a 100644 --- a/server/block_processor.py +++ b/server/block_processor.py @@ -142,6 +142,11 @@ class BlockProcessor(server.db.DB): def __init__(self, env, controller, daemon): super().__init__(env) + + # An incomplete compaction needs to be cancelled otherwise + # restarting it will corrupt the history + self.cancel_history_compaction() + self.daemon = daemon self.controller = controller @@ -332,7 +337,7 @@ class BlockProcessor(server.db.DB): self.wall_time += now - self.last_flush self.last_flush = now self.last_flush_tx_count = self.tx_count - self.write_state(batch) + self.utxo_write_state(batch) def assert_flushed(self): '''Asserts state is fully flushed.''' diff --git a/server/db.py b/server/db.py index 4958c2b..b802fcf 100644 --- a/server/db.py +++ b/server/db.py @@ -60,6 +60,9 @@ class DB(util.LoggedClass): self.db_class = db_class(self.env.db_engine) self.logger.info('using {} for DB backend'.format(self.env.db_engine)) + # For history compaction + self.max_hist_row_entries = 12500 + self.utxo_db = None self.open_dbs() self.clean_db() @@ -134,6 +137,7 @@ class DB(util.LoggedClass): self.logger.info('height: {:,d}'.format(self.db_height)) self.logger.info('tip: {}'.format(hash_to_str(self.db_tip))) self.logger.info('tx count: {:,d}'.format(self.db_tx_count)) + self.logger.info('flush count: {:,d}'.format(self.flush_count)) if self.first_sync: self.logger.info('sync time so far: {}' .format(util.formatted_time(self.wall_time))) @@ -172,7 +176,7 @@ class DB(util.LoggedClass): self.wall_time = state['wall_time'] self.first_sync = state['first_sync'] - def write_state(self, batch): + def utxo_write_state(self, batch): '''Write (UTXO) state to the batch.''' state = { 'genesis': self.coin.GENESIS_HASH, @@ -194,7 +198,9 @@ class DB(util.LoggedClass): undo information. ''' if self.flush_count < self.utxo_flush_count: - raise self.DBError('DB corrupt: flush_count < utxo_flush_count') + # Might happen at end of compaction as both DBs cannot be + # updated atomically + self.utxo_flush_count = self.flush_count if self.flush_count > self.utxo_flush_count: self.clear_excess_history(self.utxo_flush_count) self.clear_excess_undo_info() @@ -417,7 +423,12 @@ class DB(util.LoggedClass): self.logger.info('deleted excess history entries') def write_history_state(self, batch): - state = {'flush_count': self.flush_count} + '''Write state to hist_db.''' + state = { + 'flush_count': self.flush_count, + 'comp_flush_count': self.comp_flush_count, + 'comp_cursor': self.comp_cursor, + } # History entries are not prefixed; the suffix \0\0 ensures we # look similar to other entries and aren't interfered with batch.put(b'state\0\0', repr(state).encode()) @@ -429,8 +440,12 @@ class DB(util.LoggedClass): if not isinstance(state, dict): raise self.DBError('failed reading state from history DB') self.flush_count = state['flush_count'] + self.comp_flush_count = state.get('comp_flush_count', -1) + self.comp_cursor = state.get('comp_cursor', -1) else: self.flush_count = 0 + self.comp_flush_count = -1 + self.comp_cursor = -1 def flush_history(self, history): self.flush_count += 1 @@ -495,3 +510,162 @@ class DB(util.LoggedClass): ''' for tx_num in self.get_history_txnums(hashX, limit): yield self.fs_tx_hash(tx_num) + + # + # History compaction + # + + # comp_cursor is a cursor into compaction progress. + # -1: no compaction in progress + # 0-65535: Compaction in progress; all prefixes < comp_cursor have + # been compacted, and later ones have not. + # 65536: compaction complete in-memory but not flushed + # + # comp_flush_count applies during compaction, and is a flush count + # for history with prefix < comp_cursor. flush_count applies + # to still uncompacted history. It is -1 when no compaction is + # taking place. Key suffixes up to and including comp_flush_count + # are used, so a parallel history flush must first increment this + # + # When compaction is complete and the final flush takes place, + # flush_count is reset to comp_flush_count, and comp_flush_count to -1 + + def _flush_compaction(self, cursor, write_items, keys_to_delete): + '''Flush a single compaction pass as a batch.''' + # Update compaction state + if cursor == 65536: + self.flush_count = self.comp_flush_count + self.comp_cursor = -1 + self.comp_flush_count = -1 + else: + self.comp_cursor = cursor + + # History DB. Flush compacted history and updated state + with self.hist_db.write_batch() as batch: + # Important: delete first! The keyspace may overlap. + for key in keys_to_delete: + batch.delete(key) + for key, value in write_items: + batch.put(key, value) + self.write_history_state(batch) + + # If compaction was completed also update the UTXO flush count + if cursor == 65536: + self.utxo_flush_count = self.flush_count + with self.utxo_db.write_batch() as batch: + self.utxo_write_state(batch) + + def _compact_hashX(self, hashX, hist_map, hist_list, + write_items, keys_to_delete): + '''Compres history for a hashX. hist_list is an ordered list of + the histories to be compressed.''' + # History entries (tx numbers) are 4 bytes each. Distribute + # over rows of up to 50KB in size. A fixed row size means + # future compactions will not need to update the first N - 1 + # rows. + max_row_size = self.max_hist_row_entries * 4 + full_hist = b''.join(hist_list) + nrows = (len(full_hist) + max_row_size - 1) // max_row_size + if nrows > 4: + self.log_info('hashX {} is large: {:,d} entries across {:,d} rows' + .format(hash_to_str(hashX), len(full_hist) // 4, + nrows)); + + # Find what history needs to be written, and what keys need to + # be deleted. Start by assuming all keys are to be deleted, + # and then remove those that are the same on-disk as when + # compacted. + write_size = 0 + keys_to_delete.update(hist_map) + for n, chunk in enumerate(util.chunks(full_hist, max_row_size)): + key = hashX + pack('>H', n) + if hist_map.get(key) == chunk: + keys_to_delete.remove(key) + else: + write_items.append((key, chunk)) + write_size += len(chunk) + + assert n + 1 == nrows + self.comp_flush_count = max(self.comp_flush_count, n) + + return write_size + + def _compact_prefix(self, prefix, write_items, keys_to_delete): + '''Compact all history entries for hashXs beginning with the + given prefix. Update keys_to_delete and write.''' + prior_hashX = None + hist_map = {} + hist_list = [] + + key_len = self.coin.HASHX_LEN + 2 + write_size = 0 + for key, hist in self.hist_db.iterator(prefix=prefix): + # Ignore non-history entries + if len(key) != key_len: + continue + hashX = key[:-2] + if hashX != prior_hashX and prior_hashX: + write_size += self._compact_hashX(prior_hashX, hist_map, + hist_list, write_items, + keys_to_delete) + hist_map.clear() + hist_list.clear() + prior_hashX = hashX + hist_map[key] = hist + hist_list.append(hist) + + if prior_hashX: + write_size += self._compact_hashX(prior_hashX, hist_map, hist_list, + write_items, keys_to_delete) + return write_size + + def _compact_history(self, limit): + '''Inner loop of history compaction. Loops until limit bytes have + been processed. + ''' + keys_to_delete = set() + write_items = [] # A list of (key, value) pairs + write_size = 0 + + # Loop over 2-byte prefixes + cursor = self.comp_cursor + while write_size < limit and cursor < 65536: + prefix = pack('>H', cursor) + write_size += self._compact_prefix(prefix, write_items, + keys_to_delete) + cursor += 1 + + max_rows = self.comp_flush_count + 1 + self._flush_compaction(cursor, write_items, keys_to_delete) + + self.log_info('history compaction: wrote {:,d} rows ({:.1f} MB), ' + 'removed {:,d} rows, largest: {:,d}, {:.1f}% complete' + .format(len(write_items), write_size / 1000000, + len(keys_to_delete), max_rows, + 100 * cursor / 65536)) + return write_size + + async def compact_history(self, loop): + '''Start a background history compaction and reset the flush count if + its getting high. + ''' + # Do nothing if during initial sync or if a compaction hasn't + # been initiated + if self.first_sync or self.comp_cursor == -1: + return + + self.comp_flush_count = max(self.comp_flush_count, 1) + limit = 50 * 1000 * 1000 + + while self.comp_cursor != -1: + locked = self.semaphore.locked + if self.semaphore.locked: + self.log_info('compact_history: waiting on semaphore...') + with await self.semaphore: + await loop.run_in_executor(None, self._compact_history, limit) + + def cancel_history_compaction(self): + if self.comp_cursor != -1: + self.logger.warning('cancelling in-progress history compaction') + self.comp_flush_count = -1 + self.comp_cursor = -1 diff --git a/tests/server/test_compaction.py b/tests/server/test_compaction.py new file mode 100644 index 0000000..d1974b5 --- /dev/null +++ b/tests/server/test_compaction.py @@ -0,0 +1,131 @@ +# Test of compaction code in server/db.py + +import array +from collections import defaultdict +from os import environ, urandom +from struct import pack +import random + +from lib.hash import hash_to_str +from server.env import Env +from server.db import DB + + +def create_histories(db, hashX_count=100): + '''Creates a bunch of random transaction histories, and write them + to disk in a series of small flushes.''' + hashXs = [urandom(db.coin.HASHX_LEN) for n in range(hashX_count)] + mk_array = lambda : array.array('I') + histories = {hashX : mk_array() for hashX in hashXs} + this_history = defaultdict(mk_array) + tx_num = 0 + while hashXs: + hash_indexes = set(random.randrange(len(hashXs)) + for n in range(1 + random.randrange(4))) + for index in hash_indexes: + histories[hashXs[index]].append(tx_num) + this_history[hashXs[index]].append(tx_num) + + tx_num += 1 + # Occasionally flush and drop a random hashX if non-empty + if random.random() < 0.1: + db.flush_history(this_history) + this_history.clear() + index = random.randrange(0, len(hashXs)) + if histories[hashXs[index]]: + del hashXs[index] + + return histories + + +def check_hashX_compaction(db): + db.max_hist_row_entries = 40 + row_size = db.max_hist_row_entries * 4 + full_hist = array.array('I', range(100)).tobytes() + hashX = urandom(db.coin.HASHX_LEN) + pairs = ((1, 20), (26, 50), (56, 30)) + + cum = 0 + hist_list = [] + hist_map = {} + for flush_count, count in pairs: + key = hashX + pack('>H', flush_count) + hist = full_hist[cum * 4: (cum+count) * 4] + hist_map[key] = hist + hist_list.append(hist) + cum += count + + write_items = [] + keys_to_delete = set() + write_size = db._compact_hashX(hashX, hist_map, hist_list, + write_items, keys_to_delete) + # Check results for sanity + assert write_size == len(full_hist) + assert len(write_items) == 3 + assert len(keys_to_delete) == 3 + assert len(hist_map) == len(pairs) + for n, item in enumerate(write_items): + assert item == (hashX + pack('>H', n), + full_hist[n * row_size: (n + 1) * row_size]) + for flush_count, count in pairs: + assert hashX + pack('>H', flush_count) in keys_to_delete + + # Check re-compaction is null + hist_map = {key: value for key, value in write_items} + hist_list = [value for key, value in write_items] + write_items.clear() + keys_to_delete.clear() + write_size = db._compact_hashX(hashX, hist_map, hist_list, + write_items, keys_to_delete) + assert write_size == 0 + assert len(write_items) == 0 + assert len(keys_to_delete) == 0 + assert len(hist_map) == len(pairs) + + # Check re-compaction adding a single tx writes the one row + hist_list[-1] += array.array('I', [100]).tobytes() + write_size = db._compact_hashX(hashX, hist_map, hist_list, + write_items, keys_to_delete) + assert write_size == len(hist_list[-1]) + assert write_items == [(hashX + pack('>H', 2), hist_list[-1])] + assert len(keys_to_delete) == 1 + assert write_items[0][0] in keys_to_delete + assert len(hist_map) == len(pairs) + + +def check_written(db, histories): + for hashX, hist in histories.items(): + db_hist = array.array('I', db.get_history_txnums(hashX, limit=None)) + assert hist == db_hist + +def compact_history(db): + '''Synchronously compact the DB history.''' + db.first_sync = False + db.comp_cursor = 0 + + db.comp_flush_count = max(db.comp_flush_count, 1) + limit = 5 * 1000 + + write_size = 0 + while db.comp_cursor != -1: + write_size += db._compact_history(limit) + assert write_size != 0 + +def run_test(db_dir): + environ.clear() + environ['DB_DIRECTORY'] = db_dir + environ['DAEMON_URL'] = '' + env = Env() + db = DB(env) + # Test abstract compaction + check_hashX_compaction(db) + # Now test in with random data + histories = create_histories(db) + check_written(db, histories) + compact_history(db) + check_written(db, histories) + +def test_compaction(tmpdir): + db_dir = str(tmpdir) + print('Temp dir: {}'.format(db_dir)) + run_test(db_dir) From 04df5e9079c4a554b135fb379d4bbfeb9d2e978a Mon Sep 17 00:00:00 2001 From: followtheart Date: Tue, 20 Jun 2017 18:37:15 +0800 Subject: [PATCH 101/117] Add Dockerfile Add Dockerfile reference --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 21de0d2..84b75d5 100644 --- a/README.rst +++ b/README.rst @@ -18,9 +18,12 @@ Getting Started See `docs/HOWTO.rst`_. There is also an `installer`_ available that simplifies the installation on various Linux-based distributions. +There is also an `Dockerfile`_ available . .. _installer: https://github.com/bauerj/electrumx-installer +.. _Dockerfile: https://github.com/followtheart/electrumx-docker + Features ======== From 5cc3973eff6767b045eec0ba244f34cd94c00046 Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 4 Jul 2017 03:37:56 +0200 Subject: [PATCH 102/117] HOWTO: fixed paths --- docs/HOWTO.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/HOWTO.rst b/docs/HOWTO.rst index 7a58f9b..7748a84 100644 --- a/docs/HOWTO.rst +++ b/docs/HOWTO.rst @@ -403,6 +403,6 @@ copy of your certificate and key in case you need to restore them. .. _`pylru`: https://pypi.python.org/pypi/pylru .. _`IRC`: https://pypi.python.org/pypi/irc .. _`x11_hash`: https://pypi.python.org/pypi/x11_hash -.. _`contrib/python3.6/python-3.6.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/contrib/python3.6/python-3.6.sh -.. _`contrib/raspberrypi3/install_electrumx.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/contrib/raspberrypi3/install_electrumx.sh -.. _`contrib/raspberrypi3/run_electrumx.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/contrib/raspberrypi3/run_electrumx.sh +.. _`contrib/python3.6/python-3.6.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/python3.6/python-3.6.sh +.. _`contrib/raspberrypi3/install_electrumx.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/raspberrypi3/install_electrumx.sh +.. _`contrib/raspberrypi3/run_electrumx.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/raspberrypi3/run_electrumx.sh From 5f8222efb31392bcfa696f0538fa8c1b04ac1868 Mon Sep 17 00:00:00 2001 From: emilrus Date: Sat, 8 Jul 2017 11:10:29 +0300 Subject: [PATCH 103/117] Replace envuidgid with setuidgid and make run files executable (#203) * envuidgid only sets environment variables $UID and $GID causing electrum_server to fail at https://github.com/kyuupichan/electrumx/blob/master/electrumx_server.py#L35-L37 Replaced envuidgid with setuidgit which runs electrum_server with the specified uid/gid * Make run files executable --- contrib/daemontools/log/run | 0 contrib/daemontools/run | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 contrib/daemontools/log/run mode change 100644 => 100755 contrib/daemontools/run diff --git a/contrib/daemontools/log/run b/contrib/daemontools/log/run old mode 100644 new mode 100755 diff --git a/contrib/daemontools/run b/contrib/daemontools/run old mode 100644 new mode 100755 index ccb9d3c..773171b --- a/contrib/daemontools/run +++ b/contrib/daemontools/run @@ -1,3 +1,3 @@ #!/bin/sh echo "Launching ElectrumX server..." -exec 2>&1 envdir ./env /bin/sh -c 'envuidgid $USERNAME python3 $ELECTRUMX' +exec 2>&1 envdir ./env /bin/sh -c 'setuidgid $USERNAME python3 $ELECTRUMX' From 858bac217da10ad77fbb8f80801c4d43aac90f8d Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Mon, 10 Jul 2017 14:12:27 +0900 Subject: [PATCH 104/117] Move DB UTXO code into one place. --- server/block_processor.py | 2 +- server/db.py | 204 +++++++++++++++++++------------------- 2 files changed, 104 insertions(+), 102 deletions(-) diff --git a/server/block_processor.py b/server/block_processor.py index 63e2b1a..157d226 100644 --- a/server/block_processor.py +++ b/server/block_processor.py @@ -337,7 +337,7 @@ class BlockProcessor(server.db.DB): self.wall_time += now - self.last_flush self.last_flush = now self.last_flush_tx_count = self.tx_count - self.utxo_write_state(batch) + self.write_utxo_state(batch) def assert_flushed(self): '''Asserts state is fully flushed.''' diff --git a/server/db.py b/server/db.py index b802fcf..ef6963f 100644 --- a/server/db.py +++ b/server/db.py @@ -142,54 +142,6 @@ class DB(util.LoggedClass): self.logger.info('sync time so far: {}' .format(util.formatted_time(self.wall_time))) - def read_utxo_state(self): - state = self.utxo_db.get(b'state') - if not state: - self.db_height = -1 - self.db_tx_count = 0 - self.db_tip = b'\0' * 32 - self.db_version = max(self.DB_VERSIONS) - self.utxo_flush_count = 0 - self.wall_time = 0 - self.first_sync = True - else: - state = ast.literal_eval(state.decode()) - if not isinstance(state, dict): - raise self.DBError('failed reading state from DB') - self.db_version = state['db_version'] - if self.db_version not in self.DB_VERSIONS: - raise self.DBError('your DB version is {} but this software ' - 'only handles versions {}' - .format(self.db_version, self.DB_VERSIONS)) - # backwards compat - genesis_hash = state['genesis'] - if isinstance(genesis_hash, bytes): - genesis_hash = genesis_hash.decode() - if genesis_hash != self.coin.GENESIS_HASH: - raise self.DBError('DB genesis hash {} does not match coin {}' - .format(state['genesis_hash'], - self.coin.GENESIS_HASH)) - self.db_height = state['height'] - self.db_tx_count = state['tx_count'] - self.db_tip = state['tip'] - self.utxo_flush_count = state['utxo_flush_count'] - self.wall_time = state['wall_time'] - self.first_sync = state['first_sync'] - - def utxo_write_state(self, batch): - '''Write (UTXO) state to the batch.''' - state = { - 'genesis': self.coin.GENESIS_HASH, - 'height': self.db_height, - 'tx_count': self.db_tx_count, - 'tip': self.db_tip, - 'utxo_flush_count': self.utxo_flush_count, - 'wall_time': self.wall_time, - 'first_sync': self.first_sync, - 'db_version': self.db_version, - } - batch.put(b'state', repr(state).encode()) - def clean_db(self): '''Clean out stale DB items. @@ -299,6 +251,93 @@ class DB(util.LoggedClass): assert isinstance(limit, int) and limit >= 0 return limit + # -- Undo information + + def min_undo_height(self, max_height): + '''Returns a height from which we should store undo info.''' + return max_height - self.env.reorg_limit + 1 + + def undo_key(self, height): + '''DB key for undo information at the given height.''' + return b'U' + pack('>I', height) + + def read_undo_info(self, height): + '''Read undo information from a file for the current height.''' + return self.utxo_db.get(self.undo_key(height)) + + def flush_undo_infos(self, batch_put, undo_infos): + '''undo_infos is a list of (undo_info, height) pairs.''' + for undo_info, height in undo_infos: + batch_put(self.undo_key(height), b''.join(undo_info)) + + def clear_excess_undo_info(self): + '''Clear excess undo info. Only most recent N are kept.''' + prefix = b'U' + min_height = self.min_undo_height(self.db_height) + keys = [] + for key, hist in self.utxo_db.iterator(prefix=prefix): + height, = unpack('>I', key[-4:]) + if height >= min_height: + break + keys.append(key) + + if keys: + with self.utxo_db.write_batch() as batch: + for key in keys: + batch.delete(key) + self.logger.info('deleted {:,d} stale undo entries' + .format(len(keys))) + + # -- UTXO database + + def read_utxo_state(self): + state = self.utxo_db.get(b'state') + if not state: + self.db_height = -1 + self.db_tx_count = 0 + self.db_tip = b'\0' * 32 + self.db_version = max(self.DB_VERSIONS) + self.utxo_flush_count = 0 + self.wall_time = 0 + self.first_sync = True + else: + state = ast.literal_eval(state.decode()) + if not isinstance(state, dict): + raise self.DBError('failed reading state from DB') + self.db_version = state['db_version'] + if self.db_version not in self.DB_VERSIONS: + raise self.DBError('your DB version is {} but this software ' + 'only handles versions {}' + .format(self.db_version, self.DB_VERSIONS)) + # backwards compat + genesis_hash = state['genesis'] + if isinstance(genesis_hash, bytes): + genesis_hash = genesis_hash.decode() + if genesis_hash != self.coin.GENESIS_HASH: + raise self.DBError('DB genesis hash {} does not match coin {}' + .format(state['genesis_hash'], + self.coin.GENESIS_HASH)) + self.db_height = state['height'] + self.db_tx_count = state['tx_count'] + self.db_tip = state['tip'] + self.utxo_flush_count = state['utxo_flush_count'] + self.wall_time = state['wall_time'] + self.first_sync = state['first_sync'] + + def write_utxo_state(self, batch): + '''Write (UTXO) state to the batch.''' + state = { + 'genesis': self.coin.GENESIS_HASH, + 'height': self.db_height, + 'tx_count': self.db_tx_count, + 'tip': self.db_tip, + 'utxo_flush_count': self.utxo_flush_count, + 'wall_time': self.wall_time, + 'first_sync': self.first_sync, + 'db_version': self.db_version, + } + batch.put(b'state', repr(state).encode()) + def get_balance(self, hashX): '''Returns the confirmed balance of an address.''' return sum(utxo.value for utxo in self.get_utxos(hashX, limit=None)) @@ -322,24 +361,6 @@ class DB(util.LoggedClass): tx_hash, height = self.fs_tx_hash(tx_num) yield UTXO(tx_num, tx_pos, tx_hash, height, value) - def db_hashX(self, tx_hash, idx_packed): - '''Return (hashX, tx_num_packed) for the given TXO. - - Both are None if not found.''' - # Key: b'h' + compressed_tx_hash + tx_idx + tx_num - # Value: hashX - prefix = b'h' + tx_hash[:4] + idx_packed - - # Find which entry, if any, the TX_HASH matches. - for db_key, hashX in self.utxo_db.iterator(prefix=prefix): - tx_num_packed = db_key[-4:] - tx_num, = unpack('I', height) - - def read_undo_info(self, height): - '''Read undo information from a file for the current height.''' - return self.utxo_db.get(self.undo_key(height)) + def _db_hashX(self, tx_hash, idx_packed): + '''Return (hashX, tx_num_packed) for the given TXO. - def flush_undo_infos(self, batch_put, undo_infos): - '''undo_infos is a list of (undo_info, height) pairs.''' - for undo_info, height in undo_infos: - batch_put(self.undo_key(height), b''.join(undo_info)) + Both are None if not found.''' + # Key: b'h' + compressed_tx_hash + tx_idx + tx_num + # Value: hashX + prefix = b'h' + tx_hash[:4] + idx_packed - def clear_excess_undo_info(self): - '''Clear excess undo info. Only most recent N are kept.''' - prefix = b'U' - min_height = self.min_undo_height(self.db_height) - keys = [] - for key, hist in self.utxo_db.iterator(prefix=prefix): - height, = unpack('>I', key[-4:]) - if height >= min_height: - break - keys.append(key) + # Find which entry, if any, the TX_HASH matches. + for db_key, hashX in self.utxo_db.iterator(prefix=prefix): + tx_num_packed = db_key[-4:] + tx_num, = unpack(' Date: Mon, 10 Jul 2017 15:53:23 +0900 Subject: [PATCH 105/117] Add comprehensive tests of lib/hash.py --- lib/hash.py | 10 +++---- tests/lib/test_hash.py | 68 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 tests/lib/test_hash.py diff --git a/lib/hash.py b/lib/hash.py index 4aba0fe..bd23cf4 100644 --- a/lib/hash.py +++ b/lib/hash.py @@ -34,13 +34,11 @@ from lib.util import bytes_to_int, int_to_bytes def sha256(x): '''Simple wrapper of hashlib sha256.''' - assert isinstance(x, (bytes, bytearray, memoryview)) return hashlib.sha256(x).digest() def ripemd160(x): '''Simple wrapper of hashlib ripemd160.''' - assert isinstance(x, (bytes, bytearray, memoryview)) h = hashlib.new('ripemd160') h.update(x) return h.digest() @@ -63,13 +61,15 @@ def hash160(x): return ripemd160(sha256(x)) -def hash_to_str(x): +def hash_to_hex_str(x): '''Convert a big-endian binary hash to displayed hex string. Display form of a binary hash is reversed and converted to hex. ''' return bytes(reversed(x)).hex() +# Temporary +hash_to_str = hash_to_hex_str def hex_str_to_hash(x): '''Convert a displayed hex string to a binary hash.''' @@ -98,7 +98,7 @@ class Base58(object): def decode(txt): """Decodes txt into a big-endian bytearray.""" if not isinstance(txt, str): - raise Base58Error('a string is required') + raise TypeError('a string is required') if not txt: raise Base58Error('string cannot be empty') @@ -151,7 +151,5 @@ class Base58(object): def encode_check(payload): """Encodes a payload bytearray (which includes the version byte(s)) into a Base58Check string.""" - assert isinstance(payload, (bytes, bytearray, memoryview)) - be_bytes = payload + double_sha256(payload)[:4] return Base58.encode(be_bytes) diff --git a/tests/lib/test_hash.py b/tests/lib/test_hash.py new file mode 100644 index 0000000..f1b9d46 --- /dev/null +++ b/tests/lib/test_hash.py @@ -0,0 +1,68 @@ +# +# Tests of lib/hash.py +# + +import pytest + +import lib.hash as lib_hash + + +def test_sha256(): + assert lib_hash.sha256(b'sha256') == b'][\t\xf6\xdc\xb2\xd5:_\xff\xc6\x0cJ\xc0\xd5_\xab\xdfU`i\xd6c\x15E\xf4*\xa6\xe3P\x0f.' + with pytest.raises(TypeError): + lib_hash.sha256('sha256') + +def ripemd160(x): + assert lib_hash.ripemd160(b'ripemd160') == b'\x903\x91\xa1\xc0I\x9e\xc8\xdf\xb5\x1aSK\xa5VW\xf9|W\xd5' + with pytest.raises(TypeError): + lib_hash.ripemd160('ripemd160') + +def test_double_sha256(): + assert lib_hash.double_sha256(b'double_sha256') == b'ksn\x8e\xb7\xb9\x0f\xf6\xd9\xad\x88\xd9#\xa1\xbcU(j1Bx\xce\xd5;s\xectL\xe7\xc5\xb4\x00' + +def test_hmac_sha512(): + assert lib_hash.hmac_sha512(b'key', b'message') == b"\xe4w8M|\xa2)\xdd\x14&\xe6Kc\xeb\xf2\xd3n\xbdm~f\x9ag5BNr\xeal\x01\xd3\xf8\xb5n\xb3\x9c6\xd8#/T'\x99\x9b\x8d\x1a?\x9c\xd1\x12\x8f\xc6\x9fMu\xb44!h\x10\xfa6~\x98" + +def test_hash160(): + assert lib_hash.hash160(b'hash_160') == b'\xb3\x96\x94\xfc\x978R\xa7)XqY\xbb\xdc\xeb\xac\xa7%\xb8$' + +def test_hash_to_hex_str(): + assert lib_hash.hash_to_hex_str(b'hash_to_str') == '7274735f6f745f68736168' + +def test_hex_str_to_hash(): + assert lib_hash.hex_str_to_hash('7274735f6f745f68736168') == b'hash_to_str' + +def test_Base58_char_value(): + chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' + for value, c in enumerate(chars): + assert lib_hash.Base58.char_value(c) == value + for c in (' ', 'I', '0', 'l', 'O'): + with pytest.raises(lib_hash.Base58Error): + lib_hash.Base58.char_value(c) + +def test_Base58_decode(): + with pytest.raises(TypeError): + lib_hash.Base58.decode(b'foo') + with pytest.raises(lib_hash.Base58Error): + lib_hash.Base58.decode('') + assert lib_hash.Base58.decode('123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz') == b'\x00\x01\x11\xd3\x8e_\xc9\x07\x1f\xfc\xd2\x0bJv<\xc9\xaeO%+\xb4\xe4\x8f\xd6j\x83^%*\xda\x93\xffH\rm\xd4=\xc6*d\x11U\xa5' + assert lib_hash.Base58.decode('3i37NcgooY8f1S') == b'0123456789' + +def test_Base58_encode(): + with pytest.raises(TypeError): + lib_hash.Base58.encode('foo') + assert lib_hash.Base58.encode(b'') == '' + assert lib_hash.Base58.encode(b'\0') == '1' + assert lib_hash.Base58.encode(b'0123456789') == '3i37NcgooY8f1S' + +def test_Base58_decode_check(): + with pytest.raises(TypeError): + lib_hash.Base58.decode_check(b'foo') + assert lib_hash.Base58.decode_check('4t9WKfuAB8') == b'foo' + with pytest.raises(lib_hash.Base58Error): + lib_hash.Base58.decode_check('4t9WKfuAB9') + +def test_Base58_encode_check(): + with pytest.raises(TypeError): + lib_hash.Base58.encode_check('foo') + assert lib_hash.Base58.encode_check(b'foo') == '4t9WKfuAB8' From 9dfaedc7271c4f1dc7f360b7d39526ebf455e06e Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 11 Jul 2017 19:54:00 +0900 Subject: [PATCH 106/117] Add bip32.py and tests. --- lib/coins.py | 61 ++---- tests/wallet/test_bip32.py | 367 +++++++++++++++++++++++++++++++++++++ wallet/bip32.py | 307 +++++++++++++++++++++++++++++++ 3 files changed, 690 insertions(+), 45 deletions(-) create mode 100644 tests/wallet/test_bip32.py create mode 100644 wallet/bip32.py diff --git a/lib/coins.py b/lib/coins.py index 82c48d6..40caa28 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -42,8 +42,9 @@ from lib.script import ScriptPubKey from lib.tx import Deserializer, DeserializerSegWit, DeserializerAuxPow, \ DeserializerZcash, DeserializerTxTime, DeserializerReddcoin from server.block_processor import BlockProcessor -from server.daemon import Daemon, LegacyRPCDaemon -from server.session import ElectrumX +from server.daemon import Daemon, DashDaemon, LegacyRPCDaemon +from server.session import ElectrumX, DashElectrumX + Block = namedtuple("Block", "header transactions") @@ -67,6 +68,8 @@ class Coin(object): DESERIALIZER = Deserializer DAEMON = Daemon BLOCK_PROCESSOR = BlockProcessor + XPUB_VERBYTES = bytes('????', 'utf-8') + XPRV_VERBYTES = bytes('????', 'utf-8') IRC_PREFIX = None IRC_SERVER = "irc.freenode.net" IRC_PORT = 6667 @@ -153,7 +156,7 @@ class Coin(object): def lookup_xverbytes(verbytes): '''Return a (is_xpub, coin_class) pair given xpub/xprv verbytes.''' # Order means BTC testnet will override NMC testnet - for coin in Coin.coin_classes(): + for coin in util.subclasses(Coin): if verbytes == coin.XPUB_VERBYTES: return True, coin if verbytes == coin.XPRV_VERBYTES: @@ -229,7 +232,7 @@ class Coin(object): raise CoinError('invalid address: {}'.format(address)) @classmethod - def prvkey_WIF(cls, privkey_bytes, compressed): + def privkey_WIF(cls, privkey_bytes, compressed): '''Return the private key encoded in Wallet Import Format.''' payload = bytearray(cls.WIF_BYTE) + privkey_bytes if compressed: @@ -299,10 +302,7 @@ class Coin(object): } -class CoinAuxPow(Coin): - # Set NAME and NET to avoid exception in Coin::lookup_coin_class - NAME = '' - NET = '' +class AuxPowMixin(object): STATIC_BLOCK_HEADERS = False DESERIALIZER = DeserializerAuxPow @@ -411,8 +411,8 @@ class Litecoin(Coin): NAME = "Litecoin" SHORTNAME = "LTC" NET = "mainnet" - XPUB_VERBYTES = bytes.fromhex("0488b21e") - XPRV_VERBYTES = bytes.fromhex("0488ade4") + XPUB_VERBYTES = bytes.fromhex("019d9cfe") + XPRV_VERBYTES = bytes.fromhex("019da462") P2PKH_VERBYTE = bytes.fromhex("30") P2SH_VERBYTES = [bytes.fromhex("32"), bytes.fromhex("05")] WIF_BYTE = bytes.fromhex("b0") @@ -437,8 +437,8 @@ class Litecoin(Coin): class LitecoinTestnet(Litecoin): SHORTNAME = "XLT" NET = "testnet" - XPUB_VERBYTES = bytes.fromhex("043587cf") - XPRV_VERBYTES = bytes.fromhex("04358394") + XPUB_VERBYTES = bytes.fromhex("0436ef7d") + XPRV_VERBYTES = bytes.fromhex("0436f6e1") P2PKH_VERBYTE = bytes.fromhex("6f") P2SH_VERBYTES = [bytes.fromhex("3a"), bytes.fromhex("c4")] WIF_BYTE = bytes.fromhex("ef") @@ -456,12 +456,10 @@ class LitecoinTestnet(Litecoin): ] -class Viacoin(CoinAuxPow): +class Viacoin(AuxPowMixin, Coin): NAME="Viacoin" SHORTNAME = "VIA" NET = "mainnet" - XPUB_VERBYTES = bytes.fromhex("0488B21E") - XPRV_VERBYTES = bytes.fromhex("0488ADE4") P2PKH_VERBYTE = bytes.fromhex("47") P2SH_VERBYTES = [bytes.fromhex("21")] WIF_BYTE = bytes.fromhex("c7") @@ -485,8 +483,6 @@ class Viacoin(CoinAuxPow): class ViacoinTestnet(Viacoin): SHORTNAME = "TVI" NET = "testnet" - XPUB_VERBYTES = bytes.fromhex("043587CF") - XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("7f") P2SH_VERBYTES = [bytes.fromhex("c4")] WIF_BYTE = bytes.fromhex("ff") @@ -505,7 +501,7 @@ class ViacoinTestnetSegWit(ViacoinTestnet): # Source: namecoin.org -class Namecoin(CoinAuxPow): +class Namecoin(AuxPowMixin, Coin): NAME = "Namecoin" SHORTNAME = "NMC" NET = "mainnet" @@ -527,8 +523,6 @@ class NamecoinTestnet(Namecoin): NAME = "Namecoin" SHORTNAME = "XNM" NET = "testnet" - XPUB_VERBYTES = bytes.fromhex("043587cf") - XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("6f") P2SH_VERBYTES = [bytes.fromhex("c4")] WIF_BYTE = bytes.fromhex("ef") @@ -536,7 +530,7 @@ class NamecoinTestnet(Namecoin): 'a4cccff2a4767a8eee39c11db367b008') -class Dogecoin(CoinAuxPow): +class Dogecoin(AuxPowMixin, Coin): NAME = "Dogecoin" SHORTNAME = "DOGE" NET = "mainnet" @@ -559,8 +553,6 @@ class DogecoinTestnet(Dogecoin): NAME = "Dogecoin" SHORTNAME = "XDT" NET = "testnet" - XPUB_VERBYTES = bytes.fromhex("043587cf") - XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("71") P2SH_VERBYTES = [bytes.fromhex("c4")] WIF_BYTE = bytes.fromhex("f1") @@ -570,8 +562,6 @@ class DogecoinTestnet(Dogecoin): # Source: https://github.com/dashpay/dash class Dash(Coin): - from server.session import DashElectrumX - from server.daemon import DashDaemon NAME = "Dash" SHORTNAME = "DASH" NET = "mainnet" @@ -627,12 +617,10 @@ class DashTestnet(Dash): ] -class Argentum(CoinAuxPow): +class Argentum(AuxPowMixin, Coin): NAME = "Argentum" SHORTNAME = "ARG" NET = "mainnet" - XPUB_VERBYTES = bytes.fromhex("0488b21e") - XPRV_VERBYTES = bytes.fromhex("0488ade4") P2PKH_VERBYTE = bytes.fromhex("17") P2SH_VERBYTES = [bytes.fromhex("05")] WIF_BYTE = bytes.fromhex("97") @@ -649,8 +637,6 @@ class Argentum(CoinAuxPow): class ArgentumTestnet(Argentum): SHORTNAME = "XRG" NET = "testnet" - XPUB_VERBYTES = bytes.fromhex("043587cf") - XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("6f") P2SH_VERBYTES = [bytes.fromhex("c4")] WIF_BYTE = bytes.fromhex("ef") @@ -661,8 +647,6 @@ class DigiByte(Coin): NAME = "DigiByte" SHORTNAME = "DGB" NET = "mainnet" - XPUB_VERBYTES = bytes.fromhex("0488b21e") - XPRV_VERBYTES = bytes.fromhex("0488ade4") P2PKH_VERBYTE = bytes.fromhex("1E") P2SH_VERBYTES = [bytes.fromhex("05")] WIF_BYTE = bytes.fromhex("80") @@ -679,8 +663,6 @@ class DigiByte(Coin): class DigiByteTestnet(DigiByte): NET = "testnet" - XPUB_VERBYTES = bytes.fromhex("043587cf") - XPRV_VERBYTES = bytes.fromhex("04358394") P2PKH_VERBYTE = bytes.fromhex("6f") P2SH_VERBYTES = [bytes.fromhex("c4")] WIF_BYTE = bytes.fromhex("ef") @@ -696,8 +678,6 @@ class FairCoin(Coin): NAME = "FairCoin" SHORTNAME = "FAIR" NET = "mainnet" - XPUB_VERBYTES = bytes.fromhex("0488b21e") - XPRV_VERBYTES = bytes.fromhex("0488ade4") P2PKH_VERBYTE = bytes.fromhex("5f") P2SH_VERBYTES = [bytes.fromhex("24")] WIF_BYTE = bytes.fromhex("df") @@ -745,8 +725,6 @@ class Zcash(Coin): NAME = "Zcash" SHORTNAME = "ZEC" NET = "mainnet" - XPUB_VERBYTES = bytes.fromhex("0488b21e") - XPRV_VERBYTES = bytes.fromhex("0488ade4") P2PKH_VERBYTE = bytes.fromhex("1CB8") P2SH_VERBYTES = [bytes.fromhex("1CBD")] WIF_BYTE = bytes.fromhex("80") @@ -789,9 +767,6 @@ class Einsteinium(Coin): NAME = "Einsteinium" SHORTNAME = "EMC2" NET = "mainnet" - # TODO add correct values for XPUB, XPRIV - XPUB_VERBYTES = bytes.fromhex("0488b21e") - XPRV_VERBYTES = bytes.fromhex("0488ade4") P2PKH_VERBYTE = bytes.fromhex("21") P2SH_VERBYTES = [bytes.fromhex("05")] WIF_BYTE = bytes.fromhex("a1") @@ -810,8 +785,6 @@ class Blackcoin(Coin): NAME = "Blackcoin" SHORTNAME = "BLK" NET = "mainnet" - XPUB_VERBYTES = bytes.fromhex("0488B21E") - XPRV_VERBYTES = bytes.fromhex("0488ADE4") P2PKH_VERBYTE = bytes.fromhex("19") P2SH_VERBYTES = [bytes.fromhex("55")] WIF_BYTE = bytes.fromhex("99") @@ -866,8 +839,6 @@ class Reddcoin(Coin): NAME = "Reddcoin" SHORTNAME = "RDD" NET = "mainnet" - XPUB_VERBYTES = bytes.fromhex("0488B21E") - XPRV_VERBYTES = bytes.fromhex("0488ADE4") P2PKH_VERBYTE = bytes.fromhex("3d") P2SH_VERBYTES = [bytes.fromhex("05")] WIF_BYTE = bytes.fromhex("bd") diff --git a/tests/wallet/test_bip32.py b/tests/wallet/test_bip32.py new file mode 100644 index 0000000..0fd1eac --- /dev/null +++ b/tests/wallet/test_bip32.py @@ -0,0 +1,367 @@ +# +# Tests of wallet/bip32.py +# + +import pytest + +import wallet.bip32 as bip32 +from lib.coins import Bitcoin, CoinError +from lib.hash import Base58 + + +MXPRV = 'xprv9s21ZrQH143K2gMVrSwwojnXigqHgm1khKZGTCm7K8w4PmuDEUrudk11ZBxhGPUiUeVcrfGLoZmt8rFNRDLp18jmKMcVma89z7PJd2Vn7R9' +MPRIVKEY = b';\xf4\xbfH\xd20\xea\x94\x01_\x10\x1b\xc3\xb0\xff\xc9\x17$?K\x02\xe5\x82R\xe5\xb3A\xdb\x87&E\x00' +MXPUB = 'xpub661MyMwAqRbcFARxxUUxAsjGGifn6Djc4YUsFbAisUU3GaEMn2BABYKVQTHrDtwvSfgY2bK8aFGyCNmB52SKjkFGP18sSRTNn1sCeez7Utd' + +mpubkey, mpubcoin = bip32.from_extended_key_string(MXPUB) +mprivkey, mprivcoin = bip32.from_extended_key_string(MXPRV) + + +def test_from_extended_key(): + # Tests the failure modes of from_extended_key. + with pytest.raises(TypeError): + bip32._from_extended_key('') + with pytest.raises(ValueError): + bip32._from_extended_key(b'') + with pytest.raises(CoinError): + bip32._from_extended_key(bytes(78)) + # Invalid prefix byte + raw = Base58.decode_check(MXPRV) + with pytest.raises(ValueError): + bip32._from_extended_key(raw[:45] + b'\1' + raw[46:]) + + +class TestPubKey(object): + + def test_constructor(self): + cls = bip32.PubKey + raw_pubkey = b'\2' * 33 + chain_code = bytes(32) + + # Invalid constructions + with pytest.raises(TypeError): + cls(' ' * 33, chain_code, 0, 0) + with pytest.raises(ValueError): + cls(bytes(32), chain_code, -1, 0) + with pytest.raises(ValueError): + cls(bytes(33), chain_code, -1, 0) + with pytest.raises(ValueError): + cls(chain_code, chain_code, 0, 0) + with pytest.raises(TypeError): + cls(raw_pubkey, '0' * 32, 0, 0) + with pytest.raises(ValueError): + cls(raw_pubkey, bytes(31), 0, 0) + with pytest.raises(ValueError): + cls(raw_pubkey, chain_code, -1, 0) + with pytest.raises(ValueError): + cls(raw_pubkey, chain_code, 1 << 32, 0) + with pytest.raises(ValueError): + cls(raw_pubkey, chain_code, 0, -1) + with pytest.raises(ValueError): + cls(raw_pubkey, chain_code, 0, 256) + + # These are OK + cls(b'\2' + b'\2' * 32, chain_code, 0, 0) + cls(b'\3' + b'\2' * 32, chain_code, 0, 0) + cls(raw_pubkey, chain_code, (1 << 32) - 1, 0) + cls(raw_pubkey, chain_code, 0, 255) + cls(raw_pubkey, chain_code, 0, 255, mpubkey) + + # Construction from verifying key + dup = cls(mpubkey.verifying_key, chain_code, 0, 0) + assert mpubkey.ec_point() == dup.ec_point() + + # Construction from raw pubkey bytes + pubkey = mpubkey.pubkey_bytes + dup = cls(pubkey, chain_code, 0, 0) + assert mpubkey.ec_point() == dup.ec_point() + + # Construction from PubKey + with pytest.raises(TypeError): + cls(mpubkey, chain_code, 0, 0) + + def test_from_extended_key_string(self): + assert mpubcoin == Bitcoin + assert mpubkey.n == 0 + assert mpubkey.depth == 0 + assert mpubkey.parent is None + assert mpubkey.chain_code == b'>V\x83\x92`\r\x17\xb3"\xa6\x7f\xaf\xc0\x930\xf7\x1e\xdc\x12i\x9c\xe4\xc0,a\x1a\x04\xec\x16\x19\xaeK' + assert mpubkey.ec_point().x() == 44977109961578369385937116592536468905742111247230478021459394832226142714624 + + def test_extended_key_string(self): + # Implictly tests extended_key() + assert mpubkey.extended_key_string(Bitcoin) == MXPUB + chg_master = mpubkey.child(1) + chg5 = chg_master.child(5) + assert chg5.address(Bitcoin) == '1BsEFqGtcZnVBbPeimcfAFTitQdTLvUXeX' + assert chg5.extended_key_string(Bitcoin) == 'xpub6AzPNZ1SAS7zmSnj6gakQ6tAKPzRVdQzieL3eCnoeT3A89nJaJKuUYWoZuYp8xWhCs1gF9yXAwGg7zKYhvCfhk9jrb1bULhLkQCwtB1Nnn1' + + ext_key_base58 = chg5.extended_key_string(Bitcoin) + assert ext_key_base58 == 'xpub6AzPNZ1SAS7zmSnj6gakQ6tAKPzRVdQzieL3eCnoeT3A89nJaJKuUYWoZuYp8xWhCs1gF9yXAwGg7zKYhvCfhk9jrb1bULhLkQCwtB1Nnn1' + + # Check can recreate + dup, coin = bip32.from_extended_key_string(ext_key_base58) + assert coin is Bitcoin + assert dup.chain_code == chg5.chain_code + assert dup.n == chg5.n == 5 + assert dup.depth == chg5.depth == 2 + assert dup.ec_point() == chg5.ec_point() + + def test_child(self): + '''Test child derivations agree with Electrum.''' + rec_master = mpubkey.child(0) + assert rec_master.address(Bitcoin) == '18zW4D1Vxx9jVPGzsFzgXj8KrSLHt7w2cg' + chg_master = mpubkey.child(1) + assert chg_master.parent is mpubkey + assert chg_master.address(Bitcoin) == '1G8YpbkZd7bySHjpdQK3kMcHhc6BvHr5xy' + rec0 = rec_master.child(0) + assert rec0.address(Bitcoin) == '13nASW7rdE2dnSycrAP9VePhRmaLg9ziaw' + rec19 = rec_master.child(19) + assert rec19.address(Bitcoin) == '15QrXnPQ8aS8yCpA5tJkyvXfXpw8F8k3fB' + chg0 = chg_master.child(0) + assert chg0.parent is chg_master + assert chg0.address(Bitcoin) == '1L6fNSVhWjuMKNDigA99CweGEWtcqqhzDj' + + with pytest.raises(ValueError): + mpubkey.child(-1) + with pytest.raises(ValueError): + mpubkey.child(1 << 31) + # OK + mpubkey.child((1 << 31) - 1) + + def test_address(self): + assert mpubkey.address(Bitcoin) == '1ENCpq6mbb1KYcaodGG7eTpSpYvPnDjFmU' + + def test_identifier(self): + assert mpubkey.identifier() == b'\x92\x9c=\xb8\xd6\xe7\xebR\x90Td\x85\x1c\xa7\x0c\x8aE`\x87\xdd' + + def test_fingerprint(self): + assert mpubkey.fingerprint() == b'\x92\x9c=\xb8' + + def test_parent_fingerprint(self): + assert mpubkey.parent_fingerprint() == bytes(4) + child = mpubkey.child(0) + assert child.parent_fingerprint() == mpubkey.fingerprint() + + def test_pubkey_bytes(self): + # Also tests _exponent_to_bytes + pubkey = mpubkey.pubkey_bytes + assert pubkey == b'\x02cp$a\x18\xa7\xc2\x18\xfdUt\x96\xeb\xb2\xb0\x86-Y\xc6Hn\x88\xf8>\x07\xfd\x12\xce\x8a\x88\xfb\x00' + + +class TestPrivKey(object): + + def test_constructor(self): + # Includes full tests of _signing_key_from_privkey and + # _privkey_secret_exponent + cls = bip32.PrivKey + chain_code = bytes(32) + + # These are invalid + with pytest.raises(TypeError): + cls('0' * 32, chain_code, 0, 0) + with pytest.raises(ValueError): + cls(b'0' * 31, chain_code, 0, 0) + with pytest.raises(ValueError): + cls(MPRIVKEY, chain_code, -1, 0) + with pytest.raises(ValueError): + cls(MPRIVKEY, chain_code, 1 << 32, 0) + with pytest.raises(ValueError): + cls(MPRIVKEY, chain_code, 0, -1) + with pytest.raises(ValueError): + cls(MPRIVKEY, chain_code, 0, 256) + # Invalid exponents + with pytest.raises(ValueError): + cls(bip32._exponent_to_bytes(0), chain_code, 0, 0) + with pytest.raises(ValueError): + cls(bip32._exponent_to_bytes(cls.CURVE.order), chain_code, 0, 0) + + # These are good + cls(MPRIVKEY, chain_code, 0, 0) + cls(MPRIVKEY, chain_code, (1 << 32) - 1, 0) + cls(MPRIVKEY, chain_code, 0, 0) + cls(bip32._exponent_to_bytes(cls.CURVE.order - 1), chain_code, 0, 0) + privkey = cls(MPRIVKEY, chain_code, 0, 255) + + # Construction from signing key + dup = cls(privkey.signing_key, chain_code, 0, 0) + assert dup.ec_point() == privkey.ec_point() + + # Construction from PrivKey + with pytest.raises(TypeError): + cls(privkey, chain_code, 0, 0) + + def test_secret_exponent(self): + assert mprivkey.secret_exponent() == 27118888947022743980605817563635166434451957861641813930891160184742578898176 + + def test_identifier(self): + assert mprivkey.identifier() == mpubkey.identifier() + + def test_address(self): + assert mprivkey.address(Bitcoin) == mpubkey.address(Bitcoin) + + def test_fingerprint(self): + assert mprivkey.fingerprint() == mpubkey.fingerprint() + + def test_parent_fingerprint(self): + assert mprivkey.parent_fingerprint() == bytes(4) + child = mprivkey.child(0) + assert child.parent_fingerprint() == mprivkey.fingerprint() + + def test_from_extended_key_string(self): + # Also tests privkey_bytes and public_key + assert mprivcoin is Bitcoin + assert mprivkey.privkey_bytes == MPRIVKEY + assert mprivkey.ec_point() == mpubkey.ec_point() + assert mprivkey.public_key.chain_code == mpubkey.chain_code + assert mprivkey.public_key.n == mpubkey.n + assert mprivkey.public_key.depth == mpubkey.depth + + def test_extended_key_string(self): + # Also tests extended_key, WIF and privkey_bytes + assert mprivkey.extended_key_string(Bitcoin) == MXPRV + chg_master = mprivkey.child(1) + chg5 = chg_master.child(5) + assert chg5.WIF(Bitcoin) == 'L5kTYMuajTGWdYiMoD4V8k6LS4Bg3HFMA5UGTfxG9Wh7UKu9CHFC' + ext_key_base58 = chg5.extended_key_string(Bitcoin) + assert ext_key_base58 == 'xprv9x12y3UYL4ZhYxiFzf3k2xwRmN9w6Ah9MRQSqpPC67WBFMTA2m1evkCKidz7UYBa5i8QwxmU9Ju7giqEmcPRXKXwzgAJwssNeZNQLPT3LAY' + + # Check can recreate + dup, coin = bip32.from_extended_key_string(ext_key_base58) + assert coin is Bitcoin + assert dup.chain_code == chg5.chain_code + assert dup.n == chg5.n == 5 + assert dup.depth == chg5.depth == 2 + assert dup.ec_point() == chg5.ec_point() + + def test_child(self): + '''Test child derivations agree with Electrum.''' + # Also tests WIF, address + rec_master = mprivkey.child(0) + assert rec_master.address(Bitcoin) == '18zW4D1Vxx9jVPGzsFzgXj8KrSLHt7w2cg' + chg_master = mprivkey.child(1) + assert chg_master.parent is mprivkey + assert chg_master.address(Bitcoin) == '1G8YpbkZd7bySHjpdQK3kMcHhc6BvHr5xy' + rec0 = rec_master.child(0) + assert rec0.WIF(Bitcoin) == 'L2M6WWMdu3YfWxvLGF76HZgHCA6idwVQx5QL91vfdqeZi8XAgWkz' + rec19 = rec_master.child(19) + assert rec19.WIF(Bitcoin) == 'KwMHa1fynU2J2iBGCuBZxumM2qDXHe5tVPU9VecNGQv3UCqnET7X' + chg0 = chg_master.child(0) + assert chg0.parent is chg_master + assert chg0.WIF(Bitcoin) == 'L4J1esD4rYuBHXwjg72yi7Rw4G3iF2yUHt7LN9trpC3snCppUbq8' + + with pytest.raises(ValueError): + mprivkey.child(-1) + with pytest.raises(ValueError): + mprivkey.child(1 << 32) + # OK + mprivkey.child((1 << 32) - 1) + + +class TestVectors(): + + def test_vector1(self): + seed = bytes.fromhex("000102030405060708090a0b0c0d0e0f") + + # Chain m + m = bip32.PrivKey.from_seed(seed) + xprv = m.extended_key_string(Bitcoin) + assert xprv == "xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi" + xpub = m.public_key.extended_key_string(Bitcoin) + assert xpub == "xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8" + + # Chain m/0H + m1 = m.child(0 + m.HARDENED) + xprv = m1.extended_key_string(Bitcoin) + assert xprv == "xprv9uHRZZhk6KAJC1avXpDAp4MDc3sQKNxDiPvvkX8Br5ngLNv1TxvUxt4cV1rGL5hj6KCesnDYUhd7oWgT11eZG7XnxHrnYeSvkzY7d2bhkJ7" + xpub = m1.public_key.extended_key_string(Bitcoin) + assert xpub == "xpub68Gmy5EdvgibQVfPdqkBBCHxA5htiqg55crXYuXoQRKfDBFA1WEjWgP6LHhwBZeNK1VTsfTFUHCdrfp1bgwQ9xv5ski8PX9rL2dZXvgGDnw" + + # Chain m/0H/1 + m2 = m1.child(1) + xprv = m2.extended_key_string(Bitcoin) + assert xprv == "xprv9wTYmMFdV23N2TdNG573QoEsfRrWKQgWeibmLntzniatZvR9BmLnvSxqu53Kw1UmYPxLgboyZQaXwTCg8MSY3H2EU4pWcQDnRnrVA1xe8fs" + xpub = m2.public_key.extended_key_string(Bitcoin) + assert xpub == "xpub6ASuArnXKPbfEwhqN6e3mwBcDTgzisQN1wXN9BJcM47sSikHjJf3UFHKkNAWbWMiGj7Wf5uMash7SyYq527Hqck2AxYysAA7xmALppuCkwQ" + + # Chain m/0H/1/2H + m3 = m2.child(2 + m.HARDENED) + xprv = m3.extended_key_string(Bitcoin) + assert xprv == "xprv9z4pot5VBttmtdRTWfWQmoH1taj2axGVzFqSb8C9xaxKymcFzXBDptWmT7FwuEzG3ryjH4ktypQSAewRiNMjANTtpgP4mLTj34bhnZX7UiM" + xpub = m3.public_key.extended_key_string(Bitcoin) + assert xpub == "xpub6D4BDPcP2GT577Vvch3R8wDkScZWzQzMMUm3PWbmWvVJrZwQY4VUNgqFJPMM3No2dFDFGTsxxpG5uJh7n7epu4trkrX7x7DogT5Uv6fcLW5" + + # Chain m/0H/1/2H/2 + m4 = m3.child(2) + xprv = m4.extended_key_string(Bitcoin) + assert xprv == "xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334" + xpub = m4.public_key.extended_key_string(Bitcoin) + assert xpub == "xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV" + + # Chain m/0H/1/2H/2/1000000000 + m5 = m4.child(1000000000) + xprv = m5.extended_key_string(Bitcoin) + assert xprv == "xprvA41z7zogVVwxVSgdKUHDy1SKmdb533PjDz7J6N6mV6uS3ze1ai8FHa8kmHScGpWmj4WggLyQjgPie1rFSruoUihUZREPSL39UNdE3BBDu76" + xpub = m5.public_key.extended_key_string(Bitcoin) + assert xpub == "xpub6H1LXWLaKsWFhvm6RVpEL9P4KfRZSW7abD2ttkWP3SSQvnyA8FSVqNTEcYFgJS2UaFcxupHiYkro49S8yGasTvXEYBVPamhGW6cFJodrTHy" + + def test_vector2(self): + seed = bytes.fromhex("fffcf9f6f3f0edeae7e4e1dedbd8d5d2cfccc9c6c3c0bdbab7b4b1aeaba8a5a29f9c999693908d8a8784817e7b7875726f6c696663605d5a5754514e4b484542") + # Chain m + m = bip32.PrivKey.from_seed(seed) + xprv = m.extended_key_string(Bitcoin) + assert xprv == "xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U" + xpub = m.public_key.extended_key_string(Bitcoin) + assert xpub == "xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB" + + # Chain m/0 + m1 = m.child(0) + xprv = m1.extended_key_string(Bitcoin) + assert xprv == "xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt" + xpub = m1.public_key.extended_key_string(Bitcoin) + assert xpub == "xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH" + + # Chain m/0H/2147483647H + m2 = m1.child(2147483647 + m.HARDENED) + xprv = m2.extended_key_string(Bitcoin) + assert xprv == "xprv9wSp6B7kry3Vj9m1zSnLvN3xH8RdsPP1Mh7fAaR7aRLcQMKTR2vidYEeEg2mUCTAwCd6vnxVrcjfy2kRgVsFawNzmjuHc2YmYRmagcEPdU9" + xpub = m2.public_key.extended_key_string(Bitcoin) + assert xpub == "xpub6ASAVgeehLbnwdqV6UKMHVzgqAG8Gr6riv3Fxxpj8ksbH9ebxaEyBLZ85ySDhKiLDBrQSARLq1uNRts8RuJiHjaDMBU4Zn9h8LZNnBC5y4a" + + # Chain m/0H/2147483647H/1 + m3 = m2.child(1) + xprv = m3.extended_key_string(Bitcoin) + xpub = m3.public_key.extended_key_string(Bitcoin) + assert xprv == "xprv9zFnWC6h2cLgpmSA46vutJzBcfJ8yaJGg8cX1e5StJh45BBciYTRXSd25UEPVuesF9yog62tGAQtHjXajPPdbRCHuWS6T8XA2ECKADdw4Ef" + assert xpub == "xpub6DF8uhdarytz3FWdA8TvFSvvAh8dP3283MY7p2V4SeE2wyWmG5mg5EwVvmdMVCQcoNJxGoWaU9DCWh89LojfZ537wTfunKau47EL2dhHKon" + + # Chain m/0/2147483647H/1/2147483646H + m4 = m3.child(2147483646 + m.HARDENED) + xprv = m4.extended_key_string(Bitcoin) + xpub = m4.public_key.extended_key_string(Bitcoin) + assert xprv == "xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc" + assert xpub == "xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL" + + # Chain m/0/2147483647H/1/2147483646H/2 + m5 = m4.child(2) + xprv = m5.extended_key_string(Bitcoin) + xpub = m5.public_key.extended_key_string(Bitcoin) + assert xprv == "xprvA2nrNbFZABcdryreWet9Ea4LvTJcGsqrMzxHx98MMrotbir7yrKCEXw7nadnHM8Dq38EGfSh6dqA9QWTyefMLEcBYJUuekgW4BYPJcr9E7j" + assert xpub == "xpub6FnCn6nSzZAw5Tw7cgR9bi15UV96gLZhjDstkXXxvCLsUXBGXPdSnLFbdpq8p9HmGsApME5hQTZ3emM2rnY5agb9rXpVGyy3bdW6EEgAtqt" + + def test_vector3(self): + seed = bytes.fromhex("4b381541583be4423346c643850da4b320e46a87ae3d2a4e6da11eba819cd4acba45d239319ac14f863b8d5ab5a0d0c64d2e8a1e7d1457df2e5a3c51c73235be") + + # Chain m + m = bip32.PrivKey.from_seed(seed) + xprv = m.extended_key_string(Bitcoin) + xpub = m.public_key.extended_key_string(Bitcoin) + assert xprv == "xprv9s21ZrQH143K25QhxbucbDDuQ4naNntJRi4KUfWT7xo4EKsHt2QJDu7KXp1A3u7Bi1j8ph3EGsZ9Xvz9dGuVrtHHs7pXeTzjuxBrCmmhgC6" + assert xpub == "xpub661MyMwAqRbcEZVB4dScxMAdx6d4nFc9nvyvH3v4gJL378CSRZiYmhRoP7mBy6gSPSCYk6SzXPTf3ND1cZAceL7SfJ1Z3GC8vBgp2epUt13" + + # Chain m/0H + m1 = m.child(0 + m.HARDENED) + xprv = m1.extended_key_string(Bitcoin) + xpub = m1.public_key.extended_key_string(Bitcoin) + assert xprv == "xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L" + assert xpub == "xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y" diff --git a/wallet/bip32.py b/wallet/bip32.py new file mode 100644 index 0000000..048b046 --- /dev/null +++ b/wallet/bip32.py @@ -0,0 +1,307 @@ +# Copyright (c) 2017, Neil Booth +# +# All rights reserved. +# +# See the file "LICENCE" for information about the copyright +# and warranty status of this software. + +'''Logic for BIP32 Hierarchical Key Derviation.''' + +import struct + +import ecdsa +import ecdsa.ellipticcurve as EC +import ecdsa.numbertheory as NT + +from lib.coins import Coin +from lib.hash import Base58, hmac_sha512, hash160 +from lib.util import cachedproperty, bytes_to_int, int_to_bytes + + +class DerivationError(Exception): + '''Raised when an invalid derivation occurs.''' + + +class _KeyBase(object): + '''A BIP32 Key, public or private.''' + + CURVE = ecdsa.SECP256k1 + + def __init__(self, chain_code, n, depth, parent): + if not isinstance(chain_code, (bytes, bytearray)): + raise TypeError('chain code must be raw bytes') + if len(chain_code) != 32: + raise ValueError('invalid chain code') + if not 0 <= n < 1 << 32: + raise ValueError('invalid child number') + if not 0 <= depth < 256: + raise ValueError('invalid depth') + if parent is not None: + if not isinstance(parent, type(self)): + raise TypeError('parent key has bad type') + self.chain_code = chain_code + self.n = n + self.depth = depth + self.parent = parent + + def _hmac_sha512(self, msg): + '''Use SHA-512 to provide an HMAC, returned as a pair of 32-byte + objects. + ''' + hmac = hmac_sha512(self.chain_code, msg) + return hmac[:32], hmac[32:] + + def _extended_key(self, ver_bytes, raw_serkey): + '''Return the 78-byte extended key given prefix version bytes and + serialized key bytes. + ''' + if not isinstance(ver_bytes, (bytes, bytearray)): + raise TypeError('ver_bytes must be raw bytes') + if len(ver_bytes) != 4: + raise ValueError('ver_bytes must have length 4') + if not isinstance(raw_serkey, (bytes, bytearray)): + raise TypeError('raw_serkey must be raw bytes') + if len(raw_serkey) != 33: + raise ValueError('raw_serkey must have length 33') + + return (ver_bytes + bytes([self.depth]) + + self.parent_fingerprint() + struct.pack('>I', self.n) + + self.chain_code + raw_serkey) + + def fingerprint(self): + '''Return the key's fingerprint as 4 bytes.''' + return self.identifier()[:4] + + def parent_fingerprint(self): + '''Return the parent key's fingerprint as 4 bytes.''' + return self.parent.fingerprint() if self.parent else bytes(4) + + def extended_key_string(self, coin): + '''Return an extended key as a base58 string.''' + return Base58.encode_check(self.extended_key(coin)) + + +class PubKey(_KeyBase): + '''A BIP32 public key.''' + + def __init__(self, pubkey, chain_code, n, depth, parent=None): + super().__init__(chain_code, n, depth, parent) + if isinstance(pubkey, ecdsa.VerifyingKey): + self.verifying_key = pubkey + else: + self.verifying_key = self._verifying_key_from_pubkey(pubkey) + self.addresses = {} + + @classmethod + def _verifying_key_from_pubkey(cls, pubkey): + '''Converts a 33-byte compressed pubkey into an ecdsa.VerifyingKey + object''' + if not isinstance(pubkey, (bytes, bytearray)): + raise TypeError('pubkey must be raw bytes') + if len(pubkey) != 33: + raise ValueError('pubkey must be 33 bytes') + if pubkey[0] not in (2, 3): + raise ValueError('invalid pubkey prefix byte') + curve = cls.CURVE.curve + + is_odd = pubkey[0] == 3 + x = bytes_to_int(pubkey[1:]) + + # p is the finite field order + a, b, p = curve.a(), curve.b(), curve.p() + y2 = pow(x, 3, p) + b + if a: + y2 += a * pow(x, 2, p) + y = NT.square_root_mod_prime(y2 % p, p) + if bool(y & 1) != is_odd: + y = p - y + point = EC.Point(curve, x, y) + + return ecdsa.VerifyingKey.from_public_point(point, curve=cls.CURVE) + + @cachedproperty + def pubkey_bytes(self): + '''Return the compressed public key as 33 bytes.''' + point = self.verifying_key.pubkey.point + prefix = bytes([2 + (point.y() & 1)]) + padded_bytes = _exponent_to_bytes(point.x()) + return prefix + padded_bytes + + def address(self, coin): + "The public key as a P2PKH address" + address = self.addresses.get(coin) + if not address: + address = coin.P2PKH_address_from_pubkey(self.pubkey_bytes) + self.addresses[coin] = address + return address + + def ec_point(self): + return self.verifying_key.pubkey.point + + def child(self, n): + '''Return the derived child extended pubkey at index N.''' + if not 0 <= n < (1 << 31): + raise ValueError('invalid BIP32 public key child number') + + msg = self.pubkey_bytes + struct.pack('>I', n) + L, R = self._hmac_sha512(msg) + + curve = self.CURVE + L = bytes_to_int(L) + if L >= curve.order: + raise DerivationError + + point = curve.generator * L + self.ec_point() + if point == EC.INFINITY: + raise DerivationError + + verkey = ecdsa.VerifyingKey.from_public_point(point, curve=curve) + + return PubKey(verkey, R, n, self.depth + 1, self) + + def identifier(self): + '''Return the key's identifier as 20 bytes.''' + return hash160(self.pubkey_bytes) + + def extended_key(self, coin): + '''Return a raw extended public key.''' + return self._extended_key(coin.XPUB_VERBYTES, self.pubkey_bytes) + + +class PrivKey(_KeyBase): + '''A BIP32 private key.''' + + HARDENED = 1 << 31 + + def __init__(self, privkey, chain_code, n, depth, parent=None): + super().__init__(chain_code, n, depth, parent) + if isinstance(privkey, ecdsa.SigningKey): + self.signing_key = privkey + else: + self.signing_key = self._signing_key_from_privkey(privkey) + + @classmethod + def _signing_key_from_privkey(cls, privkey): + '''Converts a 32-byte privkey into an ecdsa.SigningKey object.''' + exponent = cls._privkey_secret_exponent(privkey) + return ecdsa.SigningKey.from_secret_exponent(exponent, curve=cls.CURVE) + + @classmethod + def _privkey_secret_exponent(cls, privkey): + '''Return the private key as a secret exponent if it is a valid private + key.''' + if not isinstance(privkey, (bytes, bytearray)): + raise TypeError('privkey must be raw bytes') + if len(privkey) != 32: + raise ValueError('privkey must be 32 bytes') + exponent = bytes_to_int(privkey) + if not 1 <= exponent < cls.CURVE.order: + raise ValueError('privkey represents an invalid exponent') + + return exponent + + @classmethod + def from_seed(cls, seed): + # This hard-coded message string seems to be coin-independent... + hmac = hmac_sha512(b'Bitcoin seed', seed) + privkey, chain_code = hmac[:32], hmac[32:] + return cls(privkey, chain_code, 0, 0) + + @cachedproperty + def privkey_bytes(self): + '''Return the serialized private key (no leading zero byte).''' + return _exponent_to_bytes(self.secret_exponent()) + + @cachedproperty + def public_key(self): + '''Return the corresponding extended public key.''' + verifying_key = self.signing_key.get_verifying_key() + parent_pubkey = self.parent.public_key if self.parent else None + return PubKey(verifying_key, self.chain_code, self.n, self.depth, + parent_pubkey) + + def ec_point(self): + return self.public_key.ec_point() + + def secret_exponent(self): + '''Return the private key as a secret exponent.''' + return self.signing_key.privkey.secret_multiplier + + def WIF(self, coin): + '''Return the private key encoded in Wallet Import Format.''' + return coin.privkey_WIF(self.privkey_bytes, compressed=True) + + def address(self, coin): + "The public key as a P2PKH address" + return self.public_key.address(coin) + + def child(self, n): + '''Return the derived child extended privkey at index N.''' + if not 0 <= n < (1 << 32): + raise ValueError('invalid BIP32 private key child number') + + if n >= self.HARDENED: + serkey = b'\0' + self.privkey_bytes + else: + serkey = self.public_key.pubkey_bytes + + msg = serkey + struct.pack('>I', n) + L, R = self._hmac_sha512(msg) + + curve = self.CURVE + L = bytes_to_int(L) + exponent = (L + bytes_to_int(self.privkey_bytes)) % curve.order + if exponent == 0 or L >= curve.order: + raise DerivationError + + privkey = _exponent_to_bytes(exponent) + + return PrivKey(privkey, R, n, self.depth + 1, self) + + def identifier(self): + '''Return the key's identifier as 20 bytes.''' + return self.public_key.identifier() + + def extended_key(self, coin): + '''Return a raw extended private key.''' + return self._extended_key(coin.XPRV_VERBYTES, + b'\0' + self.privkey_bytes) + + +def _exponent_to_bytes(exponent): + '''Convert an exponent to 32 big-endian bytes''' + return (bytes(32) + int_to_bytes(exponent))[-32:] + +def _from_extended_key(ekey): + '''Return a PubKey or PrivKey from an extended key raw bytes.''' + if not isinstance(ekey, (bytes, bytearray)): + raise TypeError('extended key must be raw bytes') + if len(ekey) != 78: + raise ValueError('extended key must have length 78') + + is_public, coin = Coin.lookup_xverbytes(ekey[:4]) + depth = ekey[4] + fingerprint = ekey[5:9] # Not used + n, = struct.unpack('>I', ekey[9:13]) + chain_code = ekey[13:45] + + if is_public: + pubkey = ekey[45:] + key = PubKey(pubkey, chain_code, n, depth) + else: + if ekey[45] is not 0: + raise ValueError('invalid extended private key prefix byte') + privkey = ekey[46:] + key = PrivKey(privkey, chain_code, n, depth) + + return key, coin + +def from_extended_key_string(ekey_str): + '''Given an extended key string, such as + + xpub6BsnM1W2Y7qLMiuhi7f7dbAwQZ5Cz5gYJCRzTNainXzQXYjFwtuQXHd + 3qfi3t3KJtHxshXezfjft93w4UE7BGMtKwhqEHae3ZA7d823DVrL + + return a (key, coin) pair. key is either a PubKey or PrivKey. + ''' + return _from_extended_key(Base58.decode_check(ekey_str)) From eb51706316ee984e2e67c8c47461f11022a28d16 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 12 Jul 2017 14:43:57 +0900 Subject: [PATCH 107/117] Update travis.yml --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a519091..eb9809d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,11 @@ python: # command to install dependencies install: - pip install aiohttp + - pip install ecdsa - pip install plyvel - pip install pyrocksdb - pip install pytest-cov - pip install python-coveralls # command to run tests script: pytest --cov=server --cov=lib -after_success: coveralls \ No newline at end of file +after_success: coveralls From 1addbd6815eb77b55cab17007952d182fd03df75 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 12 Jul 2017 14:53:30 +0900 Subject: [PATCH 108/117] Add wallet dir to coverage --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index eb9809d..1bc13d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,5 +18,5 @@ install: - pip install pytest-cov - pip install python-coveralls # command to run tests -script: pytest --cov=server --cov=lib +script: pytest --cov=server --cov=lib --cov=wallet after_success: coveralls From 54c3ae4c5d432403d402993d003502240fa24fbd Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 12 Jul 2017 15:09:17 +0900 Subject: [PATCH 109/117] Improve bip32 test coverage --- tests/wallet/test_bip32.py | 30 ++++++++++++++++++++++++++++++ wallet/bip32.py | 3 +-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/tests/wallet/test_bip32.py b/tests/wallet/test_bip32.py index 0fd1eac..db5b743 100644 --- a/tests/wallet/test_bip32.py +++ b/tests/wallet/test_bip32.py @@ -59,6 +59,8 @@ class TestPubKey(object): cls(raw_pubkey, chain_code, 0, -1) with pytest.raises(ValueError): cls(raw_pubkey, chain_code, 0, 256) + with pytest.raises(ValueError): + cls(b'\0' + b'\2' * 32, chain_code, 0, 0) # These are OK cls(b'\2' + b'\2' * 32, chain_code, 0, 0) @@ -88,6 +90,18 @@ class TestPubKey(object): assert mpubkey.chain_code == b'>V\x83\x92`\r\x17\xb3"\xa6\x7f\xaf\xc0\x930\xf7\x1e\xdc\x12i\x9c\xe4\xc0,a\x1a\x04\xec\x16\x19\xaeK' assert mpubkey.ec_point().x() == 44977109961578369385937116592536468905742111247230478021459394832226142714624 + def test_extended_key(self): + # Test argument validation + with pytest.raises(TypeError): + mpubkey._extended_key('foot', bytes(33)) + with pytest.raises(ValueError): + mpubkey._extended_key(b'foo', bytes(33)) + with pytest.raises(TypeError): + mpubkey._extended_key(bytes(4), ' ' * 33) + with pytest.raises(ValueError): + mpubkey._extended_key(b'foot', bytes(32)) + mpubkey._extended_key(b'foot', bytes(33)) + def test_extended_key_string(self): # Implictly tests extended_key() assert mpubkey.extended_key_string(Bitcoin) == MXPUB @@ -183,6 +197,10 @@ class TestPrivKey(object): cls(bip32._exponent_to_bytes(cls.CURVE.order - 1), chain_code, 0, 0) privkey = cls(MPRIVKEY, chain_code, 0, 255) + # Construction with bad parent + with pytest.raises(TypeError): + cls(MPRIVKEY, chain_code, 0, 0, privkey.public_key) + # Construction from signing key dup = cls(privkey.signing_key, chain_code, 0, 0) assert dup.ec_point() == privkey.ec_point() @@ -217,6 +235,18 @@ class TestPrivKey(object): assert mprivkey.public_key.n == mpubkey.n assert mprivkey.public_key.depth == mpubkey.depth + def test_extended_key(self): + # Test argument validation + with pytest.raises(TypeError): + mprivkey._extended_key('foot', bytes(33)) + with pytest.raises(ValueError): + mprivkey._extended_key(b'foo', bytes(33)) + with pytest.raises(TypeError): + mprivkey._extended_key(bytes(4), ' ' * 33) + with pytest.raises(ValueError): + mprivkey._extended_key(b'foot', bytes(32)) + mprivkey._extended_key(b'foot', bytes(33)) + def test_extended_key_string(self): # Also tests extended_key, WIF and privkey_bytes assert mprivkey.extended_key_string(Bitcoin) == MXPRV diff --git a/wallet/bip32.py b/wallet/bip32.py index 048b046..8215061 100644 --- a/wallet/bip32.py +++ b/wallet/bip32.py @@ -110,8 +110,7 @@ class PubKey(_KeyBase): # p is the finite field order a, b, p = curve.a(), curve.b(), curve.p() y2 = pow(x, 3, p) + b - if a: - y2 += a * pow(x, 2, p) + assert a == 0 # Otherwise y2 += a * pow(x, 2, p) y = NT.square_root_mod_prime(y2 % p, p) if bool(y & 1) != is_odd: y = p - y From 4665ba6315bfd484432cf7ebbdd4fcbbdf9e608c Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Thu, 13 Jul 2017 10:50:04 +0900 Subject: [PATCH 110/117] Improve daemon JSON RPC compatibility - give an ID to each request - allow client session to be customized by derived classes Based on changes suggested by erasmospunk --- server/daemon.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/server/daemon.py b/server/daemon.py index 5d56ce5..5e9b311 100644 --- a/server/daemon.py +++ b/server/daemon.py @@ -45,6 +45,7 @@ class Daemon(util.LoggedClass): self.workqueue_semaphore = asyncio.Semaphore(value=10) self.down = False self.last_error_time = 0 + self.req_id = 0 # assignment of asyncio.TimeoutError are essentially ignored if aiohttp.__version__.startswith('1.'): self.ClientHttpProcessingError = aiohttp.ClientHttpProcessingError @@ -53,6 +54,11 @@ class Daemon(util.LoggedClass): self.ClientHttpProcessingError = asyncio.TimeoutError self.ClientPayloadError = aiohttp.ClientPayloadError + def next_req_id(self): + '''Retrns the next request ID.''' + self.req_id += 1 + return self.req_id + def set_urls(self, urls): '''Set the URLS to the given list, and switch to the first one.''' if not urls: @@ -79,9 +85,13 @@ class Daemon(util.LoggedClass): return True return False + def client_session(self): + '''An aiohttp client session.''' + return aiohttp.ClientSession() + async def _send_data(self, data): async with self.workqueue_semaphore: - async with aiohttp.ClientSession() as session: + async with self.client_session() as session: async with session.post(self.url(), data=data) as resp: # If bitcoind can't find a tx, for some reason # it returns 500 but fills out the JSON. @@ -158,7 +168,7 @@ class Daemon(util.LoggedClass): raise self.DaemonWarmingUpError raise DaemonError(err) - payload = {'method': method} + payload = {'method': method, 'id': self.next_req_id()} if params: payload['params'] = params return await self._send(payload, processor) @@ -177,7 +187,8 @@ class Daemon(util.LoggedClass): return [item['result'] for item in result] raise DaemonError(errs) - payload = [{'method': method, 'params': p} for p in params_iterable] + payload = [{'method': method, 'params': p, 'id': self.next_req_id()} + for p in params_iterable] if payload: return await self._send(payload, processor) return [] From 3612b88e2e30f83fe8b1034de1c41ca3557bc7ee Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 16 Jul 2017 13:58:02 +0900 Subject: [PATCH 111/117] Permit underscores in hostnames. Add tests. --- lib/util.py | 4 +++- tests/lib/test_util.py | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/util.py b/lib/util.py index 2c09127..40e5869 100644 --- a/lib/util.py +++ b/lib/util.py @@ -259,7 +259,9 @@ def address_string(address): return fmt.format(host, port) # See http://stackoverflow.com/questions/2532053/validate-a-hostname-string -SEGMENT_REGEX = re.compile("(?!-)[A-Z\d-]{1,63}(? 255: return False diff --git a/tests/lib/test_util.py b/tests/lib/test_util.py index 4142679..b55e6b6 100644 --- a/tests/lib/test_util.py +++ b/tests/lib/test_util.py @@ -56,3 +56,30 @@ def test_increment_byte_string(): assert util.increment_byte_string(b'1') == b'2' assert util.increment_byte_string(b'\x01\x01') == b'\x01\x02' assert util.increment_byte_string(b'\xff\xff') is None + +def test_is_valid_hostname(): + is_valid_hostname = util.is_valid_hostname + assert not is_valid_hostname('') + assert is_valid_hostname('a') + assert is_valid_hostname('_') + # Hyphens + assert not is_valid_hostname('-b') + assert not is_valid_hostname('a.-b') + assert is_valid_hostname('a-b') + assert not is_valid_hostname('b-') + assert not is_valid_hostname('b-.c') + # Dots + assert is_valid_hostname('a.') + assert is_valid_hostname('foo1.Foo') + assert not is_valid_hostname('foo1..Foo') + assert is_valid_hostname('12Foo.Bar.Bax_') + assert is_valid_hostname('12Foo.Bar.Baz_12') + # 63 octets in part + assert is_valid_hostname('a.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN' + 'OPQRSTUVWXYZ0123456789_.bar') + # Over 63 octets in part + assert not is_valid_hostname('a.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN' + 'OPQRSTUVWXYZ0123456789_1.bar') + len255 = ('a' * 62 + '.') * 4 + 'abc' + assert is_valid_hostname(len255) + assert not is_valid_hostname(len255 + 'd') From 3ca464e5b1831f4ac58d49a187446e0249290221 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 16 Jul 2017 15:01:23 +0900 Subject: [PATCH 112/117] Update BTC server list --- lib/coins.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index 40caa28..0de4e5b 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -341,13 +341,14 @@ class Bitcoin(Coin): 'electrum3.hachre.de p10000 s t', 'electrum.hsmiths.com s t', 'erbium1.sytes.net s t', - 'fdkbwjykvl2f3hup.onion p10000 s t', - 'h.1209k.com p10000 s t', + 'fdkhv2bb7hqel2e7.onion s t', + 'h.1209k.com s t', 'helicarrier.bauerj.eu s t', 'hsmiths4fyqlw5xw.onion s t', 'ozahtqwp25chjdjd.onion s t', 'us11.einfachmalnettsein.de s t', 'ELEX01.blackpole.online s t', + 'electrum_abc.criptolayer.net s50012', ] From 3f9e2363c2ddbded7e0d9431e578855c5f54887b Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 16 Jul 2017 15:07:13 +0900 Subject: [PATCH 113/117] Prepare 1.0.12 --- README.rst | 61 ++++++++++------------------------------------- server/version.py | 2 +- 2 files changed, 13 insertions(+), 50 deletions(-) diff --git a/README.rst b/README.rst index 84b75d5..0bb90bf 100644 --- a/README.rst +++ b/README.rst @@ -133,6 +133,18 @@ Roadmap ChangeLog ========= +Version 1.0.12 +-------------- + +- handle legacy daemons, add support for Blackcoin and Peercoin (erasmospunk) +- implement history compression; can currently only be done from a script + with the server down +- Add dockerfile reference (followtheart) +- doc, runfile fixes (Henry, emilrus) +- add bip32 implementation, currently unused +- daemon compatibility improvements (erasmospunk) +- permit underscores in hostnames, updated Bitcoin server list + Version 1.0.11 -------------- @@ -249,50 +261,6 @@ Version 1.0 * Minor doc tweaks only -Version 0.99.4 --------------- - -* Add support for Bitcoin Unlimited's nolnet; set **NET** to nolnet -* Choose 2 peers per bucket -* Minor bugfix - -Version 0.99.3 --------------- - -* Require Python 3.5.3. 3.5.2 has asyncio API and socket-related issues. - Resolves `#135`_ -* Remove peer semaphore -* Improved Base58 handling for >1 byte version prefix (erasmospunk) - -Version 0.99.2 --------------- - -* don't announce self if a non-public IP address -* logging tweaks - -Version 0.99.1 --------------- - -* Add more verbose logging in attempt to understand issue `#135`_ -* REPORT_TCP_PORT_TOR and REPORT_SSL_PORT_TOR were ignored when constructing - IRC real names. Fixes `#136`_ -* Only serve chunk requests in forward direction; disconnect clients iterating - backwards. Minimizes bandwidth consumption caused by misbehaving Electrum - clients. Closes `#132`_ -* Tor coin peers would always be scheduled for check, fixes `#138`_ (fr3aker) - -Version 0.99 ------------- - -Preparation for release of 1.0, which will only have bug fixes and -documentation updates. - -* improve handling of daemon going down so that incoming connections - are not blocked. Also improve logging thereof. Fixes `#100`_. -* add facility to disable peer discovery and/or self announcement, - see `docs/ENVIRONMENT.rst`_. -* add FairCoin (thokon00) - **Neil Booth** kyuupichan@gmail.com https://github.com/kyuupichan @@ -300,12 +268,7 @@ documentation updates. .. _#83: https://github.com/kyuupichan/electrumx/issues/83 -.. _#100: https://github.com/kyuupichan/electrumx/issues/100 .. _#128: https://github.com/kyuupichan/electrumx/issues/128 -.. _#132: https://github.com/kyuupichan/electrumx/issues/132 -.. _#135: https://github.com/kyuupichan/electrumx/issues/135 -.. _#136: https://github.com/kyuupichan/electrumx/issues/136 -.. _#138: https://github.com/kyuupichan/electrumx/issues/138 .. _#152: https://github.com/kyuupichan/electrumx/issues/152 .. _#157: https://github.com/kyuupichan/electrumx/issues/157 .. _#158: https://github.com/kyuupichan/electrumx/issues/158 diff --git a/server/version.py b/server/version.py index 6b84b68..59f8b42 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.11' +VERSION = 'ElectrumX 1.0.12' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From 87d24f38dc8b21c11a6ff9b6191930c8bb756f1c Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 19 Jul 2017 23:38:35 +0900 Subject: [PATCH 114/117] More logical mempool hash handling Fixes the issue whereby notifications weren't sent as long as new blocks kept coming in. Now a new height notification, with an appropriate mempool update, is sent after each batch of blocks is processed. --- server/block_processor.py | 30 +++++++++++++++++++++--------- server/daemon.py | 13 ++----------- server/mempool.py | 25 +++++++++++++------------ 3 files changed, 36 insertions(+), 32 deletions(-) diff --git a/server/block_processor.py b/server/block_processor.py index 157d226..4b9fcda 100644 --- a/server/block_processor.py +++ b/server/block_processor.py @@ -29,7 +29,6 @@ class Prefetcher(LoggedClass): def __init__(self, bp): super().__init__() self.bp = bp - self.caught_up = False # Access to fetched_height should be protected by the semaphore self.fetched_height = None self.semaphore = asyncio.Semaphore() @@ -84,7 +83,14 @@ class Prefetcher(LoggedClass): Repeats until the queue is full or caught up. ''' daemon = self.bp.daemon - daemon_height = await daemon.height(self.bp.caught_up_event.is_set()) + # If caught up, refresh the mempool before the current height + caught_up = self.bp.caught_up_event.is_set() + if caught_up: + mempool = await daemon.mempool_hashes() + else: + mempool = [] + + daemon_height = await daemon.height() with await self.semaphore: while self.cache_size < self.min_cache_size: # Try and catch up all blocks but limit to room in cache. @@ -94,14 +100,15 @@ class Prefetcher(LoggedClass): count = min(daemon_height - self.fetched_height, cache_room) count = min(500, max(count, 0)) if not count: - if not self.caught_up: - self.caught_up = True + if caught_up: + self.bp.set_mempool_hashes(mempool) + else: self.bp.on_prefetcher_first_caught_up() return False first = self.fetched_height + 1 hex_hashes = await daemon.block_hex_hashes(first, count) - if self.caught_up: + if caught_up: self.logger.info('new block height {:,d} hash {}' .format(first + count-1, hex_hashes[-1])) blocks = await daemon.raw_blocks(hex_hashes) @@ -121,7 +128,7 @@ class Prefetcher(LoggedClass): else: self.ave_size = (size + (10 - count) * self.ave_size) // 10 - self.bp.on_prefetched_blocks(blocks, first) + self.bp.on_prefetched_blocks(blocks, first, mempool) self.cache_size += size self.fetched_height += count @@ -188,9 +195,10 @@ class BlockProcessor(server.db.DB): '''Add the task to our task queue.''' self.task_queue.put_nowait(task) - def on_prefetched_blocks(self, blocks, first): + def on_prefetched_blocks(self, blocks, first, mempool): '''Called by the prefetcher when it has prefetched some blocks.''' - self.add_task(partial(self.check_and_advance_blocks, blocks, first)) + self.add_task(partial(self.check_and_advance_blocks, blocks, first, + mempool)) def on_prefetcher_first_caught_up(self): '''Called by the prefetcher when it first catches up.''' @@ -225,7 +233,10 @@ class BlockProcessor(server.db.DB): self.open_dbs() self.caught_up_event.set() - async def check_and_advance_blocks(self, blocks, first): + def set_mempool_hashes(self, mempool): + self.controller.mempool.set_hashes(mempool) + + async def check_and_advance_blocks(self, blocks, first, mempool): '''Process the list of blocks passed. Detects and handles reorgs.''' self.prefetcher.processing_blocks(blocks) if first != self.height + 1: @@ -251,6 +262,7 @@ class BlockProcessor(server.db.DB): self.logger.info('processed {:,d} block{} in {:.1f}s' .format(len(blocks), s, time.time() - start)) + self.set_mempool_hashes(mempool) elif hprevs[0] != chain[0]: await self.reorg_chain() else: diff --git a/server/daemon.py b/server/daemon.py index 5e9b311..23cebbf 100644 --- a/server/daemon.py +++ b/server/daemon.py @@ -38,8 +38,6 @@ class Daemon(util.LoggedClass): super().__init__() self.set_urls(urls) self._height = None - self._mempool_hashes = set() - self.mempool_refresh_event = asyncio.Event() # Limit concurrent RPC calls to this number. # See DEFAULT_HTTP_WORKQUEUE in bitcoind, which is typically 16 self.workqueue_semaphore = asyncio.Semaphore(value=10) @@ -210,7 +208,7 @@ class Daemon(util.LoggedClass): return [bytes.fromhex(block) for block in blocks] async def mempool_hashes(self): - '''Update our record of the daemon's mempool hashes.''' + '''Return a list of the daemon's mempool hashes.''' return await self._send_single('getrawmempool') async def estimatefee(self, params): @@ -245,18 +243,11 @@ class Daemon(util.LoggedClass): '''Broadcast a transaction to the network.''' return await self._send_single('sendrawtransaction', params) - async def height(self, mempool=False): + async def height(self): '''Query the daemon for its current height.''' self._height = await self._send_single('getblockcount') - if mempool: - self._mempool_hashes = set(await self.mempool_hashes()) - self.mempool_refresh_event.set() return self._height - def cached_mempool_hashes(self): - '''Return the cached mempool hashes.''' - return self._mempool_hashes - def cached_height(self): '''Return the cached daemon height. diff --git a/server/mempool.py b/server/mempool.py index 0a6c27b..075c29d 100644 --- a/server/mempool.py +++ b/server/mempool.py @@ -37,6 +37,8 @@ class MemPool(util.LoggedClass): self.controller = controller self.coin = bp.coin self.db = bp + self.hashes = set() + self.mempool_refresh_event = asyncio.Event() self.touched = bp.touched self.touched_event = asyncio.Event() self.prioritized = set() @@ -49,6 +51,11 @@ class MemPool(util.LoggedClass): initial mempool sync.''' self.prioritized.add(tx_hash) + def set_hashes(self, hashes): + '''Save the list of mempool hashes.''' + self.hashes = set(hashes) + self.mempool_refresh_event.set() + def resync_daemon_hashes(self, unprocessed, unfetched): '''Re-sync self.txs with the list of hashes in the daemon's mempool. @@ -59,8 +66,7 @@ class MemPool(util.LoggedClass): hashXs = self.hashXs touched = self.touched - hashes = self.daemon.cached_mempool_hashes() - gone = set(txs).difference(hashes) + gone = set(txs).difference(self.hashes) for hex_hash in gone: unfetched.discard(hex_hash) unprocessed.pop(hex_hash, None) @@ -75,7 +81,7 @@ class MemPool(util.LoggedClass): del hashXs[hashX] touched.update(tx_hashXs) - new = hashes.difference(txs) + new = self.hashes.difference(txs) unfetched.update(new) for hex_hash in new: txs[hex_hash] = None @@ -92,15 +98,14 @@ class MemPool(util.LoggedClass): fetch_size = 800 process_some = self.async_process_some(unfetched, fetch_size // 2) - await self.daemon.mempool_refresh_event.wait() + await self.mempool_refresh_event.wait() self.logger.info('beginning processing of daemon mempool. ' 'This can take some time...') next_log = 0 loops = -1 # Zero during initial catchup while True: - # Avoid double notifications if processing a block - if self.touched and not self.processing_new_block(): + if self.touched: self.touched_event.set() # Log progress / state @@ -120,10 +125,10 @@ class MemPool(util.LoggedClass): try: if not todo: self.prioritized.clear() - await self.daemon.mempool_refresh_event.wait() + await self.mempool_refresh_event.wait() self.resync_daemon_hashes(unprocessed, unfetched) - self.daemon.mempool_refresh_event.clear() + self.mempool_refresh_event.clear() if unfetched: count = min(len(unfetched), fetch_size) @@ -177,10 +182,6 @@ class MemPool(util.LoggedClass): return process - def processing_new_block(self): - '''Return True if we're processing a new block.''' - return self.daemon.cached_height() > self.db.db_height - async def fetch_raw_txs(self, hex_hashes): '''Fetch a list of mempool transactions.''' raw_txs = await self.daemon.getrawtransactions(hex_hashes) From 8a18da61c293c04b7ccfa55637c42840ea2fed41 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Thu, 20 Jul 2017 13:12:38 +0900 Subject: [PATCH 115/117] Add bitcoin-segwit --- lib/coins.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/coins.py b/lib/coins.py index 0de4e5b..86554ac 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -352,6 +352,11 @@ class Bitcoin(Coin): ] +class BitcoinSegwit(Bitcoin): + NET = "bitcoin-segwit" + DESERIALIZER = DeserializerSegWit + + class BitcoinTestnet(Bitcoin): SHORTNAME = "XTN" NET = "testnet" From 7abde2e5144e0eea9c913b63b7c5cc524bc841e7 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Fri, 21 Jul 2017 12:40:37 +0900 Subject: [PATCH 116/117] Prepare 1.0.13 --- README.rst | 6 ++++++ server/version.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 0bb90bf..4995053 100644 --- a/README.rst +++ b/README.rst @@ -133,6 +133,12 @@ Roadmap ChangeLog ========= +Version 1.0.13 +-------------- + +- improve mempool handling and height notifications +- add bitcoin-segwit as a new COIN + Version 1.0.12 -------------- diff --git a/server/version.py b/server/version.py index 59f8b42..fe5ea6a 100644 --- a/server/version.py +++ b/server/version.py @@ -1,5 +1,5 @@ # Server name and protocol versions -VERSION = 'ElectrumX 1.0.12' +VERSION = 'ElectrumX 1.0.13' PROTOCOL_MIN = '1.0' PROTOCOL_MAX = '1.0' From 9279602c4b7baf9e2613960763c350ff441b6534 Mon Sep 17 00:00:00 2001 From: Pavel Zhovner Date: Sun, 23 Jul 2017 08:06:51 +0300 Subject: [PATCH 117/117] Update HOWTO.rst (#206) --- docs/HOWTO.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/HOWTO.rst b/docs/HOWTO.rst index 7748a84..fb9f5ec 100644 --- a/docs/HOWTO.rst +++ b/docs/HOWTO.rst @@ -3,7 +3,7 @@ Prerequisites ============= **ElectrumX** should run on any flavour of unix. I have run it -successfully on MaxOSX and DragonFlyBSD. It won't run out-of-the-box +successfully on MacOS and DragonFlyBSD. It won't run out-of-the-box on Windows, but the changes required to make it do so should be small - pull requests are welcome.