From a51f3aea3352380833db95e8d546699ef2a67a36 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 29 Oct 2016 16:24:49 +0900 Subject: [PATCH] Update for release --- HOWTO.rst | 27 ++++++++++++----------- README.rst | 1 + RELEASE-NOTES | 11 +++++++++ samples/scripts/NOTES | 22 +++++++++++++----- samples/scripts/env/RPC_PORT | 1 - samples/scripts/env/SSL_PORT | 1 - samples/scripts/env/TCP_PORT | 0 samples/scripts/env/{CACHE_MB => UTXO_MB} | 0 server/env.py | 5 +++-- 9 files changed, 45 insertions(+), 23 deletions(-) create mode 100644 RELEASE-NOTES delete mode 100644 samples/scripts/env/RPC_PORT delete mode 100644 samples/scripts/env/SSL_PORT delete mode 100644 samples/scripts/env/TCP_PORT rename samples/scripts/env/{CACHE_MB => UTXO_MB} (100%) diff --git a/HOWTO.rst b/HOWTO.rst index 5ccc4fe..2f7abd1 100644 --- a/HOWTO.rst +++ b/HOWTO.rst @@ -13,7 +13,7 @@ small - patches welcome. using 1.0.5. While not requirements for running ElectrumX, it is intended to be run -with supervisor software such as Daniel Bernstein's daemontools, +with supervisor software such as Daniel Bernstein's daemontools, Gerald Pape's runit package or systemd. These make administration of secure unix servers very easy, and I strongly recommend you install one of these and familiarise yourself with them. The instructions below and sample @@ -120,7 +120,7 @@ setup ElectrumX with systemd. Simply copy it to :code:`/etc/systemd/system`:: cp samples/systemd-unit /etc/systemd/system/electrumx.service -The sample unit file assumes that the repository is located at +The sample unit file assumes that the repository is located at :code:`/home/electrumx/electrumx`. If that differs on your system, you need to change the unit file accordingly. @@ -140,7 +140,6 @@ Once configured, you may want to start ElectrumX at boot:: systemctl enable electrumx - Sync Progress ============= @@ -161,13 +160,14 @@ Here is my experience with the current codebase, to given heights and rough wall-time:: Machine A Machine B DB + Metadata - 180,000 7m 10s 0.4 GiB - 245,800 1h 00m 2.7 GiB - 290,000 1h 56m 3.3 GiB - 343,000 3h 56m 6.0 GiB - 386,000 7h 28m 7.0 GiB - 404,000 9h 41m - 434,369 14h 38m 17.1 GiB + 181,000 7m 09s 0.4 GiB + 255,000 1h 02m 2.7 GiB + 289,000 1h 46m 3.3 GiB + 317,000 2h 33m + 351,000 3h 58m + 377,000 6h 06m 6.5 GiB + 403,400 8h 51m + 436,196 14h 03m 17.3 GiB Machine A: a low-spec 2011 1.6GHz AMD E-350 dual-core fanless CPU, 8GB RAM and a DragonFlyBSD HAMMER fileystem on an SSD. It requests blocks @@ -175,7 +175,7 @@ over the LAN from a bitcoind on machine B. Machine B: a late 2012 iMac running El-Capitan 10.11.6, 2.9GHz quad-core Intel i5 CPU with an HDD and 24GB RAM. Running bitcoind on -the same machine. HIST_MB of 400, CACHE_MB of 2,000. +the same machine. HIST_MB of 350, UTXO_MB of 1,600. For chains other than bitcoin-mainnet sychronization should be much faster. @@ -192,7 +192,7 @@ by bringing it down like so:: If processing the blockchain the server will start the process of flushing to disk. Once that is complete the server will exit. Be -patient as disk flushing can take a while. +patient as disk flushing can take many minutes. ElectrumX flushes to leveldb using its transaction functionality. The plyvel documentation claims this is atomic. I have written ElectrumX @@ -262,4 +262,5 @@ After flush-to-disk you may see an aiohttp error; this is the daemon timing out the connection while the disk flush was in progress. This is harmless. -The ETA is just a guide and can be quite volatile. +The ETA is just a guide and can be quite volatile. It is too optimistic +initially. diff --git a/README.rst b/README.rst index b44b5e1..ea84b5e 100644 --- a/README.rst +++ b/README.rst @@ -97,6 +97,7 @@ As I've been researching where the time is going during block chain indexing and how various cache sizes and hardware choices affect it, I'd appreciate it if anyone trying to synchronize could tell me:: + - the version of ElectrumX - their O/S and filesystem - their hardware (CPU name and speed, RAM, and disk kind) - whether their daemon was on the same host or not diff --git a/RELEASE-NOTES b/RELEASE-NOTES new file mode 100644 index 0000000..6352940 --- /dev/null +++ b/RELEASE-NOTES @@ -0,0 +1,11 @@ +Version 0.02 +------------ + +- fix bug where tx counts were incorrectly saved +- large clean-up and refactoring of code, breakout into new files +- several efficiency improvements +- initial implementation of chain reorg handling +- work on RPC and TCP server functionality. Code committed but not + functional, so currently disabled +- note that some of the enivronment variables have been renamed, + see samples/scripts/NOTES for the list \ No newline at end of file diff --git a/samples/scripts/NOTES b/samples/scripts/NOTES index 2287788..d38a327 100644 --- a/samples/scripts/NOTES +++ b/samples/scripts/NOTES @@ -5,6 +5,11 @@ NETWORK - see lib/coins.py, must be a coin NET DB_DIRECTORY - path to the database directory (if relative, to run script) USERNAME - the username the server will run as SERVER_MAIN - path to the server_main.py script (if relative, to run script) +DAEMON_URL - the URL used to connect to the daemon. Should be of the form + http://username:password@hostname:port/ + Alternatively you can specify DAEMON_USERNAME, DAEMON_PASSWORD, + DAEMON_HOST and DAEMON_PORT. DAEMON_PORT is optional and + will default appropriately for COIN. In addition either RPC_URL must be given as the full RPC URL for connecting to the daemon, or you must specify RPC_HOST, RPC_USER, @@ -14,6 +19,11 @@ the coin and network otherwise). The other environment variables are all optional and will adopt sensible defaults if not specified. +REORG_LIMIT - maximum number of blocks to be able to handle in a chain + reorganisation. ElectrumX retains some fairly compact + undo information for this many blocks in levelDB. + Default is 200. + Your performance might change by tweaking these cache settings. Cache size is only checked roughly every minute, so the caches can grow beyond the specified size. Also the Python process is often quite a @@ -25,14 +35,14 @@ HIST_MB - amount of history cache, in MB, to retain before flushing to disk. Default is 250; probably no benefit being much larger as history is append-only and not searched. -CACHE_MB- amount of UTXO and history cache, in MB, to retain before +UTXO_MB - amount of UTXO and history cache, in MB, to retain before flushing to disk. Default is 1000. This may be too large for small boxes or too small for machines with lots of RAM. Larger caches generally perform better as there is significant searching of the UTXO cache during indexing. However, I don't see much benefit in my tests pushing this - beyond 2000, and in fact beyond there performance begins to - fall. My machine has 24GB RAM; the slow down is probably - because of leveldb caching and Python GC effects. However - this may be very dependent on hardware and you may have - different results. \ No newline at end of file + too high, and in fact performance begins to fall. My + machine has 24GB RAM; the slow down is probably because of + leveldb caching and Python GC effects. However this may be + very dependent on hardware and you may have different + results. \ No newline at end of file diff --git a/samples/scripts/env/RPC_PORT b/samples/scripts/env/RPC_PORT deleted file mode 100644 index e002b36..0000000 --- a/samples/scripts/env/RPC_PORT +++ /dev/null @@ -1 +0,0 @@ -8000 diff --git a/samples/scripts/env/SSL_PORT b/samples/scripts/env/SSL_PORT deleted file mode 100644 index d1c5b6b..0000000 --- a/samples/scripts/env/SSL_PORT +++ /dev/null @@ -1 +0,0 @@ -50002 diff --git a/samples/scripts/env/TCP_PORT b/samples/scripts/env/TCP_PORT deleted file mode 100644 index e69de29..0000000 diff --git a/samples/scripts/env/CACHE_MB b/samples/scripts/env/UTXO_MB similarity index 100% rename from samples/scripts/env/CACHE_MB rename to samples/scripts/env/UTXO_MB diff --git a/server/env.py b/server/env.py index 0bee0ad..0a447ed 100644 --- a/server/env.py +++ b/server/env.py @@ -22,11 +22,12 @@ class Env(LoggedClass): self.utxo_MB = self.integer('UTXO_MB', 1000) self.hist_MB = self.integer('HIST_MB', 250) self.host = self.default('HOST', 'localhost') + self.reorg_limit = self.integer('REORG_LIMIT', 200) + self.daemon_url = self.build_daemon_url() + # Server stuff self.tcp_port = self.integer('TCP_PORT', None) self.ssl_port = self.integer('SSL_PORT', None) self.rpc_port = self.integer('RPC_PORT', 8000) - self.reorg_limit = self.integer('REORG_LIMIT', 200) - self.daemon_url = self.build_daemon_url() self.max_subscriptions = self.integer('MAX_SUBSCRIPTIONS', 10000) self.banner_file = self.default('BANNER_FILE', None) # The electrum client takes the empty string as unspecified