From 2d7403f2efed7e8f33c5cb93e2cd9144415cbb9f Mon Sep 17 00:00:00 2001 From: ThomasV Date: Sat, 20 Jan 2018 13:51:38 +0100 Subject: [PATCH] New protocol: (#330) - add method mempool.get_fee_histogram - bump protocol version to 1.2 --- docs/PROTOCOL.rst | 14 +++++++++ server/controller.py | 8 ++++++ server/mempool.py | 68 +++++++++++++++++++++++++++++++++++--------- server/session.py | 6 ++++ server/version.py | 2 +- 5 files changed, 84 insertions(+), 14 deletions(-) diff --git a/docs/PROTOCOL.rst b/docs/PROTOCOL.rst index 6009d3d..c30d615 100644 --- a/docs/PROTOCOL.rst +++ b/docs/PROTOCOL.rst @@ -774,6 +774,20 @@ Subscribe to a script hash. [**scripthash**, **status**] +mempool.get_fee_histogram +========================= + +Return a histogram of the fee rates paid by transactions in the memory +pool, weighted by transaction size. + +The histogram is an array of (fee, vsize) values, where vsize_n is the +cumulative virtual size of mempool transactions with a fee rate in the +interval [fee_(n-1), fee_n)], and fee_(n-1) > fee_n. + +Fee intervals may have variable size. The choice of appropriate +intervals is currently not part of the protocol. + + server.add_peer =============== diff --git a/server/controller.py b/server/controller.py index e7e45b9..57b80c7 100644 --- a/server/controller.py +++ b/server/controller.py @@ -825,6 +825,14 @@ class Controller(ServerBase): number = self.non_negative_integer(number) return await self.daemon_request('estimatefee', [number]) + def mempool_get_fee_histogram(self): + '''Memory pool fee histogram. + + TODO: The server should detect and discount transactions that + never get mined when they should. + ''' + return self.mempool.get_fee_histogram() + async def relayfee(self): '''The minimum fee a low-priority tx must pay in order to be accepted to the daemon's memory pool.''' diff --git a/server/mempool.py b/server/mempool.py index 4193c83..286539c 100644 --- a/server/mempool.py +++ b/server/mempool.py @@ -25,7 +25,7 @@ class MemPool(util.LoggedClass): To that end we maintain the following maps: - tx_hash -> (txin_pairs, txout_pairs) + tx_hash -> (txin_pairs, txout_pairs, tx_fee, tx_size) hashX -> set of all tx hashes in which the hashX appears A pair is a (hashX, value) tuple. tx hashes are hex strings. @@ -42,6 +42,9 @@ class MemPool(util.LoggedClass): self.txs = {} self.hashXs = defaultdict(set) # None can be a key self.synchronized_event = asyncio.Event() + self.fee_histogram = defaultdict(int) + self.compact_fee_histogram = [] + self.histogram_time = 0 def _resync_daemon_hashes(self, unprocessed, unfetched): '''Re-sync self.txs with the list of hashes in the daemon's mempool. @@ -52,6 +55,7 @@ class MemPool(util.LoggedClass): txs = self.txs hashXs = self.hashXs touched = self.touched + fee_hist = self.fee_histogram hashes = self.daemon.cached_mempool_hashes() gone = set(txs).difference(hashes) @@ -60,7 +64,11 @@ class MemPool(util.LoggedClass): unprocessed.pop(hex_hash, None) item = txs.pop(hex_hash) if item: - txin_pairs, txout_pairs = item + txin_pairs, txout_pairs, tx_fee, tx_size = item + fee_rate = tx_fee // tx_size + fee_hist[fee_rate] -= tx_size + if fee_hist[fee_rate] == 0: + fee_hist.pop(fee_rate) tx_hashXs = set(hashX for hashX, value in txin_pairs) tx_hashXs.update(hashX for hashX, value in txout_pairs) for hashX in tx_hashXs: @@ -138,6 +146,7 @@ class MemPool(util.LoggedClass): def _async_process_some(self, limit): pending = [] txs = self.txs + fee_hist = self.fee_histogram async def process(unprocessed): nonlocal pending @@ -160,10 +169,13 @@ class MemPool(util.LoggedClass): pending.extend(deferred) hashXs = self.hashXs touched = self.touched - for hex_hash, in_out_pairs in result.items(): + for hex_hash, item in result.items(): if hex_hash in txs: - txs[hex_hash] = in_out_pairs - for hashX, value in itertools.chain(*in_out_pairs): + txs[hex_hash] = item + txin_pairs, txout_pairs, tx_fee, tx_size = item + fee_rate = tx_fee // tx_size + fee_hist[fee_rate] += tx_size + for hashX, value in itertools.chain(txin_pairs, txout_pairs): touched.add(hashX) hashXs[hashX].add(hex_hash) @@ -209,7 +221,7 @@ class MemPool(util.LoggedClass): for tx_hash, raw_tx in raw_tx_map.items(): if tx_hash not in txs: continue - tx = deserializer(raw_tx).read_tx() + tx, tx_size = deserializer(raw_tx).read_tx_and_vsize() # Convert the tx outputs into (hashX, value) pairs txout_pairs = [(script_hashX(txout.pk_script), txout.value) @@ -219,7 +231,7 @@ class MemPool(util.LoggedClass): txin_pairs = [(hash_to_str(txin.prev_hash), txin.prev_idx) for txin in tx.inputs] - pending.append((tx_hash, txin_pairs, txout_pairs)) + pending.append((tx_hash, txin_pairs, txout_pairs, tx_size)) # Now process what we can result = {} @@ -229,7 +241,7 @@ class MemPool(util.LoggedClass): if self.stop: break - tx_hash, old_txin_pairs, txout_pairs = item + tx_hash, old_txin_pairs, txout_pairs, tx_size = item if tx_hash not in txs: continue @@ -259,7 +271,10 @@ class MemPool(util.LoggedClass): if mempool_missing: deferred.append(item) else: - result[tx_hash] = (txin_pairs, txout_pairs) + # Compute fee + tx_fee = (sum(v for hashX, v in txin_pairs) - + sum(v for hashX, v in txout_pairs)) + result[tx_hash] = (txin_pairs, txout_pairs, tx_fee, tx_size) return result, deferred @@ -290,9 +305,7 @@ class MemPool(util.LoggedClass): item = self.txs.get(hex_hash) if not item or not raw_tx: continue - txin_pairs, txout_pairs = item - tx_fee = (sum(v for hashX, v in txin_pairs) - - sum(v for hashX, v in txout_pairs)) + txin_pairs, txout_pairs, tx_fee, tx_size = item tx = deserializer(raw_tx).read_tx() unconfirmed = any(hash_to_str(txin.prev_hash) in self.txs for txin in tx.inputs) @@ -325,7 +338,36 @@ class MemPool(util.LoggedClass): # hashXs is a defaultdict if hashX in self.hashXs: for hex_hash in self.hashXs[hashX]: - txin_pairs, txout_pairs = self.txs[hex_hash] + txin_pairs, txout_pairs, tx_fee, tx_size = self.txs[hex_hash] value -= sum(v for h168, v in txin_pairs if h168 == hashX) value += sum(v for h168, v in txout_pairs if h168 == hashX) return value + + def get_fee_histogram(self): + now = time.time() + if now > self.histogram_time + 30: + self.update_compact_histogram() + self.histogram_time = now + return self.compact_fee_histogram + + def update_compact_histogram(self): + # For efficiency, get_fees returns a compact histogram with + # variable bin size. The compact histogram is an array of + # (fee, vsize) values. vsize_n is the cumulative virtual size + # of mempool transactions with a fee rate in the interval + # [fee_(n-1), fee_n)], and fee_(n-1) > fee_n. Fee intervals + # are chosen so as to create tranches that contain at least + # 100kb of transactions + l = list(reversed(sorted(self.fee_histogram.items()))) + out = [] + size = 0 + r = 0 + binsize = 100000 + for fee, s in l: + size += s + if size + r > binsize: + out.append((fee, size)) + r += size - binsize + size = 0 + binsize *= 1.1 + self.compact_fee_histogram = out diff --git a/server/session.py b/server/session.py index 39c2727..91ba12a 100644 --- a/server/session.py +++ b/server/session.py @@ -449,6 +449,12 @@ class ElectrumX(SessionBase): 'blockchain.transaction.get': controller.transaction_get, }) + if ptuple >= (1, 2): + # New handler as of 1.2 + handlers.update({ + 'mempool.get_fee_histogram': controller.mempool_get_fee_histogram, + }) + self.electrumx_handlers = handlers def request_handler(self, method): diff --git a/server/version.py b/server/version.py index 5c69573..f0edc0d 100644 --- a/server/version.py +++ b/server/version.py @@ -2,4 +2,4 @@ VERSION = 'ElectrumX 1.2.1' PROTOCOL_MIN = '0.9' -PROTOCOL_MAX = '1.1' +PROTOCOL_MAX = '1.2'