ThomasV
7 years ago
7 changed files with 508 additions and 480 deletions
@ -0,0 +1,494 @@ |
|||
# Electrum - lightweight Bitcoin client |
|||
# Copyright (C) 2018 The Electrum Developers |
|||
# |
|||
# Permission is hereby granted, free of charge, to any person |
|||
# obtaining a copy of this software and associated documentation files |
|||
# (the "Software"), to deal in the Software without restriction, |
|||
# including without limitation the rights to use, copy, modify, merge, |
|||
# publish, distribute, sublicense, and/or sell copies of the Software, |
|||
# and to permit persons to whom the Software is furnished to do so, |
|||
# subject to the following conditions: |
|||
# |
|||
# The above copyright notice and this permission notice shall be |
|||
# included in all copies or substantial portions of the Software. |
|||
# |
|||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
|||
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
|||
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
|||
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
|||
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
|||
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
|||
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
|||
# SOFTWARE. |
|||
|
|||
import threading |
|||
import itertools |
|||
from collections import defaultdict |
|||
|
|||
from .util import PrintError, profiler |
|||
from .transaction import Transaction |
|||
from .synchronizer import Synchronizer |
|||
from .verifier import SPV |
|||
|
|||
TX_HEIGHT_LOCAL = -2 |
|||
TX_HEIGHT_UNCONF_PARENT = -1 |
|||
TX_HEIGHT_UNCONFIRMED = 0 |
|||
|
|||
class AddTransactionException(Exception): |
|||
pass |
|||
|
|||
|
|||
class UnrelatedTransactionException(AddTransactionException): |
|||
def __str__(self): |
|||
return _("Transaction is unrelated to this wallet.") |
|||
|
|||
class AddressSynchronizer(PrintError): |
|||
""" |
|||
inherited by wallet |
|||
""" |
|||
|
|||
def __init__(self, storage): |
|||
self.storage = storage |
|||
self.network = None |
|||
# verifier (SPV) and synchronizer are started in start_threads |
|||
self.synchronizer = None |
|||
self.verifier = None |
|||
# locks: if you need to take multiple ones, acquire them in the order they are defined here! |
|||
self.lock = threading.RLock() |
|||
self.transaction_lock = threading.RLock() |
|||
# address -> list(txid, height) |
|||
self.history = storage.get('addr_history',{}) |
|||
# Verified transactions. txid -> (height, timestamp, block_pos). Access with self.lock. |
|||
self.verified_tx = storage.get('verified_tx3', {}) |
|||
# Transactions pending verification. txid -> tx_height. Access with self.lock. |
|||
self.unverified_tx = defaultdict(int) |
|||
# true when synchronized |
|||
self.up_to_date = False |
|||
self.load_transactions() |
|||
self.load_local_history() |
|||
self.load_unverified_transactions() |
|||
self.remove_local_transactions_we_dont_have() |
|||
|
|||
def load_unverified_transactions(self): |
|||
# review transactions that are in the history |
|||
for addr, hist in self.history.items(): |
|||
for tx_hash, tx_height in hist: |
|||
# add it in case it was previously unconfirmed |
|||
self.add_unverified_tx(tx_hash, tx_height) |
|||
|
|||
def start_threads(self, network): |
|||
self.network = network |
|||
if self.network is not None: |
|||
self.verifier = SPV(self.network, self) |
|||
self.synchronizer = Synchronizer(self, network) |
|||
network.add_jobs([self.verifier, self.synchronizer]) |
|||
else: |
|||
self.verifier = None |
|||
self.synchronizer = None |
|||
|
|||
def stop_threads(self): |
|||
if self.network: |
|||
self.network.remove_jobs([self.synchronizer, self.verifier]) |
|||
self.synchronizer.release() |
|||
self.synchronizer = None |
|||
self.verifier = None |
|||
# Now no references to the synchronizer or verifier |
|||
# remain so they will be GC-ed |
|||
self.storage.put('stored_height', self.get_local_height()) |
|||
self.save_transactions() |
|||
self.save_verified_tx() |
|||
self.storage.write() |
|||
|
|||
def add_address(self, address): |
|||
if address not in self.history: |
|||
self.history[address] = [] |
|||
self.set_up_to_date(False) |
|||
if self.synchronizer: |
|||
self.synchronizer.add(address) |
|||
|
|||
def get_conflicting_transactions(self, tx): |
|||
"""Returns a set of transaction hashes from the wallet history that are |
|||
directly conflicting with tx, i.e. they have common outpoints being |
|||
spent with tx. If the tx is already in wallet history, that will not be |
|||
reported as a conflict. |
|||
""" |
|||
conflicting_txns = set() |
|||
with self.transaction_lock: |
|||
for txin in tx.inputs(): |
|||
if txin['type'] == 'coinbase': |
|||
continue |
|||
prevout_hash = txin['prevout_hash'] |
|||
prevout_n = txin['prevout_n'] |
|||
spending_tx_hash = self.spent_outpoints[prevout_hash].get(prevout_n) |
|||
if spending_tx_hash is None: |
|||
continue |
|||
# this outpoint has already been spent, by spending_tx |
|||
assert spending_tx_hash in self.transactions |
|||
conflicting_txns |= {spending_tx_hash} |
|||
txid = tx.txid() |
|||
if txid in conflicting_txns: |
|||
# this tx is already in history, so it conflicts with itself |
|||
if len(conflicting_txns) > 1: |
|||
raise Exception('Found conflicting transactions already in wallet history.') |
|||
conflicting_txns -= {txid} |
|||
return conflicting_txns |
|||
|
|||
def add_transaction(self, tx_hash, tx, allow_unrelated=False): |
|||
assert tx_hash, tx_hash |
|||
assert tx, tx |
|||
assert tx.is_complete() |
|||
# we need self.transaction_lock but get_tx_height will take self.lock |
|||
# so we need to take that too here, to enforce order of locks |
|||
with self.lock, self.transaction_lock: |
|||
# NOTE: returning if tx in self.transactions might seem like a good idea |
|||
# BUT we track is_mine inputs in a txn, and during subsequent calls |
|||
# of add_transaction tx, we might learn of more-and-more inputs of |
|||
# being is_mine, as we roll the gap_limit forward |
|||
is_coinbase = tx.inputs()[0]['type'] == 'coinbase' |
|||
tx_height = self.get_tx_height(tx_hash)[0] |
|||
if not allow_unrelated: |
|||
# note that during sync, if the transactions are not properly sorted, |
|||
# it could happen that we think tx is unrelated but actually one of the inputs is is_mine. |
|||
# this is the main motivation for allow_unrelated |
|||
is_mine = any([self.is_mine(self.get_txin_address(txin)) for txin in tx.inputs()]) |
|||
is_for_me = any([self.is_mine(self.get_txout_address(txo)) for txo in tx.outputs()]) |
|||
if not is_mine and not is_for_me: |
|||
raise UnrelatedTransactionException() |
|||
# Find all conflicting transactions. |
|||
# In case of a conflict, |
|||
# 1. confirmed > mempool > local |
|||
# 2. this new txn has priority over existing ones |
|||
# When this method exits, there must NOT be any conflict, so |
|||
# either keep this txn and remove all conflicting (along with dependencies) |
|||
# or drop this txn |
|||
conflicting_txns = self.get_conflicting_transactions(tx) |
|||
if conflicting_txns: |
|||
existing_mempool_txn = any( |
|||
self.get_tx_height(tx_hash2)[0] in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT) |
|||
for tx_hash2 in conflicting_txns) |
|||
existing_confirmed_txn = any( |
|||
self.get_tx_height(tx_hash2)[0] > 0 |
|||
for tx_hash2 in conflicting_txns) |
|||
if existing_confirmed_txn and tx_height <= 0: |
|||
# this is a non-confirmed tx that conflicts with confirmed txns; drop. |
|||
return False |
|||
if existing_mempool_txn and tx_height == TX_HEIGHT_LOCAL: |
|||
# this is a local tx that conflicts with non-local txns; drop. |
|||
return False |
|||
# keep this txn and remove all conflicting |
|||
to_remove = set() |
|||
to_remove |= conflicting_txns |
|||
for conflicting_tx_hash in conflicting_txns: |
|||
to_remove |= self.get_depending_transactions(conflicting_tx_hash) |
|||
for tx_hash2 in to_remove: |
|||
self.remove_transaction(tx_hash2) |
|||
# add inputs |
|||
def add_value_from_prev_output(): |
|||
dd = self.txo.get(prevout_hash, {}) |
|||
# note: this nested loop takes linear time in num is_mine outputs of prev_tx |
|||
for addr, outputs in dd.items(): |
|||
# note: instead of [(n, v, is_cb), ...]; we could store: {n -> (v, is_cb)} |
|||
for n, v, is_cb in outputs: |
|||
if n == prevout_n: |
|||
if addr and self.is_mine(addr): |
|||
if d.get(addr) is None: |
|||
d[addr] = set() |
|||
d[addr].add((ser, v)) |
|||
return |
|||
self.txi[tx_hash] = d = {} |
|||
for txi in tx.inputs(): |
|||
if txi['type'] == 'coinbase': |
|||
continue |
|||
prevout_hash = txi['prevout_hash'] |
|||
prevout_n = txi['prevout_n'] |
|||
ser = prevout_hash + ':%d' % prevout_n |
|||
self.spent_outpoints[prevout_hash][prevout_n] = tx_hash |
|||
add_value_from_prev_output() |
|||
# add outputs |
|||
self.txo[tx_hash] = d = {} |
|||
for n, txo in enumerate(tx.outputs()): |
|||
v = txo[2] |
|||
ser = tx_hash + ':%d'%n |
|||
addr = self.get_txout_address(txo) |
|||
if addr and self.is_mine(addr): |
|||
if d.get(addr) is None: |
|||
d[addr] = [] |
|||
d[addr].append((n, v, is_coinbase)) |
|||
# give v to txi that spends me |
|||
next_tx = self.spent_outpoints[tx_hash].get(n) |
|||
if next_tx is not None: |
|||
dd = self.txi.get(next_tx, {}) |
|||
if dd.get(addr) is None: |
|||
dd[addr] = set() |
|||
if (ser, v) not in dd[addr]: |
|||
dd[addr].add((ser, v)) |
|||
self._add_tx_to_local_history(next_tx) |
|||
# add to local history |
|||
self._add_tx_to_local_history(tx_hash) |
|||
# save |
|||
self.transactions[tx_hash] = tx |
|||
return True |
|||
|
|||
def remove_transaction(self, tx_hash): |
|||
def remove_from_spent_outpoints(): |
|||
# undo spends in spent_outpoints |
|||
if tx is not None: # if we have the tx, this branch is faster |
|||
for txin in tx.inputs(): |
|||
if txin['type'] == 'coinbase': |
|||
continue |
|||
prevout_hash = txin['prevout_hash'] |
|||
prevout_n = txin['prevout_n'] |
|||
self.spent_outpoints[prevout_hash].pop(prevout_n, None) |
|||
if not self.spent_outpoints[prevout_hash]: |
|||
self.spent_outpoints.pop(prevout_hash) |
|||
else: # expensive but always works |
|||
for prevout_hash, d in list(self.spent_outpoints.items()): |
|||
for prevout_n, spending_txid in d.items(): |
|||
if spending_txid == tx_hash: |
|||
self.spent_outpoints[prevout_hash].pop(prevout_n, None) |
|||
if not self.spent_outpoints[prevout_hash]: |
|||
self.spent_outpoints.pop(prevout_hash) |
|||
# Remove this tx itself; if nothing spends from it. |
|||
# It is not so clear what to do if other txns spend from it, but it will be |
|||
# removed when those other txns are removed. |
|||
if not self.spent_outpoints[tx_hash]: |
|||
self.spent_outpoints.pop(tx_hash) |
|||
|
|||
with self.transaction_lock: |
|||
self.print_error("removing tx from history", tx_hash) |
|||
tx = self.transactions.pop(tx_hash, None) |
|||
remove_from_spent_outpoints() |
|||
self._remove_tx_from_local_history(tx_hash) |
|||
self.txi.pop(tx_hash, None) |
|||
self.txo.pop(tx_hash, None) |
|||
|
|||
def receive_tx_callback(self, tx_hash, tx, tx_height): |
|||
self.add_unverified_tx(tx_hash, tx_height) |
|||
self.add_transaction(tx_hash, tx, allow_unrelated=True) |
|||
|
|||
def receive_history_callback(self, addr, hist, tx_fees): |
|||
with self.lock: |
|||
old_hist = self.get_address_history(addr) |
|||
for tx_hash, height in old_hist: |
|||
if (tx_hash, height) not in hist: |
|||
# make tx local |
|||
self.unverified_tx.pop(tx_hash, None) |
|||
self.verified_tx.pop(tx_hash, None) |
|||
if self.verifier: |
|||
self.verifier.remove_spv_proof_for_tx(tx_hash) |
|||
self.history[addr] = hist |
|||
|
|||
for tx_hash, tx_height in hist: |
|||
# add it in case it was previously unconfirmed |
|||
self.add_unverified_tx(tx_hash, tx_height) |
|||
# if addr is new, we have to recompute txi and txo |
|||
tx = self.transactions.get(tx_hash) |
|||
if tx is None: |
|||
continue |
|||
self.add_transaction(tx_hash, tx, allow_unrelated=True) |
|||
|
|||
# Store fees |
|||
self.tx_fees.update(tx_fees) |
|||
|
|||
@profiler |
|||
def load_transactions(self): |
|||
# load txi, txo, tx_fees |
|||
self.txi = self.storage.get('txi', {}) |
|||
for txid, d in list(self.txi.items()): |
|||
for addr, lst in d.items(): |
|||
self.txi[txid][addr] = set([tuple(x) for x in lst]) |
|||
self.txo = self.storage.get('txo', {}) |
|||
self.tx_fees = self.storage.get('tx_fees', {}) |
|||
tx_list = self.storage.get('transactions', {}) |
|||
# load transactions |
|||
self.transactions = {} |
|||
for tx_hash, raw in tx_list.items(): |
|||
tx = Transaction(raw) |
|||
self.transactions[tx_hash] = tx |
|||
if self.txi.get(tx_hash) is None and self.txo.get(tx_hash) is None: |
|||
self.print_error("removing unreferenced tx", tx_hash) |
|||
self.transactions.pop(tx_hash) |
|||
# load spent_outpoints |
|||
_spent_outpoints = self.storage.get('spent_outpoints', {}) |
|||
self.spent_outpoints = defaultdict(dict) |
|||
for prevout_hash, d in _spent_outpoints.items(): |
|||
for prevout_n_str, spending_txid in d.items(): |
|||
prevout_n = int(prevout_n_str) |
|||
self.spent_outpoints[prevout_hash][prevout_n] = spending_txid |
|||
|
|||
@profiler |
|||
def load_local_history(self): |
|||
self._history_local = {} # address -> set(txid) |
|||
for txid in itertools.chain(self.txi, self.txo): |
|||
self._add_tx_to_local_history(txid) |
|||
|
|||
def remove_local_transactions_we_dont_have(self): |
|||
txid_set = set(self.txi) | set(self.txo) |
|||
for txid in txid_set: |
|||
tx_height = self.get_tx_height(txid)[0] |
|||
if tx_height == TX_HEIGHT_LOCAL and txid not in self.transactions: |
|||
self.remove_transaction(txid) |
|||
|
|||
@profiler |
|||
def save_transactions(self, write=False): |
|||
with self.transaction_lock: |
|||
tx = {} |
|||
for k,v in self.transactions.items(): |
|||
tx[k] = str(v) |
|||
self.storage.put('transactions', tx) |
|||
self.storage.put('txi', self.txi) |
|||
self.storage.put('txo', self.txo) |
|||
self.storage.put('tx_fees', self.tx_fees) |
|||
self.storage.put('addr_history', self.history) |
|||
self.storage.put('spent_outpoints', self.spent_outpoints) |
|||
if write: |
|||
self.storage.write() |
|||
|
|||
def save_verified_tx(self, write=False): |
|||
with self.lock: |
|||
self.storage.put('verified_tx3', self.verified_tx) |
|||
if write: |
|||
self.storage.write() |
|||
|
|||
def clear_history(self): |
|||
with self.lock: |
|||
with self.transaction_lock: |
|||
self.txi = {} |
|||
self.txo = {} |
|||
self.tx_fees = {} |
|||
self.spent_outpoints = defaultdict(dict) |
|||
self.history = {} |
|||
self.verified_tx = {} |
|||
self.transactions = {} |
|||
self.save_transactions() |
|||
|
|||
def get_history(self, domain=None): |
|||
# get domain |
|||
if domain is None: |
|||
domain = self.get_addresses() |
|||
domain = set(domain) |
|||
# 1. Get the history of each address in the domain, maintain the |
|||
# delta of a tx as the sum of its deltas on domain addresses |
|||
tx_deltas = defaultdict(int) |
|||
for addr in domain: |
|||
h = self.get_address_history(addr) |
|||
for tx_hash, height in h: |
|||
delta = self.get_tx_delta(tx_hash, addr) |
|||
if delta is None or tx_deltas[tx_hash] is None: |
|||
tx_deltas[tx_hash] = None |
|||
else: |
|||
tx_deltas[tx_hash] += delta |
|||
# 2. create sorted history |
|||
history = [] |
|||
for tx_hash in tx_deltas: |
|||
delta = tx_deltas[tx_hash] |
|||
height, conf, timestamp = self.get_tx_height(tx_hash) |
|||
history.append((tx_hash, height, conf, timestamp, delta)) |
|||
history.sort(key = lambda x: self.get_txpos(x[0])) |
|||
history.reverse() |
|||
# 3. add balance |
|||
c, u, x = self.get_balance(domain) |
|||
balance = c + u + x |
|||
h2 = [] |
|||
for tx_hash, height, conf, timestamp, delta in history: |
|||
h2.append((tx_hash, height, conf, timestamp, delta, balance)) |
|||
if balance is None or delta is None: |
|||
balance = None |
|||
else: |
|||
balance -= delta |
|||
h2.reverse() |
|||
# fixme: this may happen if history is incomplete |
|||
if balance not in [None, 0]: |
|||
self.print_error("Error: history not synchronized") |
|||
return [] |
|||
|
|||
return h2 |
|||
|
|||
def _add_tx_to_local_history(self, txid): |
|||
with self.transaction_lock: |
|||
for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])): |
|||
cur_hist = self._history_local.get(addr, set()) |
|||
cur_hist.add(txid) |
|||
self._history_local[addr] = cur_hist |
|||
|
|||
def _remove_tx_from_local_history(self, txid): |
|||
with self.transaction_lock: |
|||
for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])): |
|||
cur_hist = self._history_local.get(addr, set()) |
|||
try: |
|||
cur_hist.remove(txid) |
|||
except KeyError: |
|||
pass |
|||
else: |
|||
self._history_local[addr] = cur_hist |
|||
|
|||
def add_unverified_tx(self, tx_hash, tx_height): |
|||
if tx_height in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT) \ |
|||
and tx_hash in self.verified_tx: |
|||
with self.lock: |
|||
self.verified_tx.pop(tx_hash) |
|||
if self.verifier: |
|||
self.verifier.remove_spv_proof_for_tx(tx_hash) |
|||
|
|||
# tx will be verified only if height > 0 |
|||
if tx_hash not in self.verified_tx: |
|||
with self.lock: |
|||
self.unverified_tx[tx_hash] = tx_height |
|||
|
|||
def add_verified_tx(self, tx_hash, info): |
|||
# Remove from the unverified map and add to the verified map |
|||
with self.lock: |
|||
self.unverified_tx.pop(tx_hash, None) |
|||
self.verified_tx[tx_hash] = info # (tx_height, timestamp, pos) |
|||
height, conf, timestamp = self.get_tx_height(tx_hash) |
|||
self.network.trigger_callback('verified', tx_hash, height, conf, timestamp) |
|||
|
|||
def get_unverified_txs(self): |
|||
'''Returns a map from tx hash to transaction height''' |
|||
with self.lock: |
|||
return dict(self.unverified_tx) # copy |
|||
|
|||
def undo_verifications(self, blockchain, height): |
|||
'''Used by the verifier when a reorg has happened''' |
|||
txs = set() |
|||
with self.lock: |
|||
for tx_hash, item in list(self.verified_tx.items()): |
|||
tx_height, timestamp, pos = item |
|||
if tx_height >= height: |
|||
header = blockchain.read_header(tx_height) |
|||
# fixme: use block hash, not timestamp |
|||
if not header or header.get('timestamp') != timestamp: |
|||
self.verified_tx.pop(tx_hash, None) |
|||
txs.add(tx_hash) |
|||
return txs |
|||
|
|||
def get_local_height(self): |
|||
""" return last known height if we are offline """ |
|||
return self.network.get_local_height() if self.network else self.storage.get('stored_height', 0) |
|||
|
|||
def get_tx_height(self, tx_hash): |
|||
""" Given a transaction, returns (height, conf, timestamp) """ |
|||
with self.lock: |
|||
if tx_hash in self.verified_tx: |
|||
height, timestamp, pos = self.verified_tx[tx_hash] |
|||
conf = max(self.get_local_height() - height + 1, 0) |
|||
return height, conf, timestamp |
|||
elif tx_hash in self.unverified_tx: |
|||
height = self.unverified_tx[tx_hash] |
|||
return height, 0, None |
|||
else: |
|||
# local transaction |
|||
return TX_HEIGHT_LOCAL, 0, None |
|||
|
|||
def set_up_to_date(self, up_to_date): |
|||
with self.lock: |
|||
self.up_to_date = up_to_date |
|||
if up_to_date: |
|||
self.save_transactions(write=True) |
|||
# if the verifier is also up to date, persist that too; |
|||
# otherwise it will persist its results when it finishes |
|||
if self.verifier and self.verifier.is_up_to_date(): |
|||
self.save_verified_tx(write=True) |
|||
|
|||
def is_up_to_date(self): |
|||
with self.lock: return self.up_to_date |
|||
Loading…
Reference in new issue