From d8408048186c025520cd63d6ffe705f8d0316f44 Mon Sep 17 00:00:00 2001 From: Ilya Shalyapin Date: Wed, 19 Sep 2018 13:07:19 +0500 Subject: [PATCH 01/40] use system language by default --- electrum/gui/qt/__init__.py | 4 ++-- electrum/i18n.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/electrum/gui/qt/__init__.py b/electrum/gui/qt/__init__.py index 80adeaa99..4fdb1b52e 100644 --- a/electrum/gui/qt/__init__.py +++ b/electrum/gui/qt/__init__.py @@ -38,7 +38,7 @@ from PyQt5.QtWidgets import * from PyQt5.QtCore import * import PyQt5.QtCore as QtCore -from electrum.i18n import _, set_language +from electrum.i18n import _, set_language, get_default_language from electrum.plugin import run_hook from electrum.storage import WalletStorage from electrum.base_wizard import GoBack @@ -89,7 +89,7 @@ class QNetworkUpdatedSignalObject(QObject): class ElectrumGui(PrintError): def __init__(self, config, daemon, plugins): - set_language(config.get('language')) + set_language(config.get('language', get_default_language())) # Uncomment this call to verify objects are being properly # GC-ed when windows are closed #network.add_jobs([DebugMem([Abstract_Wallet, SPV, Synchronizer, diff --git a/electrum/i18n.py b/electrum/i18n.py index 9c6fad995..bb4a2da59 100644 --- a/electrum/i18n.py +++ b/electrum/i18n.py @@ -26,6 +26,8 @@ import os import gettext +from PyQt5.QtCore import QLocale + LOCALE_DIR = os.path.join(os.path.dirname(__file__), 'locale') language = gettext.translation('electrum', LOCALE_DIR, fallback=True) @@ -41,6 +43,11 @@ def set_language(x): language = gettext.translation('electrum', LOCALE_DIR, fallback=True, languages=[x]) +def get_default_language(): + system_locale = QLocale.system().name() + return languages.get(system_locale, 'en_UK') + + languages = { '': _('Default'), 'ar_SA': _('Arabic'), From 4c8103af3bd18bb7d2fee6fcc74706e4dcb06a90 Mon Sep 17 00:00:00 2001 From: Ilya Shalyapin Date: Sun, 23 Sep 2018 14:11:50 +0500 Subject: [PATCH 02/40] move get_default_language to gui.qt.util --- electrum/gui/qt/__init__.py | 2 +- electrum/gui/qt/util.py | 7 ++++++- electrum/i18n.py | 7 ------- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/electrum/gui/qt/__init__.py b/electrum/gui/qt/__init__.py index 4fdb1b52e..7557f0ecf 100644 --- a/electrum/gui/qt/__init__.py +++ b/electrum/gui/qt/__init__.py @@ -38,7 +38,7 @@ from PyQt5.QtWidgets import * from PyQt5.QtCore import * import PyQt5.QtCore as QtCore -from electrum.i18n import _, set_language, get_default_language +from electrum.i18n import _, set_language from electrum.plugin import run_hook from electrum.storage import WalletStorage from electrum.base_wizard import GoBack diff --git a/electrum/gui/qt/util.py b/electrum/gui/qt/util.py index 50ed0a5c3..804a99078 100644 --- a/electrum/gui/qt/util.py +++ b/electrum/gui/qt/util.py @@ -10,7 +10,7 @@ from PyQt5.QtGui import * from PyQt5.QtCore import * from PyQt5.QtWidgets import * -from electrum.i18n import _ +from electrum.i18n import _, languages from electrum.util import FileImportFailed, FileExportFailed from electrum.paymentrequest import PR_UNPAID, PR_PAID, PR_EXPIRED @@ -812,6 +812,11 @@ class IconCache: return self.__cache[file_name] +def get_default_language(): + name = QLocale.system().name() + return name if name in languages else 'en_UK' + + if __name__ == "__main__": app = QApplication([]) t = WaitingDialog(None, 'testing ...', lambda: [time.sleep(1)], lambda x: QMessageBox.information(None, 'done', "done")) diff --git a/electrum/i18n.py b/electrum/i18n.py index bb4a2da59..9c6fad995 100644 --- a/electrum/i18n.py +++ b/electrum/i18n.py @@ -26,8 +26,6 @@ import os import gettext -from PyQt5.QtCore import QLocale - LOCALE_DIR = os.path.join(os.path.dirname(__file__), 'locale') language = gettext.translation('electrum', LOCALE_DIR, fallback=True) @@ -43,11 +41,6 @@ def set_language(x): language = gettext.translation('electrum', LOCALE_DIR, fallback=True, languages=[x]) -def get_default_language(): - system_locale = QLocale.system().name() - return languages.get(system_locale, 'en_UK') - - languages = { '': _('Default'), 'ar_SA': _('Arabic'), From 48b0de78719dd349a096c037870ee801e9bd9ac0 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 10 Nov 2018 15:30:41 +0100 Subject: [PATCH 03/40] keystore: stronger pbkdf for encryption --- electrum/crypto.py | 95 ++++++++++++++++--- electrum/keystore.py | 58 +++++++---- .../plugins/digitalbitbox/digitalbitbox.py | 8 +- electrum/tests/test_bitcoin.py | 27 +++--- 4 files changed, 138 insertions(+), 50 deletions(-) diff --git a/electrum/crypto.py b/electrum/crypto.py index 345fbd85a..752dfec83 100644 --- a/electrum/crypto.py +++ b/electrum/crypto.py @@ -32,6 +32,7 @@ from typing import Union import pyaes from .util import assert_bytes, InvalidPassword, to_bytes, to_string +from .i18n import _ try: @@ -90,37 +91,103 @@ def aes_decrypt_with_iv(key: bytes, iv: bytes, data: bytes) -> bytes: raise InvalidPassword() -def EncodeAES(secret: bytes, msg: bytes) -> bytes: +def EncodeAES_base64(secret: bytes, msg: bytes) -> bytes: """Returns base64 encoded ciphertext.""" + e = EncodeAES_bytes(secret, msg) + return base64.b64encode(e) + + +def EncodeAES_bytes(secret: bytes, msg: bytes) -> bytes: assert_bytes(msg) iv = bytes(os.urandom(16)) ct = aes_encrypt_with_iv(secret, iv, msg) - e = iv + ct - return base64.b64encode(e) + return iv + ct + +def DecodeAES_base64(secret: bytes, ciphertext_b64: Union[bytes, str]) -> bytes: + ciphertext = bytes(base64.b64decode(ciphertext_b64)) + return DecodeAES_bytes(secret, ciphertext) -def DecodeAES(secret: bytes, ciphertext_b64: Union[bytes, str]) -> bytes: - e = bytes(base64.b64decode(ciphertext_b64)) - iv, e = e[:16], e[16:] + +def DecodeAES_bytes(secret: bytes, ciphertext: bytes) -> bytes: + assert_bytes(ciphertext) + iv, e = ciphertext[:16], ciphertext[16:] s = aes_decrypt_with_iv(secret, iv, e) return s -def pw_encode(data: str, password: Union[bytes, str]) -> str: +PW_HASH_VERSION_LATEST = 2 +KNOWN_PW_HASH_VERSIONS = (1, 2) +assert PW_HASH_VERSION_LATEST in KNOWN_PW_HASH_VERSIONS + + +class UnexpectedPasswordHashVersion(InvalidPassword): + def __init__(self, version): + self.version = version + + def __str__(self): + return "{unexpected}: {version}\n{please_update}".format( + unexpected=_("Unexpected password hash version"), + version=self.version, + please_update=_('You are most likely using an outdated version of Electrum. Please update.')) + + +def _hash_password(password: Union[bytes, str], *, version: int, salt: bytes) -> bytes: + pw = to_bytes(password, 'utf8') + if version == 1: + return sha256d(pw) + elif version == 2: + if not isinstance(salt, bytes) or len(salt) < 16: + raise Exception('too weak salt', salt) + return hashlib.pbkdf2_hmac(hash_name='sha256', + password=pw, + salt=b'ELECTRUM_PW_HASH_V2'+salt, + iterations=50_000) + else: + assert version not in KNOWN_PW_HASH_VERSIONS + raise UnexpectedPasswordHashVersion(version) + + +def pw_encode(data: str, password: Union[bytes, str, None], *, version: int) -> str: if not password: return data - secret = sha256d(password) - return EncodeAES(secret, to_bytes(data, "utf8")).decode('utf8') + if version not in KNOWN_PW_HASH_VERSIONS: + raise UnexpectedPasswordHashVersion(version) + # derive key from password + if version == 1: + salt = b'' + elif version == 2: + salt = bytes(os.urandom(16)) + else: + assert False, version + secret = _hash_password(password, version=version, salt=salt) + # encrypt given data + e = EncodeAES_bytes(secret, to_bytes(data, "utf8")) + # return base64(salt + encrypted data) + ciphertext = salt + e + ciphertext_b64 = base64.b64encode(ciphertext) + return ciphertext_b64.decode('utf8') -def pw_decode(data: str, password: Union[bytes, str]) -> str: +def pw_decode(data: str, password: Union[bytes, str, None], *, version: int) -> str: if password is None: return data - secret = sha256d(password) + if version not in KNOWN_PW_HASH_VERSIONS: + raise UnexpectedPasswordHashVersion(version) + data_bytes = bytes(base64.b64decode(data)) + # derive key from password + if version == 1: + salt = b'' + elif version == 2: + salt, data_bytes = data_bytes[:16], data_bytes[16:] + else: + assert False, version + secret = _hash_password(password, version=version, salt=salt) + # decrypt given data try: - d = to_string(DecodeAES(secret, data), "utf8") - except Exception: - raise InvalidPassword() + d = to_string(DecodeAES_bytes(secret, data_bytes), "utf8") + except Exception as e: + raise InvalidPassword() from e return d diff --git a/electrum/keystore.py b/electrum/keystore.py index a942d0751..e0e21fa60 100644 --- a/electrum/keystore.py +++ b/electrum/keystore.py @@ -35,7 +35,7 @@ from .bip32 import (bip32_public_derivation, deserialize_xpub, CKD_pub, bip32_private_key, bip32_derivation, BIP32_PRIME, is_xpub, is_xprv) from .ecc import string_to_number, number_to_string -from .crypto import pw_decode, pw_encode, sha256d +from .crypto import (pw_decode, pw_encode, sha256d, PW_HASH_VERSION_LATEST) from .util import (PrintError, InvalidPassword, hfu, WalletFileException, BitcoinException, bh2u, bfh, print_error, inv_dict) from .mnemonic import Mnemonic, load_wordlist @@ -92,8 +92,9 @@ class KeyStore(PrintError): class Software_KeyStore(KeyStore): - def __init__(self): + def __init__(self, d): KeyStore.__init__(self) + self.pw_hash_version = d.get('pw_hash_version', 1) def may_have_password(self): return not self.is_watching_only() @@ -122,6 +123,12 @@ class Software_KeyStore(KeyStore): if keypairs: tx.sign(keypairs) + def update_password(self, old_password, new_password): + raise NotImplementedError() # implemented by subclasses + + def check_password(self, password): + raise NotImplementedError() # implemented by subclasses + class Imported_KeyStore(Software_KeyStore): # keystore for imported private keys @@ -129,7 +136,7 @@ class Imported_KeyStore(Software_KeyStore): type = 'imported' def __init__(self, d): - Software_KeyStore.__init__(self) + Software_KeyStore.__init__(self, d) self.keypairs = d.get('keypairs', {}) def is_deterministic(self): @@ -142,6 +149,7 @@ class Imported_KeyStore(Software_KeyStore): return { 'type': self.type, 'keypairs': self.keypairs, + 'pw_hash_version': self.pw_hash_version, } def can_import(self): @@ -161,14 +169,14 @@ class Imported_KeyStore(Software_KeyStore): # there will only be one pubkey-privkey pair for it in self.keypairs, # and the privkey will encode a txin_type but that txin_type cannot be trusted. # Removing keys complicates this further. - self.keypairs[pubkey] = pw_encode(serialized_privkey, password) + self.keypairs[pubkey] = pw_encode(serialized_privkey, password, version=self.pw_hash_version) return txin_type, pubkey def delete_imported_key(self, key): self.keypairs.pop(key) def get_private_key(self, pubkey, password): - sec = pw_decode(self.keypairs[pubkey], password) + sec = pw_decode(self.keypairs[pubkey], password, version=self.pw_hash_version) txin_type, privkey, compressed = deserialize_privkey(sec) # this checks the password if pubkey != ecc.ECPrivkey(privkey).get_public_key_hex(compressed=compressed): @@ -189,16 +197,17 @@ class Imported_KeyStore(Software_KeyStore): if new_password == '': new_password = None for k, v in self.keypairs.items(): - b = pw_decode(v, old_password) - c = pw_encode(b, new_password) + b = pw_decode(v, old_password, version=self.pw_hash_version) + c = pw_encode(b, new_password, version=PW_HASH_VERSION_LATEST) self.keypairs[k] = c + self.pw_hash_version = PW_HASH_VERSION_LATEST class Deterministic_KeyStore(Software_KeyStore): def __init__(self, d): - Software_KeyStore.__init__(self) + Software_KeyStore.__init__(self, d) self.seed = d.get('seed', '') self.passphrase = d.get('passphrase', '') @@ -206,12 +215,14 @@ class Deterministic_KeyStore(Software_KeyStore): return True def dump(self): - d = {} + d = { + 'type': self.type, + 'pw_hash_version': self.pw_hash_version, + } if self.seed: d['seed'] = self.seed if self.passphrase: d['passphrase'] = self.passphrase - d['type'] = self.type return d def has_seed(self): @@ -226,10 +237,13 @@ class Deterministic_KeyStore(Software_KeyStore): self.seed = self.format_seed(seed) def get_seed(self, password): - return pw_decode(self.seed, password) + return pw_decode(self.seed, password, version=self.pw_hash_version) def get_passphrase(self, password): - return pw_decode(self.passphrase, password) if self.passphrase else '' + if self.passphrase: + return pw_decode(self.passphrase, password, version=self.pw_hash_version) + else: + return '' class Xpub: @@ -312,10 +326,10 @@ class BIP32_KeyStore(Deterministic_KeyStore, Xpub): return d def get_master_private_key(self, password): - return pw_decode(self.xprv, password) + return pw_decode(self.xprv, password, version=self.pw_hash_version) def check_password(self, password): - xprv = pw_decode(self.xprv, password) + xprv = pw_decode(self.xprv, password, version=self.pw_hash_version) if deserialize_xprv(xprv)[4] != deserialize_xpub(self.xpub)[4]: raise InvalidPassword() @@ -325,13 +339,14 @@ class BIP32_KeyStore(Deterministic_KeyStore, Xpub): new_password = None if self.has_seed(): decoded = self.get_seed(old_password) - self.seed = pw_encode(decoded, new_password) + self.seed = pw_encode(decoded, new_password, version=PW_HASH_VERSION_LATEST) if self.passphrase: decoded = self.get_passphrase(old_password) - self.passphrase = pw_encode(decoded, new_password) + self.passphrase = pw_encode(decoded, new_password, version=PW_HASH_VERSION_LATEST) if self.xprv is not None: - b = pw_decode(self.xprv, old_password) - self.xprv = pw_encode(b, new_password) + b = pw_decode(self.xprv, old_password, version=self.pw_hash_version) + self.xprv = pw_encode(b, new_password, version=PW_HASH_VERSION_LATEST) + self.pw_hash_version = PW_HASH_VERSION_LATEST def is_watching_only(self): return self.xprv is None @@ -362,7 +377,7 @@ class Old_KeyStore(Deterministic_KeyStore): self.mpk = d.get('mpk') def get_hex_seed(self, password): - return pw_decode(self.seed, password).encode('utf8') + return pw_decode(self.seed, password, version=self.pw_hash_version).encode('utf8') def dump(self): d = Deterministic_KeyStore.dump(self) @@ -484,8 +499,9 @@ class Old_KeyStore(Deterministic_KeyStore): if new_password == '': new_password = None if self.has_seed(): - decoded = pw_decode(self.seed, old_password) - self.seed = pw_encode(decoded, new_password) + decoded = pw_decode(self.seed, old_password, version=self.pw_hash_version) + self.seed = pw_encode(decoded, new_password, version=PW_HASH_VERSION_LATEST) + self.pw_hash_version = PW_HASH_VERSION_LATEST diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index 25e69e27d..dd93f77fd 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -4,7 +4,7 @@ # try: - from electrum.crypto import sha256d, EncodeAES, DecodeAES + from electrum.crypto import sha256d, EncodeAES_base64, DecodeAES_base64 from electrum.bitcoin import (TYPE_ADDRESS, push_script, var_int, public_key_to_p2pkh, is_address) from electrum.bip32 import serialize_xpub, deserialize_xpub @@ -396,10 +396,10 @@ class DigitalBitbox_Client(): reply = "" try: secret = sha256d(self.password) - msg = EncodeAES(secret, msg) + msg = EncodeAES_base64(secret, msg) reply = self.hid_send_plain(msg) if 'ciphertext' in reply: - reply = DecodeAES(secret, ''.join(reply["ciphertext"])) + reply = DecodeAES_base64(secret, ''.join(reply["ciphertext"])) reply = to_string(reply, 'utf8') reply = json.loads(reply) if 'error' in reply: @@ -716,7 +716,7 @@ class DigitalBitboxPlugin(HW_PluginBase): key_s = base64.b64decode(self.digitalbitbox_config['encryptionprivkey']) args = 'c=data&s=0&dt=0&uuid=%s&pl=%s' % ( self.digitalbitbox_config['comserverchannelid'], - EncodeAES(key_s, json.dumps(payload).encode('ascii')).decode('ascii'), + EncodeAES_base64(key_s, json.dumps(payload).encode('ascii')).decode('ascii'), ) try: requests.post(url, args) diff --git a/electrum/tests/test_bitcoin.py b/electrum/tests/test_bitcoin.py index e11cee4ec..6217ed5bb 100644 --- a/electrum/tests/test_bitcoin.py +++ b/electrum/tests/test_bitcoin.py @@ -11,11 +11,11 @@ from electrum.bitcoin import (public_key_to_p2pkh, address_from_private_key, from electrum.bip32 import (bip32_root, bip32_public_derivation, bip32_private_derivation, xpub_from_xprv, xpub_type, is_xprv, is_bip32_derivation, is_xpub, convert_bip32_path_to_list_of_uint32) -from electrum.crypto import sha256d +from electrum.crypto import sha256d, KNOWN_PW_HASH_VERSIONS from electrum import ecc, crypto, constants from electrum.ecc import number_to_string, string_to_number from electrum.transaction import opcodes -from electrum.util import bfh, bh2u +from electrum.util import bfh, bh2u, InvalidPassword from electrum.storage import WalletStorage from electrum.keystore import xtype_from_derivation @@ -219,23 +219,26 @@ class Test_bitcoin(SequentialTestCase): """Make sure AES is homomorphic.""" payload = u'\u66f4\u7a33\u5b9a\u7684\u4ea4\u6613\u5e73\u53f0' password = u'secret' - enc = crypto.pw_encode(payload, password) - dec = crypto.pw_decode(enc, password) - self.assertEqual(dec, payload) + for version in KNOWN_PW_HASH_VERSIONS: + enc = crypto.pw_encode(payload, password, version=version) + dec = crypto.pw_decode(enc, password, version=version) + self.assertEqual(dec, payload) @needs_test_with_all_aes_implementations def test_aes_encode_without_password(self): """When not passed a password, pw_encode is noop on the payload.""" payload = u'\u66f4\u7a33\u5b9a\u7684\u4ea4\u6613\u5e73\u53f0' - enc = crypto.pw_encode(payload, None) - self.assertEqual(payload, enc) + for version in KNOWN_PW_HASH_VERSIONS: + enc = crypto.pw_encode(payload, None, version=version) + self.assertEqual(payload, enc) @needs_test_with_all_aes_implementations def test_aes_deencode_without_password(self): """When not passed a password, pw_decode is noop on the payload.""" payload = u'\u66f4\u7a33\u5b9a\u7684\u4ea4\u6613\u5e73\u53f0' - enc = crypto.pw_decode(payload, None) - self.assertEqual(payload, enc) + for version in KNOWN_PW_HASH_VERSIONS: + enc = crypto.pw_decode(payload, None, version=version) + self.assertEqual(payload, enc) @needs_test_with_all_aes_implementations def test_aes_decode_with_invalid_password(self): @@ -243,8 +246,10 @@ class Test_bitcoin(SequentialTestCase): payload = u"blah" password = u"uber secret" wrong_password = u"not the password" - enc = crypto.pw_encode(payload, password) - self.assertRaises(Exception, crypto.pw_decode, enc, wrong_password) + for version in KNOWN_PW_HASH_VERSIONS: + enc = crypto.pw_encode(payload, password, version=version) + with self.assertRaises(InvalidPassword): + crypto.pw_decode(enc, wrong_password, version=version) def test_sha256d(self): self.assertEqual(b'\x95MZI\xfdp\xd9\xb8\xbc\xdb5\xd2R&x)\x95\x7f~\xf7\xfalt\xf8\x84\x19\xbd\xc5\xe8"\t\xf4', From 141ff99580192c920bc6bb7f6bbc9d35449daea8 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 20 Nov 2018 18:57:16 +0100 Subject: [PATCH 04/40] blockchain.py: generalise fork ids to get rid of conflicts --- electrum/blockchain.py | 252 ++++++++++++++++++++---------- electrum/gui/kivy/main_window.py | 8 +- electrum/gui/qt/network_dialog.py | 21 +-- electrum/interface.py | 65 ++------ electrum/network.py | 38 ++--- electrum/storage.py | 2 +- electrum/tests/test_blockchain.py | 239 ++++++++++++++++++++++++++++ electrum/tests/test_network.py | 20 ++- electrum/tests/test_wallet.py | 1 - 9 files changed, 474 insertions(+), 172 deletions(-) create mode 100644 electrum/tests/test_blockchain.py diff --git a/electrum/blockchain.py b/electrum/blockchain.py index 5609c41e2..c7c075312 100644 --- a/electrum/blockchain.py +++ b/electrum/blockchain.py @@ -79,26 +79,67 @@ def hash_raw_header(header: str) -> str: return hash_encode(sha256d(bfh(header))) -blockchains = {} # type: Dict[int, Blockchain] -blockchains_lock = threading.Lock() - - -def read_blockchains(config: 'SimpleConfig') -> Dict[int, 'Blockchain']: - blockchains[0] = Blockchain(config, 0, None) +# key: blockhash hex at forkpoint +# the chain at some key is the best chain that includes the given hash +blockchains = {} # type: Dict[str, Blockchain] +blockchains_lock = threading.RLock() + + +def read_blockchains(config: 'SimpleConfig'): + blockchains[constants.net.GENESIS] = Blockchain(config=config, + forkpoint=0, + parent=None, + forkpoint_hash=constants.net.GENESIS, + prev_hash=None) fdir = os.path.join(util.get_headers_dir(config), 'forks') util.make_dir(fdir) - l = filter(lambda x: x.startswith('fork_'), os.listdir(fdir)) - l = sorted(l, key = lambda x: int(x.split('_')[1])) - for filename in l: - forkpoint = int(filename.split('_')[2]) - parent_id = int(filename.split('_')[1]) - b = Blockchain(config, forkpoint, parent_id) - h = b.read_header(b.forkpoint) - if b.parent().can_connect(h, check_height=False): - blockchains[b.forkpoint] = b + # files are named as: fork2_{forkpoint}_{prev_hash}_{first_hash} + l = filter(lambda x: x.startswith('fork2_') and '.' not in x, os.listdir(fdir)) + l = sorted(l, key=lambda x: int(x.split('_')[1])) # sort by forkpoint + + def delete_chain(filename, reason): + util.print_error("[blockchain]", reason, filename) + os.unlink(os.path.join(fdir, filename)) + + def instantiate_chain(filename): + __, forkpoint, prev_hash, first_hash = filename.split('_') + forkpoint = int(forkpoint) + prev_hash = (64-len(prev_hash)) * "0" + prev_hash # left-pad with zeroes + first_hash = (64-len(first_hash)) * "0" + first_hash + # forks below the max checkpoint are not allowed + if forkpoint <= constants.net.max_checkpoint(): + delete_chain(filename, "deleting fork below max checkpoint") + return + # find parent (sorting by forkpoint guarantees it's already instantiated) + for parent in blockchains.values(): + if parent.check_hash(forkpoint - 1, prev_hash): + break else: - util.print_error("cannot connect", filename) - return blockchains + delete_chain(filename, "cannot find parent for chain") + return + b = Blockchain(config=config, + forkpoint=forkpoint, + parent=parent, + forkpoint_hash=first_hash, + prev_hash=prev_hash) + # consistency checks + h = b.read_header(b.forkpoint) + if first_hash != hash_header(h): + delete_chain(filename, "incorrect first hash for chain") + return + if not b.parent.can_connect(h, check_height=False): + delete_chain(filename, "cannot connect chain to parent") + return + chain_id = b.get_id() + assert first_hash == chain_id, (first_hash, chain_id) + blockchains[chain_id] = b + + for filename in l: + instantiate_chain(filename) + + +def get_best_chain() -> 'Blockchain': + return blockchains[constants.net.GENESIS] class Blockchain(util.PrintError): @@ -106,15 +147,20 @@ class Blockchain(util.PrintError): Manages blockchain headers and their verification """ - def __init__(self, config: SimpleConfig, forkpoint: int, parent_id: Optional[int]): + def __init__(self, config: SimpleConfig, forkpoint: int, parent: Optional['Blockchain'], + forkpoint_hash: str, prev_hash: Optional[str]): + assert isinstance(forkpoint_hash, str) and len(forkpoint_hash) == 64, forkpoint_hash + assert (prev_hash is None) or (isinstance(prev_hash, str) and len(prev_hash) == 64), prev_hash + # assert (parent is None) == (forkpoint == 0) + if 0 < forkpoint <= constants.net.max_checkpoint(): + raise Exception(f"cannot fork below max checkpoint. forkpoint: {forkpoint}") self.config = config - self.forkpoint = forkpoint - self.checkpoints = constants.net.CHECKPOINTS - self.parent_id = parent_id - assert parent_id != forkpoint + self.forkpoint = forkpoint # height of first header + self.parent = parent + self._forkpoint_hash = forkpoint_hash # blockhash at forkpoint. "first hash" + self._prev_hash = prev_hash # blockhash immediately before forkpoint self.lock = threading.RLock() - with self.lock: - self.update_size() + self.update_size() def with_lock(func): def func_wrapper(self, *args, **kwargs): @@ -122,12 +168,13 @@ class Blockchain(util.PrintError): return func(self, *args, **kwargs) return func_wrapper - def parent(self) -> 'Blockchain': - return blockchains[self.parent_id] + @property + def checkpoints(self): + return constants.net.CHECKPOINTS def get_max_child(self) -> Optional[int]: with blockchains_lock: chains = list(blockchains.values()) - children = list(filter(lambda y: y.parent_id==self.forkpoint, chains)) + children = list(filter(lambda y: y.parent==self, chains)) return max([x.forkpoint for x in children]) if children else None def get_max_forkpoint(self) -> int: @@ -137,11 +184,12 @@ class Blockchain(util.PrintError): mc = self.get_max_child() return mc if mc is not None else self.forkpoint + @with_lock def get_branch_size(self) -> int: return self.height() - self.get_max_forkpoint() + 1 def get_name(self) -> str: - return self.get_hash(self.get_max_forkpoint()).lstrip('00')[0:10] + return self.get_hash(self.get_max_forkpoint()).lstrip('0')[0:10] def check_header(self, header: dict) -> bool: header_hash = hash_header(header) @@ -159,24 +207,38 @@ class Blockchain(util.PrintError): return False def fork(parent, header: dict) -> 'Blockchain': + if not parent.can_connect(header, check_height=False): + raise Exception("forking header does not connect to parent chain") forkpoint = header.get('block_height') - self = Blockchain(parent.config, forkpoint, parent.forkpoint) + self = Blockchain(config=parent.config, + forkpoint=forkpoint, + parent=parent, + forkpoint_hash=hash_header(header), + prev_hash=parent.get_hash(forkpoint-1)) open(self.path(), 'w+').close() self.save_header(header) + # put into global dict + chain_id = self.get_id() + with blockchains_lock: + assert chain_id not in blockchains, (chain_id, list(blockchains)) + blockchains[chain_id] = self return self + @with_lock def height(self) -> int: return self.forkpoint + self.size() - 1 + @with_lock def size(self) -> int: - with self.lock: - return self._size + return self._size + @with_lock def update_size(self) -> None: p = self.path() self._size = os.path.getsize(p)//HEADER_SIZE if os.path.exists(p) else 0 - def verify_header(self, header: dict, prev_hash: str, target: int, expected_header_hash: str=None) -> None: + @classmethod + def verify_header(cls, header: dict, prev_hash: str, target: int, expected_header_hash: str=None) -> None: _hash = hash_header(header) if expected_header_hash and expected_header_hash != _hash: raise Exception("hash mismatches with expected: {} vs {}".format(expected_header_hash, _hash)) @@ -184,7 +246,7 @@ class Blockchain(util.PrintError): raise Exception("prev hash mismatch: %s vs %s" % (prev_hash, header.get('prev_block_hash'))) if constants.net.TESTNET: return - bits = self.target_to_bits(target) + bits = cls.target_to_bits(target) if bits != header.get('bits'): raise Exception("bits mismatch: %s vs %s" % (bits, header.get('bits'))) if int('0x' + _hash, 16) > target: @@ -206,21 +268,26 @@ class Blockchain(util.PrintError): self.verify_header(header, prev_hash, target, expected_header_hash) prev_hash = hash_header(header) + @with_lock def path(self): d = util.get_headers_dir(self.config) - if self.parent_id is None: + if self.parent is None: filename = 'blockchain_headers' else: - basename = 'fork_%d_%d' % (self.parent_id, self.forkpoint) + assert self.forkpoint > 0, self.forkpoint + prev_hash = self._prev_hash.lstrip('0') + first_hash = self._forkpoint_hash.lstrip('0') + basename = f'fork2_{self.forkpoint}_{prev_hash}_{first_hash}' filename = os.path.join('forks', basename) return os.path.join(d, filename) @with_lock def save_chunk(self, index: int, chunk: bytes): + assert index >= 0, index chunk_within_checkpoint_region = index < len(self.checkpoints) # chunks in checkpoint region are the responsibility of the 'main chain' - if chunk_within_checkpoint_region and self.parent_id is not None: - main_chain = blockchains[0] + if chunk_within_checkpoint_region and self.parent is not None: + main_chain = get_best_chain() main_chain.save_chunk(index, chunk) return @@ -235,18 +302,36 @@ class Blockchain(util.PrintError): self.write(chunk, delta_bytes, truncate) self.swap_with_parent() - @with_lock def swap_with_parent(self) -> None: - if self.parent_id is None: - return - parent_branch_size = self.parent().height() - self.forkpoint + 1 - if parent_branch_size >= self.size(): - return - self.print_error("swap", self.forkpoint, self.parent_id) - parent_id = self.parent_id - forkpoint = self.forkpoint - parent = self.parent() + parent_lock = self.parent.lock if self.parent is not None else threading.Lock() + with parent_lock, self.lock, blockchains_lock: # this order should not deadlock + # do the swap; possibly multiple ones + cnt = 0 + while self._swap_with_parent(): + cnt += 1 + if cnt > len(blockchains): # make sure we are making progress + raise Exception(f'swapping fork with parent too many times: {cnt}') + + def _swap_with_parent(self) -> bool: + """Check if this chain became stronger than its parent, and swap + the underlying files if so. The Blockchain instances will keep + 'containing' the same headers, but their ids change and so + they will be stored in different files.""" + if self.parent is None: + return False + parent_branch_size = self.parent.height() - self.forkpoint + 1 + if parent_branch_size >= self.size(): # FIXME most work, not length + return False + self.print_error("swap", self.forkpoint, self.parent.forkpoint) + forkpoint = self.forkpoint # type: Optional[int] + parent = self.parent # type: Optional[Blockchain] + child_old_id = self.get_id() + parent_old_id = parent.get_id() + # swap files + # child takes parent's name + # parent's new name will be something new (not child's old name) self.assert_headers_file_available(self.path()) + child_old_name = self.path() with open(self.path(), 'rb') as f: my_data = f.read() self.assert_headers_file_available(parent.path()) @@ -255,24 +340,28 @@ class Blockchain(util.PrintError): parent_data = f.read(parent_branch_size*HEADER_SIZE) self.write(parent_data, 0) parent.write(my_data, (forkpoint - parent.forkpoint)*HEADER_SIZE) - # store file path - with blockchains_lock: chains = list(blockchains.values()) - for b in chains: - b.old_path = b.path() # swap parameters - self.parent_id = parent.parent_id; parent.parent_id = parent_id - self.forkpoint = parent.forkpoint; parent.forkpoint = forkpoint - self._size = parent._size; parent._size = parent_branch_size - # move files - for b in chains: - if b in [self, parent]: continue - if b.old_path != b.path(): - self.print_error("renaming", b.old_path, b.path()) - os.rename(b.old_path, b.path()) + self.parent, parent.parent = parent.parent, self # type: Optional[Blockchain], Optional[Blockchain] + self.forkpoint, parent.forkpoint = parent.forkpoint, self.forkpoint + self._forkpoint_hash, parent._forkpoint_hash = parent._forkpoint_hash, hash_raw_header(bh2u(parent_data[:HEADER_SIZE])) + self._prev_hash, parent._prev_hash = parent._prev_hash, self._prev_hash + # parent's new name + try: + os.rename(child_old_name, parent.path()) + except OSError: + os.remove(parent.path()) + os.rename(child_old_name, parent.path()) + self.update_size() + parent.update_size() # update pointers - with blockchains_lock: - blockchains[self.forkpoint] = self - blockchains[parent.forkpoint] = parent + blockchains.pop(child_old_id, None) + blockchains.pop(parent_old_id, None) + blockchains[self.get_id()] = self + blockchains[parent.get_id()] = parent + return True + + def get_id(self) -> str: + return self._forkpoint_hash def assert_headers_file_available(self, path): if os.path.exists(path): @@ -282,19 +371,19 @@ class Blockchain(util.PrintError): else: raise FileNotFoundError('Cannot find headers file but headers_dir is there. Should be at {}'.format(path)) + @with_lock def write(self, data: bytes, offset: int, truncate: bool=True) -> None: filename = self.path() - with self.lock: - self.assert_headers_file_available(filename) - with open(filename, 'rb+') as f: - if truncate and offset != self._size * HEADER_SIZE: - f.seek(offset) - f.truncate() + self.assert_headers_file_available(filename) + with open(filename, 'rb+') as f: + if truncate and offset != self._size * HEADER_SIZE: f.seek(offset) - f.write(data) - f.flush() - os.fsync(f.fileno()) - self.update_size() + f.truncate() + f.seek(offset) + f.write(data) + f.flush() + os.fsync(f.fileno()) + self.update_size() @with_lock def save_header(self, header: dict) -> None: @@ -306,12 +395,12 @@ class Blockchain(util.PrintError): self.write(data, delta*HEADER_SIZE) self.swap_with_parent() + @with_lock def read_header(self, height: int) -> Optional[dict]: - assert self.parent_id != self.forkpoint if height < 0: return if height < self.forkpoint: - return self.parent().read_header(height) + return self.parent.read_header(height) if height > self.height(): return delta = height - self.forkpoint @@ -371,16 +460,18 @@ class Blockchain(util.PrintError): new_target = self.bits_to_target(self.target_to_bits(new_target)) return new_target - def bits_to_target(self, bits: int) -> int: + @classmethod + def bits_to_target(cls, bits: int) -> int: bitsN = (bits >> 24) & 0xff - if not (bitsN >= 0x03 and bitsN <= 0x1d): + if not (0x03 <= bitsN <= 0x1d): raise Exception("First part of bits should be in [0x03, 0x1d]") bitsBase = bits & 0xffffff - if not (bitsBase >= 0x8000 and bitsBase <= 0x7fffff): + if not (0x8000 <= bitsBase <= 0x7fffff): raise Exception("Second part of bits should be in [0x8000, 0x7fffff]") return bitsBase << (8 * (bitsN-3)) - def target_to_bits(self, target: int) -> int: + @classmethod + def target_to_bits(cls, target: int) -> int: c = ("%064x" % target)[2:] while c[:2] == '00' and len(c) > 6: c = c[2:] @@ -416,6 +507,7 @@ class Blockchain(util.PrintError): return True def connect_chunk(self, idx: int, hexdata: str) -> bool: + assert idx >= 0, idx try: data = bfh(hexdata) self.verify_chunk(idx, data) @@ -423,7 +515,7 @@ class Blockchain(util.PrintError): self.save_chunk(idx, data) return True except BaseException as e: - self.print_error('verify_chunk %d failed'%idx, str(e)) + self.print_error(f'verify_chunk idx {idx} failed: {repr(e)}') return False def get_checkpoints(self): diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index c34e5ef9f..73379bb22 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -126,10 +126,12 @@ class ElectrumWindow(App): chains = self.network.get_blockchains() def cb(name): with blockchain.blockchains_lock: blockchain_items = list(blockchain.blockchains.items()) - for index, b in blockchain_items: + for chain_id, b in blockchain_items: if name == b.get_name(): - self.network.run_from_another_thread(self.network.follow_chain_given_id(index)) - names = [blockchain.blockchains[b].get_name() for b in chains] + self.network.run_from_another_thread(self.network.follow_chain_given_id(chain_id)) + chain_objects = [blockchain.blockchains.get(chain_id) for chain_id in chains] + chain_objects = filter(lambda b: b is not None, chain_objects) + names = [b.get_name() for b in chain_objects] if len(names) > 1: cur_chain = self.network.blockchain().get_name() ChoiceDialog(_('Choose your chain'), names, cur_chain, cb).open() diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index bef853830..94ae77735 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -82,8 +82,8 @@ class NodesListWidget(QTreeWidget): server = item.data(1, Qt.UserRole) menu.addAction(_("Use as server"), lambda: self.parent.follow_server(server)) else: - index = item.data(1, Qt.UserRole) - menu.addAction(_("Follow this branch"), lambda: self.parent.follow_branch(index)) + chain_id = item.data(1, Qt.UserRole) + menu.addAction(_("Follow this branch"), lambda: self.parent.follow_branch(chain_id)) menu.exec_(self.viewport().mapToGlobal(position)) def keyPressEvent(self, event): @@ -103,22 +103,23 @@ class NodesListWidget(QTreeWidget): self.addChild = self.addTopLevelItem chains = network.get_blockchains() n_chains = len(chains) - for k, items in chains.items(): - b = blockchain.blockchains[k] + for chain_id, interfaces in chains.items(): + b = blockchain.blockchains.get(chain_id) + if b is None: continue name = b.get_name() - if n_chains >1: + if n_chains > 1: x = QTreeWidgetItem([name + '@%d'%b.get_max_forkpoint(), '%d'%b.height()]) x.setData(0, Qt.UserRole, 1) - x.setData(1, Qt.UserRole, b.forkpoint) + x.setData(1, Qt.UserRole, b.get_id()) else: x = self - for i in items: + for i in interfaces: star = ' *' if i == network.interface else '' item = QTreeWidgetItem([i.host + star, '%d'%i.tip]) item.setData(0, Qt.UserRole, 0) item.setData(1, Qt.UserRole, i.server) x.addChild(item) - if n_chains>1: + if n_chains > 1: self.addTopLevelItem(x) x.setExpanded(True) @@ -410,8 +411,8 @@ class NetworkChoiceLayout(object): self.set_protocol(p) self.set_server() - def follow_branch(self, index): - self.network.run_from_another_thread(self.network.follow_chain_given_id(index)) + def follow_branch(self, chain_id): + self.network.run_from_another_thread(self.network.follow_chain_given_id(chain_id)) self.update() def follow_server(self, server): diff --git a/electrum/interface.py b/electrum/interface.py index 68ede7554..99e2349e4 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -28,7 +28,7 @@ import ssl import sys import traceback import asyncio -from typing import Tuple, Union, List, TYPE_CHECKING +from typing import Tuple, Union, List, TYPE_CHECKING, Optional from collections import defaultdict import aiorpcx @@ -140,14 +140,14 @@ def serialize_server(host: str, port: Union[str, int], protocol: str) -> str: class Interface(PrintError): verbosity_filter = 'i' - def __init__(self, network: 'Network', server: str, config_path, proxy: dict): + def __init__(self, network: 'Network', server: str, proxy: Optional[dict]): self.ready = asyncio.Future() self.got_disconnected = asyncio.Future() self.server = server self.host, self.port, self.protocol = deserialize_server(self.server) self.port = int(self.port) - self.config_path = config_path - self.cert_path = os.path.join(self.config_path, 'certs', self.host) + assert network.config.path + self.cert_path = os.path.join(network.config.path, 'certs', self.host) self.blockchain = None self._requested_chunks = set() self.network = network @@ -281,7 +281,7 @@ class Interface(PrintError): assert self.tip_header chain = blockchain.check_header(self.tip_header) if not chain: - self.blockchain = blockchain.blockchains[0] + self.blockchain = blockchain.get_best_chain() else: self.blockchain = chain assert self.blockchain is not None @@ -502,7 +502,7 @@ class Interface(PrintError): # bad_header connects to good_header; bad_header itself is NOT in self.blockchain. bh = self.blockchain.height() - assert bh >= good + assert bh >= good, (bh, good) if bh == good: height = good + 1 self.print_error("catching up from {}".format(height)) @@ -510,53 +510,12 @@ class Interface(PrintError): # this is a new fork we don't yet have height = bad + 1 - branch = blockchain.blockchains.get(bad) - if branch is not None: - # Conflict!! As our fork handling is not completely general, - # we need to delete another fork to save this one. - # Note: This could be a potential DOS vector against Electrum. - # However, mining blocks that satisfy the difficulty requirements - # is assumed to be expensive; especially as forks below the max - # checkpoint are ignored. - self.print_error("new fork at bad height {}. conflict!!".format(bad)) - assert self.blockchain != branch - ismocking = type(branch) is dict - if ismocking: - self.print_error("TODO replace blockchain") - return 'fork_conflict', height - self.print_error('forkpoint conflicts with existing fork', branch.path()) - self._raise_if_fork_conflicts_with_default_server(branch) - await self._disconnect_from_interfaces_on_conflicting_blockchain(branch) - branch.write(b'', 0) - branch.save_header(bad_header) - self.blockchain = branch - return 'fork_conflict', height - else: - # No conflict. Just save the new fork. - self.print_error("new fork at bad height {}. NO conflict.".format(bad)) - forkfun = self.blockchain.fork if 'mock' not in bad_header else bad_header['mock']['fork'] - b = forkfun(bad_header) - with blockchain.blockchains_lock: - assert bad not in blockchain.blockchains, (bad, list(blockchain.blockchains)) - blockchain.blockchains[bad] = b - self.blockchain = b - assert b.forkpoint == bad - return 'fork_noconflict', height - - def _raise_if_fork_conflicts_with_default_server(self, chain_to_delete: Blockchain) -> None: - main_interface = self.network.interface - if not main_interface: return - if main_interface == self: return - chain_of_default_server = main_interface.blockchain - if not chain_of_default_server: return - if chain_to_delete == chain_of_default_server: - raise GracefulDisconnect('refusing to overwrite blockchain of default server') - - async def _disconnect_from_interfaces_on_conflicting_blockchain(self, chain: Blockchain) -> None: - ifaces = await self.network.disconnect_from_interfaces_on_given_blockchain(chain) - if not ifaces: return - servers = [interface.server for interface in ifaces] - self.print_error("forcing disconnect of other interfaces: {}".format(servers)) + self.print_error(f"new fork at bad height {bad}") + forkfun = self.blockchain.fork if 'mock' not in bad_header else bad_header['mock']['fork'] + b = forkfun(bad_header) # type: Blockchain + self.blockchain = b + assert b.forkpoint == bad + return 'fork', height async def _search_headers_backwards(self, height, header): async def iterate(): diff --git a/electrum/network.py b/electrum/network.py index c0218f9cc..85c8dbee1 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -177,10 +177,10 @@ class Network(PrintError): if config is None: config = {} # Do not use mutables as default values! self.config = SimpleConfig(config) if isinstance(config, dict) else config # type: SimpleConfig - blockchain.blockchains = blockchain.read_blockchains(self.config) - self.print_error("blockchains", list(blockchain.blockchains)) + blockchain.read_blockchains(self.config) + self.print_error("blockchains", list(map(lambda b: b.forkpoint, blockchain.blockchains.values()))) self._blockchain_preferred_block = self.config.get('blockchain_preferred_block', None) # type: Optional[Dict] - self._blockchain_index = 0 + self._blockchain = blockchain.get_best_chain() # Server for addresses and transactions self.default_server = self.config.get('server', None) # Sanitize default server @@ -559,17 +559,24 @@ class Network(PrintError): filtered = list(filter(lambda iface: iface.blockchain.check_hash(pref_height, pref_hash), interfaces)) if filtered: + self.print_error("switching to preferred fork") chosen_iface = random.choice(filtered) await self.switch_to_interface(chosen_iface.server) return - # try to switch to longest chain - if self.blockchain().parent_id is None: - return # already on longest chain - filtered = list(filter(lambda iface: iface.blockchain.parent_id is None, + else: + self.print_error("tried to switch to preferred fork but no interfaces are on it") + # try to switch to best chain + if self.blockchain().parent is None: + return # already on best chain + filtered = list(filter(lambda iface: iface.blockchain.parent is None, interfaces)) if filtered: + self.print_error("switching to best chain") chosen_iface = random.choice(filtered) await self.switch_to_interface(chosen_iface.server) + else: + # FIXME switch to best available? + self.print_error("tried to switch to best chain but no interfaces are on it") async def switch_to_interface(self, server: str): """Switch to server as our main interface. If no connection exists, @@ -637,7 +644,7 @@ class Network(PrintError): @ignore_exceptions # do not kill main_taskgroup @log_exceptions async def _run_new_interface(self, server): - interface = Interface(self, server, self.config.path, self.proxy) + interface = Interface(self, server, self.proxy) timeout = 10 if not self.proxy else 20 try: await asyncio.wait_for(interface.ready, timeout) @@ -661,7 +668,7 @@ class Network(PrintError): self.trigger_callback('network_updated') async def _init_headers_file(self): - b = blockchain.blockchains[0] + b = blockchain.get_best_chain() filename = b.path() length = HEADER_SIZE * len(constants.net.CHECKPOINTS) * 2016 if not os.path.exists(filename) or os.path.getsize(filename) < length: @@ -739,8 +746,8 @@ class Network(PrintError): def blockchain(self) -> Blockchain: interface = self.interface if interface and interface.blockchain is not None: - self._blockchain_index = interface.blockchain.forkpoint - return blockchain.blockchains[self._blockchain_index] + self._blockchain = interface.blockchain + return self._blockchain def get_blockchains(self): out = {} # blockchain_id -> list(interfaces) @@ -752,13 +759,6 @@ class Network(PrintError): out[chain_id] = r return out - async def disconnect_from_interfaces_on_given_blockchain(self, chain: Blockchain) -> Sequence[Interface]: - chain_id = chain.forkpoint - ifaces = self.get_blockchains().get(chain_id) or [] - for interface in ifaces: - await self.connection_down(interface.server) - return ifaces - def _set_preferred_chain(self, chain: Blockchain): height = chain.get_max_forkpoint() header_hash = chain.get_hash(height) @@ -768,7 +768,7 @@ class Network(PrintError): } self.config.set_key('blockchain_preferred_block', self._blockchain_preferred_block) - async def follow_chain_given_id(self, chain_id: int) -> None: + async def follow_chain_given_id(self, chain_id: str) -> None: bc = blockchain.blockchains.get(chain_id) if not bc: raise Exception('blockchain {} not found'.format(chain_id)) diff --git a/electrum/storage.py b/electrum/storage.py index 16a4cc90d..ad3de4c6a 100644 --- a/electrum/storage.py +++ b/electrum/storage.py @@ -125,7 +125,7 @@ class JsonDB(PrintError): # perform atomic write on POSIX systems try: os.rename(temp_path, self.path) - except: + except OSError: os.remove(self.path) os.rename(temp_path, self.path) os.chmod(self.path, mode) diff --git a/electrum/tests/test_blockchain.py b/electrum/tests/test_blockchain.py new file mode 100644 index 000000000..be29c1b03 --- /dev/null +++ b/electrum/tests/test_blockchain.py @@ -0,0 +1,239 @@ +import shutil +import tempfile +import os + +from electrum import constants, blockchain +from electrum.simple_config import SimpleConfig +from electrum.blockchain import Blockchain, deserialize_header, hash_header +from electrum.util import bh2u, bfh, make_dir + +from . import SequentialTestCase + + +class TestBlockchain(SequentialTestCase): + + HEADERS = { + 'A': deserialize_header(bfh("0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4adae5494dffff7f2002000000"), 0), + 'B': deserialize_header(bfh("0000002006226e46111a0b59caaf126043eb5bbf28c34f3a5e332a1fc7b2b73cf188910f186c8dfd970a4545f79916bc1d75c9d00432f57c89209bf3bb115b7612848f509c25f45bffff7f2000000000"), 1), + 'C': deserialize_header(bfh("00000020686bdfc6a3db73d5d93e8c9663a720a26ecb1ef20eb05af11b36cdbc57c19f7ebf2cbf153013a1c54abaf70e95198fcef2f3059cc6b4d0f7e876808e7d24d11cc825f45bffff7f2000000000"), 2), + 'D': deserialize_header(bfh("00000020122baa14f3ef54985ae546d1611559e3f487bd2a0f46e8dbb52fbacc9e237972e71019d7feecd9b8596eca9a67032c5f4641b23b5d731dc393e37de7f9c2f299e725f45bffff7f2000000000"), 3), + 'E': deserialize_header(bfh("00000020f8016f7ef3a17d557afe05d4ea7ab6bde1b2247b7643896c1b63d43a1598b747a3586da94c71753f27c075f57f44faf913c31177a0957bbda42e7699e3a2141aed25f45bffff7f2001000000"), 4), + 'F': deserialize_header(bfh("000000201d589c6643c1d121d73b0573e5ee58ab575b8fdf16d507e7e915c5fbfbbfd05e7aee1d692d1615c3bdf52c291032144ce9e3b258a473c17c745047f3431ff8e2ee25f45bffff7f2000000000"), 5), + 'O': deserialize_header(bfh("00000020b833ed46eea01d4c980f59feee44a66aa1162748b6801029565d1466790c405c3a141ce635cbb1cd2b3a4fcdd0a3380517845ba41736c82a79cab535d31128066526f45bffff7f2001000000"), 6), + 'P': deserialize_header(bfh("00000020abe8e119d1877c9dc0dc502d1a253fb9a67967c57732d2f71ee0280e8381ff0a9690c2fe7c1a4450c74dc908fe94dd96c3b0637d51475e9e06a78e944a0c7fe28126f45bffff7f2000000000"), 7), + 'Q': deserialize_header(bfh("000000202ce41d94eb70e1518bc1f72523f84a903f9705d967481e324876e1f8cf4d3452148be228a4c3f2061bafe7efdfc4a8d5a94759464b9b5c619994d45dfcaf49e1a126f45bffff7f2000000000"), 8), + 'R': deserialize_header(bfh("00000020552755b6c59f3d51e361d16281842a4e166007799665b5daed86a063dd89857415681cb2d00ff889193f6a68a93f5096aeb2d84ca0af6185a462555822552221a626f45bffff7f2000000000"), 9), + 'S': deserialize_header(bfh("00000020a13a491cbefc93cd1bb1938f19957e22a134faf14c7dee951c45533e2c750f239dc087fc977b06c24a69c682d1afd1020e6dc1f087571ccec66310a786e1548fab26f45bffff7f2000000000"), 10), + 'T': deserialize_header(bfh("00000020dbf3a9b55dfefbaf8b6e43a89cf833fa2e208bbc0c1c5d76c0d71b9e4a65337803b243756c25053253aeda309604363460a3911015929e68705bd89dff6fe064b026f45bffff7f2002000000"), 11), + 'U': deserialize_header(bfh("000000203d0932b3b0c78eccb39a595a28ae4a7c966388648d7783fd1305ec8d40d4fe5fd67cb902a7d807cee7676cb543feec3e053aa824d5dfb528d5b94f9760313d9db726f45bffff7f2001000000"), 12), + 'G': deserialize_header(bfh("00000020b833ed46eea01d4c980f59feee44a66aa1162748b6801029565d1466790c405c3a141ce635cbb1cd2b3a4fcdd0a3380517845ba41736c82a79cab535d31128066928f45bffff7f2001000000"), 6), + 'H': deserialize_header(bfh("00000020e19e687f6e7f83ca394c114144dbbbc4f3f9c9450f66331a125413702a2e1a719690c2fe7c1a4450c74dc908fe94dd96c3b0637d51475e9e06a78e944a0c7fe26a28f45bffff7f2002000000"), 7), + 'I': deserialize_header(bfh("0000002009dcb3b158293c89d7cf7ceeb513add122ebc3880a850f47afbb2747f5e48c54148be228a4c3f2061bafe7efdfc4a8d5a94759464b9b5c619994d45dfcaf49e16a28f45bffff7f2000000000"), 8), + 'J': deserialize_header(bfh("000000206a65f3bdd3374a5a6c4538008ba0b0a560b8566291f9ef4280ab877627a1742815681cb2d00ff889193f6a68a93f5096aeb2d84ca0af6185a462555822552221c928f45bffff7f2000000000"), 9), + 'K': deserialize_header(bfh("00000020bb3b421653548991998f96f8ba486b652fdb07ca16e9cee30ece033547cd1a6e9dc087fc977b06c24a69c682d1afd1020e6dc1f087571ccec66310a786e1548fca28f45bffff7f2000000000"), 10), + 'L': deserialize_header(bfh("00000020c391d74d37c24a130f4bf4737932bdf9e206dd4fad22860ec5408978eb55d46303b243756c25053253aeda309604363460a3911015929e68705bd89dff6fe064ca28f45bffff7f2000000000"), 11), + 'M': deserialize_header(bfh("000000206a65f3bdd3374a5a6c4538008ba0b0a560b8566291f9ef4280ab877627a1742815681cb2d00ff889193f6a68a93f5096aeb2d84ca0af6185a4625558225522214229f45bffff7f2000000000"), 9), + 'N': deserialize_header(bfh("00000020383dab38b57f98aa9b4f0d5ff868bc674b4828d76766bf048296f4c45fff680a9dc087fc977b06c24a69c682d1afd1020e6dc1f087571ccec66310a786e1548f4329f45bffff7f2003000000"), 10), + 'X': deserialize_header(bfh("0000002067f1857f54b7fef732cb4940f7d1b339472b3514660711a820330fd09d8fba6b03b243756c25053253aeda309604363460a3911015929e68705bd89dff6fe0649b29f45bffff7f2002000000"), 11), + 'Y': deserialize_header(bfh("00000020db33c9768a9e5f7c37d0f09aad88d48165946c87d08f7d63793f07b5c08c527fd67cb902a7d807cee7676cb543feec3e053aa824d5dfb528d5b94f9760313d9d9b29f45bffff7f2000000000"), 12), + 'Z': deserialize_header(bfh("0000002047822b67940e337fda38be6f13390b3596e4dea2549250256879722073824e7f0f2596c29203f8a0f71ae94193092dc8f113be3dbee4579f1e649fa3d6dcc38c622ef45bffff7f2003000000"), 13), + } + # tree of headers: + # - M <- N <- X <- Y <- Z + # / + # - G <- H <- I <- J <- K <- L + # / + # A <- B <- C <- D <- E <- F <- O <- P <- Q <- R <- S <- T <- U + + @classmethod + def setUpClass(cls): + super().setUpClass() + constants.set_regtest() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + constants.set_mainnet() + + def setUp(self): + super().setUp() + self.data_dir = tempfile.mkdtemp() + make_dir(os.path.join(self.data_dir, 'forks')) + self.config = SimpleConfig({'electrum_path': self.data_dir}) + blockchain.blockchains = {} + + def tearDown(self): + super().tearDown() + shutil.rmtree(self.data_dir) + + def _append_header(self, chain: Blockchain, header: dict): + self.assertTrue(chain.can_connect(header)) + chain.save_header(header) + + def test_forking_and_swapping(self): + blockchain.blockchains[constants.net.GENESIS] = chain_u = Blockchain( + config=self.config, forkpoint=0, parent=None, + forkpoint_hash=constants.net.GENESIS, prev_hash=None) + open(chain_u.path(), 'w+').close() + + self._append_header(chain_u, self.HEADERS['A']) + self._append_header(chain_u, self.HEADERS['B']) + self._append_header(chain_u, self.HEADERS['C']) + self._append_header(chain_u, self.HEADERS['D']) + self._append_header(chain_u, self.HEADERS['E']) + self._append_header(chain_u, self.HEADERS['F']) + self._append_header(chain_u, self.HEADERS['O']) + self._append_header(chain_u, self.HEADERS['P']) + self._append_header(chain_u, self.HEADERS['Q']) + self._append_header(chain_u, self.HEADERS['R']) + + chain_l = chain_u.fork(self.HEADERS['G']) + self._append_header(chain_l, self.HEADERS['H']) + self._append_header(chain_l, self.HEADERS['I']) + self._append_header(chain_l, self.HEADERS['J']) + + # do checks + self.assertEqual(2, len(blockchain.blockchains)) + self.assertEqual(1, len(os.listdir(os.path.join(self.data_dir, "forks")))) + self.assertEqual(0, chain_u.forkpoint) + self.assertEqual(None, chain_u.parent) + self.assertEqual(constants.net.GENESIS, chain_u._forkpoint_hash) + self.assertEqual(None, chain_u._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "blockchain_headers"), chain_u.path()) + self.assertEqual(10 * 80, os.stat(chain_u.path()).st_size) + self.assertEqual(6, chain_l.forkpoint) + self.assertEqual(chain_u, chain_l.parent) + self.assertEqual(hash_header(self.HEADERS['G']), chain_l._forkpoint_hash) + self.assertEqual(hash_header(self.HEADERS['F']), chain_l._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "forks", "fork2_6_5c400c7966145d56291080b6482716a16aa644eefe590f984c1da0ee46ed33b8_711a2e2a701354121a33660f45c9f9f3c4bbdb4441114c39ca837f6e7f689ee1"), chain_l.path()) + self.assertEqual(4 * 80, os.stat(chain_l.path()).st_size) + + self._append_header(chain_l, self.HEADERS['K']) + + # chains were swapped, do checks + self.assertEqual(2, len(blockchain.blockchains)) + self.assertEqual(1, len(os.listdir(os.path.join(self.data_dir, "forks")))) + self.assertEqual(6, chain_u.forkpoint) + self.assertEqual(chain_l, chain_u.parent) + self.assertEqual(hash_header(self.HEADERS['O']), chain_u._forkpoint_hash) + self.assertEqual(hash_header(self.HEADERS['F']), chain_u._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "forks", "fork2_6_5c400c7966145d56291080b6482716a16aa644eefe590f984c1da0ee46ed33b8_aff81830e28e01ef7d23277c56779a6b93f251a2d50dcc09d7c87d119e1e8ab"), chain_u.path()) + self.assertEqual(4 * 80, os.stat(chain_u.path()).st_size) + self.assertEqual(0, chain_l.forkpoint) + self.assertEqual(None, chain_l.parent) + self.assertEqual(constants.net.GENESIS, chain_l._forkpoint_hash) + self.assertEqual(None, chain_l._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "blockchain_headers"), chain_l.path()) + self.assertEqual(11 * 80, os.stat(chain_l.path()).st_size) + for b in (chain_u, chain_l): + self.assertTrue(all([b.can_connect(b.read_header(i), False) for i in range(b.height())])) + + self._append_header(chain_u, self.HEADERS['S']) + self._append_header(chain_u, self.HEADERS['T']) + self._append_header(chain_u, self.HEADERS['U']) + self._append_header(chain_l, self.HEADERS['L']) + + chain_z = chain_l.fork(self.HEADERS['M']) + self._append_header(chain_z, self.HEADERS['N']) + self._append_header(chain_z, self.HEADERS['X']) + self._append_header(chain_z, self.HEADERS['Y']) + self._append_header(chain_z, self.HEADERS['Z']) + + # chain_z became best chain, do checks + self.assertEqual(3, len(blockchain.blockchains)) + self.assertEqual(2, len(os.listdir(os.path.join(self.data_dir, "forks")))) + self.assertEqual(0, chain_z.forkpoint) + self.assertEqual(None, chain_z.parent) + self.assertEqual(constants.net.GENESIS, chain_z._forkpoint_hash) + self.assertEqual(None, chain_z._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "blockchain_headers"), chain_z.path()) + self.assertEqual(14 * 80, os.stat(chain_z.path()).st_size) + self.assertEqual(9, chain_l.forkpoint) + self.assertEqual(chain_z, chain_l.parent) + self.assertEqual(hash_header(self.HEADERS['J']), chain_l._forkpoint_hash) + self.assertEqual(hash_header(self.HEADERS['I']), chain_l._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "forks", "fork2_9_2874a1277687ab8042eff9916256b860a5b0a08b0038456c5a4a37d3bdf3656a_6e1acd473503ce0ee3cee916ca07db2f656b48baf8968f999189545316423bbb"), chain_l.path()) + self.assertEqual(3 * 80, os.stat(chain_l.path()).st_size) + self.assertEqual(6, chain_u.forkpoint) + self.assertEqual(chain_z, chain_u.parent) + self.assertEqual(hash_header(self.HEADERS['O']), chain_u._forkpoint_hash) + self.assertEqual(hash_header(self.HEADERS['F']), chain_u._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "forks", "fork2_6_5c400c7966145d56291080b6482716a16aa644eefe590f984c1da0ee46ed33b8_aff81830e28e01ef7d23277c56779a6b93f251a2d50dcc09d7c87d119e1e8ab"), chain_u.path()) + self.assertEqual(7 * 80, os.stat(chain_u.path()).st_size) + for b in (chain_u, chain_l, chain_z): + self.assertTrue(all([b.can_connect(b.read_header(i), False) for i in range(b.height())])) + + self.assertEqual(constants.net.GENESIS, chain_z.get_hash(0)) + self.assertEqual(hash_header(self.HEADERS['F']), chain_z.get_hash(5)) + self.assertEqual(hash_header(self.HEADERS['G']), chain_z.get_hash(6)) + self.assertEqual(hash_header(self.HEADERS['I']), chain_z.get_hash(8)) + self.assertEqual(hash_header(self.HEADERS['M']), chain_z.get_hash(9)) + self.assertEqual(hash_header(self.HEADERS['Z']), chain_z.get_hash(13)) + + def test_doing_multiple_swaps_after_single_new_header(self): + blockchain.blockchains[constants.net.GENESIS] = chain_u = Blockchain( + config=self.config, forkpoint=0, parent=None, + forkpoint_hash=constants.net.GENESIS, prev_hash=None) + open(chain_u.path(), 'w+').close() + + self._append_header(chain_u, self.HEADERS['A']) + self._append_header(chain_u, self.HEADERS['B']) + self._append_header(chain_u, self.HEADERS['C']) + self._append_header(chain_u, self.HEADERS['D']) + self._append_header(chain_u, self.HEADERS['E']) + self._append_header(chain_u, self.HEADERS['F']) + self._append_header(chain_u, self.HEADERS['O']) + self._append_header(chain_u, self.HEADERS['P']) + self._append_header(chain_u, self.HEADERS['Q']) + self._append_header(chain_u, self.HEADERS['R']) + self._append_header(chain_u, self.HEADERS['S']) + + self.assertEqual(1, len(blockchain.blockchains)) + self.assertEqual(0, len(os.listdir(os.path.join(self.data_dir, "forks")))) + + chain_l = chain_u.fork(self.HEADERS['G']) + self._append_header(chain_l, self.HEADERS['H']) + self._append_header(chain_l, self.HEADERS['I']) + self._append_header(chain_l, self.HEADERS['J']) + self._append_header(chain_l, self.HEADERS['K']) + # now chain_u is best chain, but it's tied with chain_l + + self.assertEqual(2, len(blockchain.blockchains)) + self.assertEqual(1, len(os.listdir(os.path.join(self.data_dir, "forks")))) + + chain_z = chain_l.fork(self.HEADERS['M']) + self._append_header(chain_z, self.HEADERS['N']) + self._append_header(chain_z, self.HEADERS['X']) + + self.assertEqual(3, len(blockchain.blockchains)) + self.assertEqual(2, len(os.listdir(os.path.join(self.data_dir, "forks")))) + + # chain_z became best chain, do checks + self.assertEqual(0, chain_z.forkpoint) + self.assertEqual(None, chain_z.parent) + self.assertEqual(constants.net.GENESIS, chain_z._forkpoint_hash) + self.assertEqual(None, chain_z._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "blockchain_headers"), chain_z.path()) + self.assertEqual(12 * 80, os.stat(chain_z.path()).st_size) + self.assertEqual(9, chain_l.forkpoint) + self.assertEqual(chain_z, chain_l.parent) + self.assertEqual(hash_header(self.HEADERS['J']), chain_l._forkpoint_hash) + self.assertEqual(hash_header(self.HEADERS['I']), chain_l._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "forks", "fork2_9_2874a1277687ab8042eff9916256b860a5b0a08b0038456c5a4a37d3bdf3656a_6e1acd473503ce0ee3cee916ca07db2f656b48baf8968f999189545316423bbb"), chain_l.path()) + self.assertEqual(2 * 80, os.stat(chain_l.path()).st_size) + self.assertEqual(6, chain_u.forkpoint) + self.assertEqual(chain_z, chain_u.parent) + self.assertEqual(hash_header(self.HEADERS['O']), chain_u._forkpoint_hash) + self.assertEqual(hash_header(self.HEADERS['F']), chain_u._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "forks", "fork2_6_5c400c7966145d56291080b6482716a16aa644eefe590f984c1da0ee46ed33b8_aff81830e28e01ef7d23277c56779a6b93f251a2d50dcc09d7c87d119e1e8ab"), chain_u.path()) + self.assertEqual(5 * 80, os.stat(chain_u.path()).st_size) + + self.assertEqual(constants.net.GENESIS, chain_z.get_hash(0)) + self.assertEqual(hash_header(self.HEADERS['F']), chain_z.get_hash(5)) + self.assertEqual(hash_header(self.HEADERS['G']), chain_z.get_hash(6)) + self.assertEqual(hash_header(self.HEADERS['I']), chain_z.get_hash(8)) + self.assertEqual(hash_header(self.HEADERS['M']), chain_z.get_hash(9)) + self.assertEqual(hash_header(self.HEADERS['X']), chain_z.get_hash(11)) + + for b in (chain_u, chain_l, chain_z): + self.assertTrue(all([b.can_connect(b.read_header(i), False) for i in range(b.height())])) diff --git a/electrum/tests/test_network.py b/electrum/tests/test_network.py index c69375bd6..ece54056f 100644 --- a/electrum/tests/test_network.py +++ b/electrum/tests/test_network.py @@ -6,6 +6,9 @@ from electrum import constants from electrum.simple_config import SimpleConfig from electrum import blockchain from electrum.interface import Interface +from electrum.crypto import sha256 +from electrum.util import bh2u + class MockTaskGroup: async def spawn(self, x): return @@ -17,10 +20,14 @@ class MockNetwork: class MockInterface(Interface): def __init__(self, config): self.config = config - super().__init__(MockNetwork(), 'mock-server:50000:t', self.config.electrum_path(), None) + network = MockNetwork() + network.config = config + super().__init__(network, 'mock-server:50000:t', None) self.q = asyncio.Queue() - self.blockchain = blockchain.Blockchain(self.config, 2002, None) + self.blockchain = blockchain.Blockchain(config=self.config, forkpoint=0, + parent=None, forkpoint_hash=constants.net.GENESIS, prev_hash=None) self.tip = 12 + self.blockchain._size = self.tip + 1 async def get_block_header(self, height, assert_mode): assert self.q.qsize() > 0, (height, assert_mode) item = await self.q.get() @@ -56,7 +63,7 @@ class TestNetwork(unittest.TestCase): self.interface.q.put_nowait({'block_height': 5, 'mock': {'binary':1,'check':lambda x: True, 'connect': lambda x: True}}) self.interface.q.put_nowait({'block_height': 6, 'mock': {'binary':1,'check':lambda x: True, 'connect': lambda x: True}}) ifa = self.interface - self.assertEqual(('fork_noconflict', 8), asyncio.get_event_loop().run_until_complete(ifa.sync_until(8, next_height=7))) + self.assertEqual(('fork', 8), asyncio.get_event_loop().run_until_complete(ifa.sync_until(8, next_height=7))) self.assertEqual(self.interface.q.qsize(), 0) def test_fork_conflict(self): @@ -70,7 +77,7 @@ class TestNetwork(unittest.TestCase): self.interface.q.put_nowait({'block_height': 5, 'mock': {'binary':1,'check':lambda x: True, 'connect': lambda x: True}}) self.interface.q.put_nowait({'block_height': 6, 'mock': {'binary':1,'check':lambda x: True, 'connect': lambda x: True}}) ifa = self.interface - self.assertEqual(('fork_conflict', 8), asyncio.get_event_loop().run_until_complete(ifa.sync_until(8, next_height=7))) + self.assertEqual(('fork', 8), asyncio.get_event_loop().run_until_complete(ifa.sync_until(8, next_height=7))) self.assertEqual(self.interface.q.qsize(), 0) def test_can_connect_during_backward(self): @@ -87,7 +94,10 @@ class TestNetwork(unittest.TestCase): self.assertEqual(self.interface.q.qsize(), 0) def mock_fork(self, bad_header): - return blockchain.Blockchain(self.config, bad_header['block_height'], None) + forkpoint = bad_header['block_height'] + b = blockchain.Blockchain(config=self.config, forkpoint=forkpoint, parent=None, + forkpoint_hash=bh2u(sha256(str(forkpoint))), prev_hash=bh2u(sha256(str(forkpoint-1)))) + return b def test_chain_false_during_binary(self): blockchain.blockchains = {} diff --git a/electrum/tests/test_wallet.py b/electrum/tests/test_wallet.py index 9117392ea..c6366f3e3 100644 --- a/electrum/tests/test_wallet.py +++ b/electrum/tests/test_wallet.py @@ -64,7 +64,6 @@ class TestWalletStorage(WalletTestCase): storage.put(key, value) storage.write() - contents = "" with open(self.wallet_path, "r") as f: contents = f.read() self.assertEqual(some_dict, json.loads(contents)) From 65ce3deeaa33828407cf3a873c7ce5c48fa0b6d4 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 22 Nov 2018 17:13:43 +0100 Subject: [PATCH 05/40] blockchain: chain hierarchy based on most work, not length --- electrum/blockchain.py | 43 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/electrum/blockchain.py b/electrum/blockchain.py index c7c075312..d1238a2e8 100644 --- a/electrum/blockchain.py +++ b/electrum/blockchain.py @@ -141,6 +141,11 @@ def read_blockchains(config: 'SimpleConfig'): def get_best_chain() -> 'Blockchain': return blockchains[constants.net.GENESIS] +# block hash -> chain work; up to and including that block +_CHAINWORK_CACHE = { + "0000000000000000000000000000000000000000000000000000000000000000": 0, # virtual block at height -1 +} # type: Dict[str, int] + class Blockchain(util.PrintError): """ @@ -319,10 +324,10 @@ class Blockchain(util.PrintError): they will be stored in different files.""" if self.parent is None: return False - parent_branch_size = self.parent.height() - self.forkpoint + 1 - if parent_branch_size >= self.size(): # FIXME most work, not length + if self.parent.get_chainwork() >= self.get_chainwork(): return False self.print_error("swap", self.forkpoint, self.parent.forkpoint) + parent_branch_size = self.parent.height() - self.forkpoint + 1 forkpoint = self.forkpoint # type: Optional[int] parent = self.parent # type: Optional[Blockchain] child_old_id = self.get_id() @@ -481,6 +486,40 @@ class Blockchain(util.PrintError): bitsBase >>= 8 return bitsN << 24 | bitsBase + def chainwork_of_header_at_height(self, height: int) -> int: + """work done by single header at given height""" + chunk_idx = height // 2016 - 1 + target = self.get_target(chunk_idx) + work = ((2 ** 256 - target - 1) // (target + 1)) + 1 + return work + + @with_lock + def get_chainwork(self, height=None) -> int: + if height is None: + height = max(0, self.height()) + if constants.net.TESTNET: + # On testnet/regtest, difficulty works somewhat different. + # It's out of scope to properly implement that. + return height + last_retarget = height // 2016 * 2016 - 1 + cached_height = last_retarget + while _CHAINWORK_CACHE.get(self.get_hash(cached_height)) is None: + if cached_height <= -1: + break + cached_height -= 2016 + assert cached_height >= -1, cached_height + running_total = _CHAINWORK_CACHE[self.get_hash(cached_height)] + while cached_height < last_retarget: + cached_height += 2016 + work_in_single_header = self.chainwork_of_header_at_height(cached_height) + work_in_chunk = 2016 * work_in_single_header + running_total += work_in_chunk + _CHAINWORK_CACHE[self.get_hash(cached_height)] = running_total + cached_height += 2016 + work_in_single_header = self.chainwork_of_header_at_height(cached_height) + work_in_last_partial_chunk = (height % 2016 + 1) * work_in_single_header + return running_total + work_in_last_partial_chunk + def can_connect(self, header: dict, check_height: bool=True) -> bool: if header is None: return False From d296a1be65d2553bce9faaf448c3f3b8ff459bbf Mon Sep 17 00:00:00 2001 From: Calin Culianu Date: Mon, 26 Nov 2018 13:36:51 +0200 Subject: [PATCH 06/40] [macOS] Added optional code signing capability to the OSX build scripts. --- contrib/build-osx/base.sh | 23 +++++++++++++++++++++++ contrib/build-osx/make_osx | 29 +++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/contrib/build-osx/base.sh b/contrib/build-osx/base.sh index c5a5c0d69..e7454f7a8 100644 --- a/contrib/build-osx/base.sh +++ b/contrib/build-osx/base.sh @@ -2,6 +2,7 @@ RED='\033[0;31m' BLUE='\033[0,34m' +YELLOW='\033[0;33m' NC='\033[0m' # No Color function info { printf "\r💬 ${BLUE}INFO:${NC} ${1}\n" @@ -10,3 +11,25 @@ function fail { printf "\r🗯 ${RED}ERROR:${NC} ${1}\n" exit 1 } +function warn { + printf "\r⚠️ ${YELLOW}WARNING:${NC} ${1}\n" +} + +function DoCodeSignMaybe { # ARGS: infoName fileOrDirName codesignIdentity + infoName="$1" + file="$2" + identity="$3" + deep="" + if [ -z "$identity" ]; then + # we are ok with them not passing anything -- master script calls us always even if no identity is specified + return + fi + if [ -d "$file" ]; then + deep="--deep" + fi + if [ -z "$infoName" ] || [ -z "$file" ] || [ -z "$identity" ] || [ ! -e "$file" ]; then + fail "Argument error to internal function DoCodeSignMaybe()" + fi + info "Code signing ${infoName}..." + codesign -f -v $deep -s "$identity" "$file" || fail "Could not code sign ${infoName}" +} diff --git a/contrib/build-osx/make_osx b/contrib/build-osx/make_osx index 599480e23..ecccbef38 100755 --- a/contrib/build-osx/make_osx +++ b/contrib/build-osx/make_osx @@ -17,6 +17,24 @@ VERSION=`git describe --tags --dirty --always` which brew > /dev/null 2>&1 || fail "Please install brew from https://brew.sh/ to continue" +# Code Signing: See https://developer.apple.com/library/archive/documentation/Security/Conceptual/CodeSigningGuide/Procedures/Procedures.html +APP_SIGN="" +if [ -n "$1" ]; then + # Test the identity is valid for signing by doing this hack. There is no other way to do this. + cp -f /bin/ls ./CODESIGN_TEST + codesign -s "$1" --dryrun -f ./CODESIGN_TEST > /dev/null 2>&1 + res=$? + rm -f ./CODESIGN_TEST + if ((res)); then + fail "Code signing identity \"$1\" appears to be invalid." + fi + unset res + APP_SIGN="$1" + info "Code signing enabled using identity \"$APP_SIGN\"" +else + warn "Code signing DISABLED. Specify a valid macOS Developer identity installed on the system as the first argument to this script to enable signing." +fi + info "Installing Python $PYTHON_VERSION" export PATH="~/.pyenv/bin:~/.pyenv/shims:~/Library/Python/3.6/bin:$PATH" if [ -d "~/.pyenv" ]; then @@ -54,6 +72,7 @@ info "Downloading libusb..." curl https://homebrew.bintray.com/bottles/libusb-1.0.22.el_capitan.bottle.tar.gz | \ tar xz --directory $BUILDDIR cp $BUILDDIR/libusb/1.0.22/lib/libusb-1.0.dylib contrib/build-osx +DoCodeSignMaybe "libusb" "contrib/build-osx/libusb-1.0.dylib" "$APP_SIGN" # If APP_SIGN is empty will be a noop info "Building libsecp256k1" brew install autoconf automake libtool @@ -66,6 +85,7 @@ git clean -f -x -q make popd cp $BUILDDIR/secp256k1/.libs/libsecp256k1.0.dylib contrib/build-osx +DoCodeSignMaybe "libsecp256k1" "contrib/build-osx/libsecp256k1.0.dylib" "$APP_SIGN" # If APP_SIGN is empty will be a noop info "Installing requirements..." @@ -96,5 +116,14 @@ plutil -insert 'CFBundleURLTypes' \ -- dist/$PACKAGE.app/Contents/Info.plist \ || fail "Could not add keys to Info.plist. Make sure the program 'plutil' exists and is installed." +DoCodeSignMaybe "app bundle" "dist/${PACKAGE}.app" "$APP_SIGN" # If APP_SIGN is empty will be a noop + info "Creating .DMG" hdiutil create -fs HFS+ -volname $PACKAGE -srcfolder dist/$PACKAGE.app dist/electrum-$VERSION.dmg || fail "Could not create .DMG" + +DoCodeSignMaybe ".DMG" "dist/electrum-${VERSION}.dmg" "$APP_SIGN" # If APP_SIGN is empty will be a noop + +if [ -z "$APP_SIGN" ]; then + warn "App was built successfully but was not code signed. Users may get security warnings from macOS." + warn "Specify a valid code signing identity as the first argument to this script to enable code signing." +fi From a34d42492def4f173094afe54e142941bfe3ddba Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 26 Nov 2018 03:47:41 +0100 Subject: [PATCH 07/40] android docker build --- electrum/gui/kivy/Makefile | 2 +- electrum/gui/kivy/Readme.md | 165 ++++++------------------- electrum/gui/kivy/tools/Dockerfile | 142 +++++++++++++++++++++ electrum/gui/kivy/tools/build.sh | 9 ++ electrum/gui/kivy/tools/buildozer.spec | 4 +- 5 files changed, 191 insertions(+), 131 deletions(-) create mode 100644 electrum/gui/kivy/tools/Dockerfile create mode 100755 electrum/gui/kivy/tools/build.sh diff --git a/electrum/gui/kivy/Makefile b/electrum/gui/kivy/Makefile index 7e87afa6b..868a774a0 100644 --- a/electrum/gui/kivy/Makefile +++ b/electrum/gui/kivy/Makefile @@ -11,7 +11,7 @@ prepare: @cp tools/buildozer.spec ../../../buildozer.spec # copy electrum to main.py @cp ../../../run_electrum ../../../main.py - @-if [ ! -d "../../.buildozer" ];then \ + @-if [ ! -d "../../../.buildozer" ];then \ cd ../../..; buildozer android debug;\ cp -f electrum/gui/kivy/tools/blacklist.txt .buildozer/android/platform/python-for-android/src/blacklist.txt;\ rm -rf ./.buildozer/android/platform/python-for-android/dist;\ diff --git a/electrum/gui/kivy/Readme.md b/electrum/gui/kivy/Readme.md index cbafb938f..83bc01a36 100644 --- a/electrum/gui/kivy/Readme.md +++ b/electrum/gui/kivy/Readme.md @@ -3,147 +3,56 @@ The Kivy GUI is used with Electrum on Android devices. To generate an APK file, follow these instructions. -Recommended env: Ubuntu 18.04 +## Android binary with Docker -## 1. Preliminaries +This assumes an Ubuntu host, but it should not be too hard to adapt to another +similar system. The docker commands should be executed in the project's root +folder. -Make sure the current user can write `/opt` (e.g. `sudo chown username: /opt`). +1. Install Docker -We assume that you already got Electrum to run from source on this machine, -hence have e.g. `git`, `python3-pip` and `python3-setuptools`. + ``` + $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + $ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + $ sudo apt-get update + $ sudo apt-get install -y docker-ce + ``` -## 2. Install kivy - -Install kivy for python3 as described [here](https://kivy.org/docs/installation/installation-linux.html). -So for example: -```sh -sudo add-apt-repository ppa:kivy-team/kivy -sudo apt-get install python3-kivy -``` - - -## 3. Install python-for-android (p4a) -p4a is used to package Electrum, Python, SDL and a bootstrap Java app into an APK file. -We need some functionality not in p4a master, so for the time being we have our own fork. - -Something like this should work: - -```sh -cd /opt -git clone https://github.com/kivy/python-for-android -cd python-for-android -git remote add sombernight https://github.com/SomberNight/python-for-android -git fetch --all -git checkout f74226666af69f9915afaee9ef9292db85a6c617 -``` - -## 4. Install buildozer -4.1 Buildozer is a frontend to p4a. Luckily we don't need to patch it: - -```sh -cd /opt -git clone https://github.com/kivy/buildozer -cd buildozer -sudo python3 setup.py install -``` - -4.2 Install additional dependencies: - -```sh -sudo apt-get install python-pip -``` - -(from [buildozer docs](https://buildozer.readthedocs.io/en/latest/installation.html#targeting-android)) -```sh -sudo pip install --upgrade cython==0.21 -sudo dpkg --add-architecture i386 -sudo apt-get update -sudo apt-get install build-essential ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 -``` - -4.3 Download Android NDK -```sh -cd /opt -wget https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip -unzip android-ndk-r14b-linux-x86_64.zip -``` - -## 5. Some more dependencies - -```sh -python3 -m pip install colorama appdirs sh jinja2 cython==0.29 -sudo apt-get install autotools-dev autoconf libtool pkg-config python3.7 -``` +2. Build image + ``` + $ sudo docker build -t electrum-android-builder-img electrum/gui/kivy/tools + ``` -## 6. Create the UI Atlas -In the `electrum/gui/kivy` directory of Electrum, run `make theming`. +3. Build binaries -## 7. Download Electrum dependencies -```sh -sudo contrib/make_packages -``` + ``` + $ sudo docker run \ + --name electrum-android-builder-cont \ + --rm \ + -v $PWD:/home/user/wspace/electrum \ + --workdir /home/user/wspace/electrum \ + electrum-android-builder-img \ + ./electrum/gui/kivy/tools/build.sh + ``` + This mounts the project dir inside the container, + and so the modifications will affect it, e.g. `.buildozer` folder + will be created. -## 8. Try building the APK and fail +4. The generated binary is in `./bin`. -### 1. Try and fail: -```sh -contrib/make_apk -``` -Symlink android tools: +## FAQ -```sh -ln -sf ~/.buildozer/android/platform/android-sdk-24/tools ~/.buildozer/android/platform/android-sdk-24/tools.save -``` +### I changed something but I don't see any differences on the phone. What did I do wrong? +You probably need to clear the cache: `rm -rf .buildozer/android/platform/build/{build,dists}` -### 2. Try and fail: -```sh -contrib/make_apk +### How do I get an interactive shell inside docker? ``` - -During this build attempt, buildozer downloaded some tools, -e.g. those needed in the next step. - -## 9. Update the Android SDK build tools - -### Method 1: Using the GUI - - Start the Android SDK manager in GUI mode: - - ~/.buildozer/android/platform/android-sdk-24/tools/android - - Check the latest SDK available and install it - ("Android SDK Tools" and "Android SDK Platform-tools"). - Close the SDK manager. Repeat until there is no newer version. - - Reopen the SDK manager, and install the latest build tools - ("Android SDK Build-tools"), 28.0.3 at the time of writing. - - Install "Android 9">"SDK Platform". - Install "Android Support Repository" from the SDK manager (under "Extras"). - -### Method 2: Using the command line: - - Repeat the following command until there is nothing to install: - - ~/.buildozer/android/platform/android-sdk-24/tools/android update sdk -u -t tools,platform-tools - - Install Build Tools, android API 19 and Android Support Library: - - ~/.buildozer/android/platform/android-sdk-24/tools/android update sdk -u -t build-tools-28.0.3,android-28,extra-android-m2repository - - (FIXME: build-tools is not getting installed?! use GUI for now.) - -## 10. Build the APK - -```sh -contrib/make_apk +$ sudo docker run -it --rm \ + -v $PWD:/home/user/wspace/electrum \ + --workdir /home/user/wspace/electrum \ + electrum-android-builder-img ``` - -# FAQ - -## I changed something but I don't see any differences on the phone. What did I do wrong? -You probably need to clear the cache: `rm -rf .buildozer/android/platform/build/{build,dists}` diff --git a/electrum/gui/kivy/tools/Dockerfile b/electrum/gui/kivy/tools/Dockerfile new file mode 100644 index 000000000..0300d8620 --- /dev/null +++ b/electrum/gui/kivy/tools/Dockerfile @@ -0,0 +1,142 @@ +# based on https://github.com/kivy/python-for-android/blob/master/Dockerfile + +FROM ubuntu:18.04 + +ENV ANDROID_HOME="/opt/android" + +RUN apt -y update -qq \ + && apt -y install -qq --no-install-recommends curl unzip git python3-pip python3-setuptools \ + && apt -y autoremove \ + && apt -y clean + + +ENV ANDROID_NDK_HOME="${ANDROID_HOME}/android-ndk" +ENV ANDROID_NDK_VERSION="14b" +ENV ANDROID_NDK_HOME_V="${ANDROID_NDK_HOME}-r${ANDROID_NDK_VERSION}" + +# get the latest version from https://developer.android.com/ndk/downloads/index.html +ENV ANDROID_NDK_ARCHIVE="android-ndk-r${ANDROID_NDK_VERSION}-linux-x86_64.zip" +ENV ANDROID_NDK_DL_URL="https://dl.google.com/android/repository/${ANDROID_NDK_ARCHIVE}" + +# download and install Android NDK +RUN curl --location --progress-bar \ + "${ANDROID_NDK_DL_URL}" \ + --output "${ANDROID_NDK_ARCHIVE}" \ + && mkdir --parents "${ANDROID_NDK_HOME_V}" \ + && unzip -q "${ANDROID_NDK_ARCHIVE}" -d "${ANDROID_HOME}" \ + && ln -sfn "${ANDROID_NDK_HOME_V}" "${ANDROID_NDK_HOME}" \ + && rm -rf "${ANDROID_NDK_ARCHIVE}" + + +ENV ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk" + +# get the latest version from https://developer.android.com/studio/index.html +ENV ANDROID_SDK_TOOLS_VERSION="4333796" +ENV ANDROID_SDK_TOOLS_ARCHIVE="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip" +ENV ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}" + +# download and install Android SDK +RUN curl --location --progress-bar \ + "${ANDROID_SDK_TOOLS_DL_URL}" \ + --output "${ANDROID_SDK_TOOLS_ARCHIVE}" \ + && mkdir --parents "${ANDROID_SDK_HOME}" \ + && unzip -q "${ANDROID_SDK_TOOLS_ARCHIVE}" -d "${ANDROID_SDK_HOME}" \ + && rm -rf "${ANDROID_SDK_TOOLS_ARCHIVE}" + +# update Android SDK, install Android API, Build Tools... +RUN mkdir --parents "${ANDROID_SDK_HOME}/.android/" \ + && echo '### User Sources for Android SDK Manager' \ + > "${ANDROID_SDK_HOME}/.android/repositories.cfg" + +# accept Android licenses (JDK necessary!) +RUN apt -y update -qq \ + && apt -y install -qq --no-install-recommends openjdk-8-jdk \ + && apt -y autoremove \ + && apt -y clean +RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" --licenses > /dev/null + +# download platforms, API, build tools +RUN "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-24" && \ + "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-28" && \ + "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;28.0.3" && \ + "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "extras;android;m2repository" && \ + chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager" + + +ENV USER="user" +ENV HOME_DIR="/home/${USER}" +ENV WORK_DIR="${HOME_DIR}/wspace" \ + PATH="${HOME_DIR}/.local/bin:${PATH}" + +# install system dependencies +RUN apt -y update -qq \ + && apt -y install -qq --no-install-recommends \ + python virtualenv python-pip wget lbzip2 patch sudo \ + software-properties-common + +# install kivy +RUN add-apt-repository ppa:kivy-team/kivy \ + && apt -y update -qq \ + && apt -y install -qq --no-install-recommends python3-kivy \ + && apt -y autoremove \ + && apt -y clean +RUN python3 -m pip install image + +# build dependencies +# https://buildozer.readthedocs.io/en/latest/installation.html#android-on-ubuntu-16-04-64bit +RUN dpkg --add-architecture i386 \ + && apt -y update -qq \ + && apt -y install -qq --no-install-recommends \ + build-essential ccache git python2.7 python2.7-dev \ + libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \ + libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 \ + zip zlib1g-dev zlib1g:i386 \ + && apt -y autoremove \ + && apt -y clean + +# specific recipes dependencies (e.g. libffi requires autoreconf binary) +RUN apt -y update -qq \ + && apt -y install -qq --no-install-recommends \ + autoconf automake cmake gettext libltdl-dev libtool pkg-config \ + python3.7 \ + && apt -y autoremove \ + && apt -y clean + + +# prepare non root env +RUN useradd --create-home --shell /bin/bash ${USER} + +# with sudo access and no password +RUN usermod -append --groups sudo ${USER} +RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + + +WORKDIR ${WORK_DIR} + +# user needs ownership/write access to these directories +RUN chown --recursive ${USER} ${WORK_DIR} ${ANDROID_SDK_HOME} +RUN chown ${USER} /opt +USER ${USER} + + +RUN pip install --upgrade cython==0.29 +RUN python3 -m pip install --upgrade cython==0.29 + +# install buildozer +RUN cd /opt \ + && git clone https://github.com/kivy/buildozer \ + && cd buildozer \ + && python3 -m pip install -e . + +# install python-for-android +RUN cd /opt \ + && git clone https://github.com/kivy/python-for-android \ + && cd python-for-android \ + && git remote add sombernight https://github.com/SomberNight/python-for-android \ + && git fetch --all \ + && git checkout f74226666af69f9915afaee9ef9292db85a6c617 \ + && python3 -m pip install -e . + +# build env vars +ENV USE_SDK_WRAPPER=1 +ENV GRADLE_OPTS="-Xmx1536M -Dorg.gradle.jvmargs='-Xmx1536M'" diff --git a/electrum/gui/kivy/tools/build.sh b/electrum/gui/kivy/tools/build.sh new file mode 100755 index 000000000..fa8de30b7 --- /dev/null +++ b/electrum/gui/kivy/tools/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +pushd electrum/gui/kivy +make theming +popd + +sudo ./contrib/make_packages + +./contrib/make_apk diff --git a/electrum/gui/kivy/tools/buildozer.spec b/electrum/gui/kivy/tools/buildozer.spec index dca6daab8..2c4e77454 100644 --- a/electrum/gui/kivy/tools/buildozer.spec +++ b/electrum/gui/kivy/tools/buildozer.spec @@ -70,10 +70,10 @@ android.ndk = 14b android.private_storage = True # (str) Android NDK directory (if empty, it will be automatically downloaded.) -android.ndk_path = /opt/android-ndk-r14b +android.ndk_path = /opt/android/android-ndk # (str) Android SDK directory (if empty, it will be automatically downloaded.) -#android.sdk_path = +android.sdk_path = /opt/android/android-sdk # (str) Android entry point, default is ok for Kivy-based app #android.entrypoint = org.renpy.android.PythonActivity From 37b009a342b7a815bb31c63d3d496e019d9b1efa Mon Sep 17 00:00:00 2001 From: Janus Date: Mon, 26 Nov 2018 21:21:02 +0100 Subject: [PATCH 08/40] qt history view custom fiat input fixes previously, when you submitted a fiat value with thousands separator, it would be discarded. --- electrum/exchange_rate.py | 6 ++- electrum/gui/qt/history_list.py | 6 ++- electrum/tests/test_wallet.py | 71 +++++++++++++++++++++++++++++++++ electrum/util.py | 27 +++---------- electrum/wallet.py | 70 +++++++++++++++++++++----------- 5 files changed, 133 insertions(+), 47 deletions(-) diff --git a/electrum/exchange_rate.py b/electrum/exchange_rate.py index ae57cab73..4f3e9cb65 100644 --- a/electrum/exchange_rate.py +++ b/electrum/exchange_rate.py @@ -464,9 +464,13 @@ class FxThread(ThreadJob): d = get_exchanges_by_ccy(history) return d.get(ccy, []) + @staticmethod + def remove_thousands_separator(text): + return text.replace(',', '') # FIXME use THOUSAND_SEPARATOR in util + def ccy_amount_str(self, amount, commas): prec = CCY_PRECISIONS.get(self.ccy, 2) - fmt_str = "{:%s.%df}" % ("," if commas else "", max(0, prec)) + fmt_str = "{:%s.%df}" % ("," if commas else "", max(0, prec)) # FIXME use util.THOUSAND_SEPARATOR and util.DECIMAL_POINT try: rounded_amount = round(amount, prec) except decimal.InvalidOperation: diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index eb410d953..18e7d1dd3 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -275,10 +275,11 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): if value and value < 0: item.setForeground(3, red_brush) item.setForeground(4, red_brush) - if fiat_value and not tx_item['fiat_default']: + if fiat_value is not None and not tx_item['fiat_default']: item.setForeground(6, blue_brush) if tx_hash: item.setData(0, Qt.UserRole, tx_hash) + item.setData(0, Qt.UserRole+1, value) self.insertTopLevelItem(0, item) if current_tx == tx_hash: self.setCurrentItem(item) @@ -286,6 +287,7 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): def on_edited(self, item, column, prior): '''Called only when the text actually changes''' key = item.data(0, Qt.UserRole) + value = item.data(0, Qt.UserRole+1) text = item.text(column) # fixme if column == 3: @@ -293,7 +295,7 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): self.update_labels() self.parent.update_completions() elif column == 6: - self.parent.wallet.set_fiat_value(key, self.parent.fx.ccy, text) + self.parent.wallet.set_fiat_value(key, self.parent.fx.ccy, text, self.parent.fx, value) self.on_update() def on_doubleclick(self, item, column): diff --git a/electrum/tests/test_wallet.py b/electrum/tests/test_wallet.py index 9117392ea..ecfefae62 100644 --- a/electrum/tests/test_wallet.py +++ b/electrum/tests/test_wallet.py @@ -3,9 +3,16 @@ import tempfile import sys import os import json +from decimal import Decimal +from unittest import TestCase +import time from io import StringIO from electrum.storage import WalletStorage, FINAL_SEED_VERSION +from electrum.wallet import Abstract_Wallet +from electrum.exchange_rate import ExchangeBase, FxThread +from electrum.util import TxMinedStatus +from electrum.bitcoin import COIN from . import SequentialTestCase @@ -68,3 +75,67 @@ class TestWalletStorage(WalletTestCase): with open(self.wallet_path, "r") as f: contents = f.read() self.assertEqual(some_dict, json.loads(contents)) + +class FakeExchange(ExchangeBase): + def __init__(self, rate): + super().__init__(lambda self: None, lambda self: None) + self.quotes = {'TEST': rate} + +class FakeFxThread: + def __init__(self, exchange): + self.exchange = exchange + self.ccy = 'TEST' + + remove_thousands_separator = staticmethod(FxThread.remove_thousands_separator) + timestamp_rate = FxThread.timestamp_rate + ccy_amount_str = FxThread.ccy_amount_str + history_rate = FxThread.history_rate + +class FakeWallet: + def __init__(self, fiat_value): + super().__init__() + self.fiat_value = fiat_value + self.transactions = self.verified_tx = {'abc': 'Tx'} + + def get_tx_height(self, txid): + # because we use a current timestamp, and history is empty, + # FxThread.history_rate will use spot prices + return TxMinedStatus(height=10, conf=10, timestamp=time.time(), header_hash='def') + + default_fiat_value = Abstract_Wallet.default_fiat_value + price_at_timestamp = Abstract_Wallet.price_at_timestamp + class storage: + put = lambda self, x: None + +txid = 'abc' +ccy = 'TEST' + +class TestFiat(TestCase): + def setUp(self): + self.value_sat = COIN + self.fiat_value = {} + self.wallet = FakeWallet(fiat_value=self.fiat_value) + self.fx = FakeFxThread(FakeExchange(Decimal('1000.001'))) + default_fiat = Abstract_Wallet.default_fiat_value(self.wallet, txid, self.fx, self.value_sat) + self.assertEqual(Decimal('1000.001'), default_fiat) + self.assertEqual('1,000.00', self.fx.ccy_amount_str(default_fiat, commas=True)) + + def test_save_fiat_and_reset(self): + self.assertEqual(False, Abstract_Wallet.set_fiat_value(self.wallet, txid, ccy, '1000.01', self.fx, self.value_sat)) + saved = self.fiat_value[ccy][txid] + self.assertEqual('1,000.01', self.fx.ccy_amount_str(Decimal(saved), commas=True)) + self.assertEqual(True, Abstract_Wallet.set_fiat_value(self.wallet, txid, ccy, '', self.fx, self.value_sat)) + self.assertNotIn(txid, self.fiat_value[ccy]) + # even though we are not setting it to the exact fiat value according to the exchange rate, precision is truncated away + self.assertEqual(True, Abstract_Wallet.set_fiat_value(self.wallet, txid, ccy, '1,000.002', self.fx, self.value_sat)) + + def test_too_high_precision_value_resets_with_no_saved_value(self): + self.assertEqual(True, Abstract_Wallet.set_fiat_value(self.wallet, txid, ccy, '1,000.001', self.fx, self.value_sat)) + + def test_empty_resets(self): + self.assertEqual(True, Abstract_Wallet.set_fiat_value(self.wallet, txid, ccy, '', self.fx, self.value_sat)) + self.assertNotIn(ccy, self.fiat_value) + + def test_save_garbage(self): + self.assertEqual(False, Abstract_Wallet.set_fiat_value(self.wallet, txid, ccy, 'garbage', self.fx, self.value_sat)) + self.assertNotIn(ccy, self.fiat_value) diff --git a/electrum/util.py b/electrum/util.py index 92526980f..4f06d5739 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -39,6 +39,7 @@ import urllib.request, urllib.parse, urllib.error import builtins import json import time +from typing import NamedTuple, Optional import aiohttp from aiohttp_socks import SocksConnector, SocksVer @@ -129,31 +130,15 @@ class UserCancelled(Exception): '''An exception that is suppressed from the user''' pass -class Satoshis(object): - __slots__ = ('value',) - - def __new__(cls, value): - self = super(Satoshis, cls).__new__(cls) - self.value = value - return self - - def __repr__(self): - return 'Satoshis(%d)'%self.value +class Satoshis(NamedTuple): + value: int def __str__(self): return format_satoshis(self.value) + " BTC" -class Fiat(object): - __slots__ = ('value', 'ccy') - - def __new__(cls, value, ccy): - self = super(Fiat, cls).__new__(cls) - self.ccy = ccy - self.value = value - return self - - def __repr__(self): - return 'Fiat(%s)'% self.__str__() +class Fiat(NamedTuple): + value: Optional[Decimal] + ccy: str def __str__(self): if self.value is None or self.value.is_nan(): diff --git a/electrum/wallet.py b/electrum/wallet.py index c104d7bee..a76347217 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -247,24 +247,37 @@ class Abstract_Wallet(AddressSynchronizer): self.storage.put('labels', self.labels) return changed - def set_fiat_value(self, txid, ccy, text): + def set_fiat_value(self, txid, ccy, text, fx, value): if txid not in self.transactions: return - if not text: + # since fx is inserting the thousands separator, + # and not util, also have fx remove it + text = fx.remove_thousands_separator(text) + def_fiat = self.default_fiat_value(txid, fx, value) + formatted = fx.ccy_amount_str(def_fiat, commas=False) + def_fiat_rounded = Decimal(formatted) + reset = not text + if not reset: + try: + text_dec = Decimal(text) + text_dec_rounded = Decimal(fx.ccy_amount_str(text_dec, commas=False)) + reset = text_dec_rounded == def_fiat_rounded + except: + # garbage. not resetting, but not saving either + return False + if reset: d = self.fiat_value.get(ccy, {}) if d and txid in d: d.pop(txid) else: - return - else: - try: - Decimal(text) - except: - return + # avoid saving empty dict + return True if ccy not in self.fiat_value: self.fiat_value[ccy] = {} - self.fiat_value[ccy][txid] = text + if not reset: + self.fiat_value[ccy][txid] = text self.storage.put('fiat_value', self.fiat_value) + return reset def get_fiat_value(self, txid, ccy): fiat_value = self.fiat_value.get(ccy, {}).get(txid) @@ -423,21 +436,11 @@ class Abstract_Wallet(AddressSynchronizer): income += value # fiat computations if fx and fx.is_enabled() and fx.get_history_config(): - fiat_value = self.get_fiat_value(tx_hash, fx.ccy) - fiat_default = fiat_value is None - fiat_rate = self.price_at_timestamp(tx_hash, fx.timestamp_rate) - fiat_value = fiat_value if fiat_value is not None else value / Decimal(COIN) * fiat_rate - fiat_fee = tx_fee / Decimal(COIN) * fiat_rate if tx_fee is not None else None - item['fiat_value'] = Fiat(fiat_value, fx.ccy) - item['fiat_fee'] = Fiat(fiat_fee, fx.ccy) if fiat_fee else None - item['fiat_default'] = fiat_default + fiat_fields = self.get_tx_item_fiat(tx_hash, value, fx, tx_fee) + fiat_value = fiat_fields['fiat_value'].value + item.update(fiat_fields) if value < 0: - acquisition_price = - value / Decimal(COIN) * self.average_price(tx_hash, fx.timestamp_rate, fx.ccy) - liquidation_price = - fiat_value - item['acquisition_price'] = Fiat(acquisition_price, fx.ccy) - cg = liquidation_price - acquisition_price - item['capital_gain'] = Fiat(cg, fx.ccy) - capital_gains += cg + capital_gains += fiat_fields['capital_gain'].value fiat_expenditures += -fiat_value else: fiat_income += fiat_value @@ -478,6 +481,27 @@ class Abstract_Wallet(AddressSynchronizer): 'summary': summary } + def default_fiat_value(self, tx_hash, fx, value): + return value / Decimal(COIN) * self.price_at_timestamp(tx_hash, fx.timestamp_rate) + + def get_tx_item_fiat(self, tx_hash, value, fx, tx_fee): + item = {} + fiat_value = self.get_fiat_value(tx_hash, fx.ccy) + fiat_default = fiat_value is None + fiat_rate = self.price_at_timestamp(tx_hash, fx.timestamp_rate) + fiat_value = fiat_value if fiat_value is not None else self.default_fiat_value(tx_hash, fx, value) + fiat_fee = tx_fee / Decimal(COIN) * fiat_rate if tx_fee is not None else None + item['fiat_value'] = Fiat(fiat_value, fx.ccy) + item['fiat_fee'] = Fiat(fiat_fee, fx.ccy) if fiat_fee else None + item['fiat_default'] = fiat_default + if value < 0: + acquisition_price = - value / Decimal(COIN) * self.average_price(tx_hash, fx.timestamp_rate, fx.ccy) + liquidation_price = - fiat_value + item['acquisition_price'] = Fiat(acquisition_price, fx.ccy) + cg = liquidation_price - acquisition_price + item['capital_gain'] = Fiat(cg, fx.ccy) + return item + def get_label(self, tx_hash): label = self.labels.get(tx_hash, '') if label is '': From c5b8706225ad028f3fd306928088b48a7b830ecb Mon Sep 17 00:00:00 2001 From: ThomasV Date: Tue, 27 Nov 2018 18:34:36 +0100 Subject: [PATCH 09/40] simplify test --- electrum/wallet.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/electrum/wallet.py b/electrum/wallet.py index a76347217..cc71079d2 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -272,9 +272,9 @@ class Abstract_Wallet(AddressSynchronizer): else: # avoid saving empty dict return True - if ccy not in self.fiat_value: - self.fiat_value[ccy] = {} - if not reset: + else: + if ccy not in self.fiat_value: + self.fiat_value[ccy] = {} self.fiat_value[ccy][txid] = text self.storage.put('fiat_value', self.fiat_value) return reset From d4d5e32c91788115c754b933a1e8e6e0338fb196 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 27 Nov 2018 21:15:31 +0100 Subject: [PATCH 10/40] qt history list: fix Qt.UserRole collision --- electrum/gui/qt/history_list.py | 30 ++++++++++++++++-------------- electrum/gui/qt/util.py | 2 +- electrum/wallet.py | 8 ++++---- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index 18e7d1dd3..83e93d065 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -60,6 +60,8 @@ TX_ICONS = [ class HistoryList(MyTreeWidget, AcceptFileDragDrop): filter_columns = [2, 3, 4] # Date, Description, Amount + TX_HASH_ROLE = Qt.UserRole + TX_VALUE_ROLE = Qt.UserRole + 1 def __init__(self, parent=None): MyTreeWidget.__init__(self, parent, self.create_menu, [], 3) @@ -231,7 +233,7 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): self.years = [str(i) for i in range(start_date.year, end_date.year + 1)] self.period_combo.insertItems(1, self.years) item = self.currentItem() - current_tx = item.data(0, Qt.UserRole) if item else None + current_tx = item.data(0, self.TX_HASH_ROLE) if item else None self.clear() if fx: fx.history_used_spot = False blue_brush = QBrush(QColor("#1E1EFF")) @@ -242,23 +244,23 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): height = tx_item['height'] conf = tx_item['confirmations'] timestamp = tx_item['timestamp'] - value = tx_item['value'].value + value_sat = tx_item['value'].value balance = tx_item['balance'].value label = tx_item['label'] tx_mined_status = TxMinedStatus(height, conf, timestamp, None) status, status_str = self.wallet.get_tx_status(tx_hash, tx_mined_status) has_invoice = self.wallet.invoices.paid.get(tx_hash) icon = self.icon_cache.get(":icons/" + TX_ICONS[status]) - v_str = self.parent.format_amount(value, is_diff=True, whitespaces=True) + v_str = self.parent.format_amount(value_sat, is_diff=True, whitespaces=True) balance_str = self.parent.format_amount(balance, whitespaces=True) entry = ['', tx_hash, status_str, label, v_str, balance_str] fiat_value = None - if value is not None and fx and fx.show_history(): + if value_sat is not None and fx and fx.show_history(): fiat_value = tx_item['fiat_value'].value value_str = fx.format_fiat(fiat_value) entry.append(value_str) # fixme: should use is_mine - if value < 0: + if value_sat < 0: entry.append(fx.format_fiat(tx_item['acquisition_price'].value)) entry.append(fx.format_fiat(tx_item['capital_gain'].value)) item = SortableTreeWidgetItem(entry) @@ -272,22 +274,22 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): item.setTextAlignment(i, Qt.AlignRight | Qt.AlignVCenter) if i!=2: item.setFont(i, monospace_font) - if value and value < 0: + if value_sat and value_sat < 0: item.setForeground(3, red_brush) item.setForeground(4, red_brush) if fiat_value is not None and not tx_item['fiat_default']: item.setForeground(6, blue_brush) if tx_hash: - item.setData(0, Qt.UserRole, tx_hash) - item.setData(0, Qt.UserRole+1, value) + item.setData(0, self.TX_HASH_ROLE, tx_hash) + item.setData(0, self.TX_VALUE_ROLE, value_sat) self.insertTopLevelItem(0, item) if current_tx == tx_hash: self.setCurrentItem(item) def on_edited(self, item, column, prior): '''Called only when the text actually changes''' - key = item.data(0, Qt.UserRole) - value = item.data(0, Qt.UserRole+1) + key = item.data(0, self.TX_HASH_ROLE) + value_sat = item.data(0, self.TX_VALUE_ROLE) text = item.text(column) # fixme if column == 3: @@ -295,14 +297,14 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): self.update_labels() self.parent.update_completions() elif column == 6: - self.parent.wallet.set_fiat_value(key, self.parent.fx.ccy, text, self.parent.fx, value) + self.parent.wallet.set_fiat_value(key, self.parent.fx.ccy, text, self.parent.fx, value_sat) self.on_update() def on_doubleclick(self, item, column): if self.permit_edit(item, column): super(HistoryList, self).on_doubleclick(item, column) else: - tx_hash = item.data(0, Qt.UserRole) + tx_hash = item.data(0, self.TX_HASH_ROLE) self.show_transaction(tx_hash) def show_transaction(self, tx_hash): @@ -317,7 +319,7 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): child_count = root.childCount() for i in range(child_count): item = root.child(i) - txid = item.data(0, Qt.UserRole) + txid = item.data(0, self.TX_HASH_ROLE) label = self.wallet.get_label(txid) item.setText(3, label) @@ -340,7 +342,7 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): if not item: return column = self.currentColumn() - tx_hash = item.data(0, Qt.UserRole) + tx_hash = item.data(0, self.TX_HASH_ROLE) if not tx_hash: return tx = self.wallet.transactions.get(tx_hash) diff --git a/electrum/gui/qt/util.py b/electrum/gui/qt/util.py index 5e1912a3e..02a736834 100644 --- a/electrum/gui/qt/util.py +++ b/electrum/gui/qt/util.py @@ -791,7 +791,7 @@ def get_parent_main_window(widget): return None class SortableTreeWidgetItem(QTreeWidgetItem): - DataRole = Qt.UserRole + 1 + DataRole = Qt.UserRole + 100 def __lt__(self, other): column = self.treeWidget().sortColumn() diff --git a/electrum/wallet.py b/electrum/wallet.py index cc71079d2..9a60e180f 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -247,13 +247,13 @@ class Abstract_Wallet(AddressSynchronizer): self.storage.put('labels', self.labels) return changed - def set_fiat_value(self, txid, ccy, text, fx, value): + def set_fiat_value(self, txid, ccy, text, fx, value_sat): if txid not in self.transactions: return # since fx is inserting the thousands separator, # and not util, also have fx remove it text = fx.remove_thousands_separator(text) - def_fiat = self.default_fiat_value(txid, fx, value) + def_fiat = self.default_fiat_value(txid, fx, value_sat) formatted = fx.ccy_amount_str(def_fiat, commas=False) def_fiat_rounded = Decimal(formatted) reset = not text @@ -481,8 +481,8 @@ class Abstract_Wallet(AddressSynchronizer): 'summary': summary } - def default_fiat_value(self, tx_hash, fx, value): - return value / Decimal(COIN) * self.price_at_timestamp(tx_hash, fx.timestamp_rate) + def default_fiat_value(self, tx_hash, fx, value_sat): + return value_sat / Decimal(COIN) * self.price_at_timestamp(tx_hash, fx.timestamp_rate) def get_tx_item_fiat(self, tx_hash, value, fx, tx_fee): item = {} From 4a7ce238fd312d5ed78fdcce92beec7f2f37ebb4 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 27 Nov 2018 21:32:55 +0100 Subject: [PATCH 11/40] qt history list: fix sort order of fiat columns --- electrum/gui/qt/history_list.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index 83e93d065..9d05ac74f 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -266,7 +266,6 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): item = SortableTreeWidgetItem(entry) item.setIcon(0, icon) item.setToolTip(0, str(conf) + " confirmation" + ("s" if conf != 1 else "")) - item.setData(0, SortableTreeWidgetItem.DataRole, (status, conf)) if has_invoice: item.setIcon(3, self.icon_cache.get(":icons/seal")) for i in range(len(entry)): @@ -279,6 +278,15 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): item.setForeground(4, red_brush) if fiat_value is not None and not tx_item['fiat_default']: item.setForeground(6, blue_brush) + # sort orders + item.setData(0, SortableTreeWidgetItem.DataRole, (status, conf)) + item.setData(4, SortableTreeWidgetItem.DataRole, value_sat) + item.setData(5, SortableTreeWidgetItem.DataRole, balance) + if fiat_value is not None: + item.setData(6, SortableTreeWidgetItem.DataRole, fiat_value) + if value_sat < 0: + item.setData(7, SortableTreeWidgetItem.DataRole, tx_item['acquisition_price'].value) + item.setData(8, SortableTreeWidgetItem.DataRole, tx_item['capital_gain'].value) if tx_hash: item.setData(0, self.TX_HASH_ROLE, tx_hash) item.setData(0, self.TX_VALUE_ROLE, value_sat) From e12af33626622809640fc29804e0ef14ae3dfb9b Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 28 Nov 2018 12:35:53 +0100 Subject: [PATCH 12/40] wallet: cache more in get_tx_fee closes #4879 --- electrum/address_synchronizer.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index e0d1ec241..909ed028d 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -717,12 +717,15 @@ class AddressSynchronizer(PrintError): return None if hasattr(tx, '_cached_fee'): return tx._cached_fee - is_relevant, is_mine, v, fee = self.get_wallet_delta(tx) - if fee is None: - txid = tx.txid() - fee = self.tx_fees.get(txid) - if fee is not None: - tx._cached_fee = fee + with self.lock, self.transaction_lock: + is_relevant, is_mine, v, fee = self.get_wallet_delta(tx) + if fee is None: + txid = tx.txid() + fee = self.tx_fees.get(txid) + # cache fees. if wallet is synced, cache all; + # otherwise only cache non-None, as None can still change while syncing + if self.up_to_date or fee is not None: + tx._cached_fee = fee return fee def get_addr_io(self, address): From 99325618a6c6cf27e29fa6015359342d1b5268d0 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 28 Nov 2018 15:52:38 +0100 Subject: [PATCH 13/40] wallet: add FIXME re fiat coin_price calculation --- electrum/wallet.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/electrum/wallet.py b/electrum/wallet.py index 9a60e180f..f3f3efe4c 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -1185,6 +1185,9 @@ class Abstract_Wallet(AddressSynchronizer): """ if txin_value is None: return Decimal('NaN') + # FIXME: this mutual recursion will be really slow and might even reach + # max recursion depth if there are no FX rates available as then + # nothing will be cached. cache_key = "{}:{}:{}".format(str(txid), str(ccy), str(txin_value)) result = self.coin_price_cache.get(cache_key, None) if result is not None: From 505cb2f65db2ef83d02e082acabcd8819dd347b2 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 28 Nov 2018 16:43:44 +0100 Subject: [PATCH 14/40] build-wine: update git version --- contrib/build-wine/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/build-wine/docker/Dockerfile b/contrib/build-wine/docker/Dockerfile index 340bad00d..8ed7a5467 100644 --- a/contrib/build-wine/docker/Dockerfile +++ b/contrib/build-wine/docker/Dockerfile @@ -20,7 +20,7 @@ RUN dpkg --add-architecture i386 && \ wine-stable-i386:i386=3.0.1~bionic \ wine-stable:amd64=3.0.1~bionic \ winehq-stable:amd64=3.0.1~bionic \ - git=1:2.17.1-1ubuntu0.3 \ + git=1:2.17.1-1ubuntu0.4 \ p7zip-full=16.02+dfsg-6 \ make=4.1-9.1ubuntu1 \ mingw-w64=5.0.3-1 \ From 243a0e3cf1f3953b2c6c9796fe739613153e191e Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 27 Nov 2018 21:35:26 +0100 Subject: [PATCH 15/40] android docker: make_apk optionally takes "release" as arg --- contrib/make_apk | 2 ++ electrum/gui/kivy/Readme.md | 16 +++++++++++----- electrum/gui/kivy/tools/build.sh | 9 --------- 3 files changed, 13 insertions(+), 14 deletions(-) delete mode 100755 electrum/gui/kivy/tools/build.sh diff --git a/contrib/make_apk b/contrib/make_apk index 773aeab54..6940222cd 100755 --- a/contrib/make_apk +++ b/contrib/make_apk @@ -2,6 +2,8 @@ pushd ./electrum/gui/kivy/ +make theming + if [[ -n "$1" && "$1" == "release" ]] ; then echo -n Keystore Password: read -s password diff --git a/electrum/gui/kivy/Readme.md b/electrum/gui/kivy/Readme.md index 83bc01a36..f394ddf13 100644 --- a/electrum/gui/kivy/Readme.md +++ b/electrum/gui/kivy/Readme.md @@ -24,22 +24,28 @@ folder. $ sudo docker build -t electrum-android-builder-img electrum/gui/kivy/tools ``` -3. Build binaries +3. Prepare pure python dependencies ``` - $ sudo docker run \ + $ sudo ./contrib/make_packages + ``` + +4. Build binaries + + ``` + $ sudo docker run -it --rm \ --name electrum-android-builder-cont \ - --rm \ -v $PWD:/home/user/wspace/electrum \ + -v ~/.keystore:/home/user/.keystore \ --workdir /home/user/wspace/electrum \ electrum-android-builder-img \ - ./electrum/gui/kivy/tools/build.sh + ./contrib/make_apk ``` This mounts the project dir inside the container, and so the modifications will affect it, e.g. `.buildozer` folder will be created. -4. The generated binary is in `./bin`. +5. The generated binary is in `./bin`. diff --git a/electrum/gui/kivy/tools/build.sh b/electrum/gui/kivy/tools/build.sh deleted file mode 100755 index fa8de30b7..000000000 --- a/electrum/gui/kivy/tools/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -pushd electrum/gui/kivy -make theming -popd - -sudo ./contrib/make_packages - -./contrib/make_apk From d0e6b8c89dee017e4ad790d63e4f642a7e28e67c Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 28 Nov 2018 20:54:57 +0100 Subject: [PATCH 16/40] hw: fix passphrase dialog with confirmation closes #4876 --- electrum/gui/qt/installwizard.py | 4 ++-- electrum/gui/qt/password_dialog.py | 17 ++++++++--------- electrum/plugins/hw_wallet/qt.py | 14 ++++++++++---- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index dc4ce28e4..ffd188671 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -432,7 +432,7 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard): return slayout.is_ext def pw_layout(self, msg, kind, force_disable_encrypt_cb): - playout = PasswordLayout(None, msg, kind, self.next_button, + playout = PasswordLayout(msg=msg, kind=kind, OK_button=self.next_button, force_disable_encrypt_cb=force_disable_encrypt_cb) playout.encrypt_cb.setChecked(True) self.exec_layout(playout.layout()) @@ -446,7 +446,7 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard): @wizard_dialog def request_storage_encryption(self, run_next): - playout = PasswordLayoutForHW(None, MSG_HW_STORAGE_ENCRYPTION, PW_NEW, self.next_button) + playout = PasswordLayoutForHW(MSG_HW_STORAGE_ENCRYPTION) playout.encrypt_cb.setChecked(True) self.exec_layout(playout.layout()) return playout.encrypt_cb.isChecked() diff --git a/electrum/gui/qt/password_dialog.py b/electrum/gui/qt/password_dialog.py index 66b3f51b9..3202618ec 100644 --- a/electrum/gui/qt/password_dialog.py +++ b/electrum/gui/qt/password_dialog.py @@ -60,7 +60,7 @@ class PasswordLayout(object): titles = [_("Enter Password"), _("Change Password"), _("Enter Passphrase")] - def __init__(self, wallet, msg, kind, OK_button, force_disable_encrypt_cb=False): + def __init__(self, msg, kind, OK_button, wallet=None, force_disable_encrypt_cb=False): self.wallet = wallet self.pw = QLineEdit() @@ -169,12 +169,9 @@ class PasswordLayout(object): class PasswordLayoutForHW(object): - def __init__(self, wallet, msg, kind, OK_button): + def __init__(self, msg, wallet=None): self.wallet = wallet - self.kind = kind - self.OK_button = OK_button - vbox = QVBoxLayout() label = QLabel(msg + "\n") label.setWordWrap(True) @@ -254,9 +251,11 @@ class ChangePasswordDialogForSW(ChangePasswordDialogBase): else: msg = _('Your wallet is password protected and encrypted.') msg += ' ' + _('Use this dialog to change your password.') - self.playout = PasswordLayout( - wallet, msg, PW_CHANGE, OK_button, - force_disable_encrypt_cb=not wallet.can_have_keystore_encryption()) + self.playout = PasswordLayout(msg=msg, + kind=PW_CHANGE, + OK_button=OK_button, + wallet=wallet, + force_disable_encrypt_cb=not wallet.can_have_keystore_encryption()) def run(self): if not self.exec_(): @@ -276,7 +275,7 @@ class ChangePasswordDialogForHW(ChangePasswordDialogBase): msg = _('Your wallet file is encrypted.') msg += '\n' + _('Note: If you enable this setting, you will need your hardware device to open your wallet.') msg += '\n' + _('Use this dialog to toggle encryption.') - self.playout = PasswordLayoutForHW(wallet, msg, PW_CHANGE, OK_button) + self.playout = PasswordLayoutForHW(msg) def run(self): if not self.exec_(): diff --git a/electrum/plugins/hw_wallet/qt.py b/electrum/plugins/hw_wallet/qt.py index 2b5215eb6..fc188ad00 100644 --- a/electrum/plugins/hw_wallet/qt.py +++ b/electrum/plugins/hw_wallet/qt.py @@ -27,7 +27,8 @@ import threading from PyQt5.Qt import QVBoxLayout, QLabel -from electrum.gui.qt.password_dialog import PasswordDialog, PW_PASSPHRASE + +from electrum.gui.qt.password_dialog import PasswordLayout, PW_PASSPHRASE from electrum.gui.qt.util import * from electrum.i18n import _ @@ -114,11 +115,16 @@ class QtHandlerBase(QObject, PrintError): def passphrase_dialog(self, msg, confirm): # If confirm is true, require the user to enter the passphrase twice parent = self.top_level_window() + d = WindowModalDialog(parent, _("Enter Passphrase")) if confirm: - d = PasswordDialog(parent, None, msg, PW_PASSPHRASE) - confirmed, p, passphrase = d.run() + OK_button = OkButton(d) + playout = PasswordLayout(msg=msg, kind=PW_PASSPHRASE, OK_button=OK_button) + vbox = QVBoxLayout() + vbox.addLayout(playout.layout()) + vbox.addLayout(Buttons(CancelButton(d), OK_button)) + d.setLayout(vbox) + passphrase = playout.new_password() if d.exec_() else None else: - d = WindowModalDialog(parent, _("Enter Passphrase")) pw = QLineEdit() pw.setEchoMode(2) pw.setMinimumWidth(200) From db89286ec3fa19156b7a85628e93469ef1616546 Mon Sep 17 00:00:00 2001 From: Calin Culianu Date: Thu, 29 Nov 2018 00:09:06 +0200 Subject: [PATCH 17/40] [macOS] Added QR scanner facility using platform-native helper app. --- .gitmodules | 3 +++ contrib/CalinsQRReader | 1 + contrib/build-osx/base.sh | 2 +- contrib/build-osx/make_osx | 9 +++++++++ contrib/build-osx/osx.spec | 3 +++ electrum/qrscanner.py | 25 ++++++++++++++++++++++++- 6 files changed, 41 insertions(+), 2 deletions(-) create mode 160000 contrib/CalinsQRReader diff --git a/.gitmodules b/.gitmodules index 5a0f914f1..34cfeafbd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "contrib/deterministic-build/electrum-locale"] path = contrib/deterministic-build/electrum-locale url = https://github.com/spesmilo/electrum-locale +[submodule "contrib/CalinsQRReader"] + path = contrib/CalinsQRReader + url = https://github.com/spesmilo/CalinsQRReader diff --git a/contrib/CalinsQRReader b/contrib/CalinsQRReader new file mode 160000 index 000000000..20189155a --- /dev/null +++ b/contrib/CalinsQRReader @@ -0,0 +1 @@ +Subproject commit 20189155a461cf7fbad14357e58fbc8e7c964608 diff --git a/contrib/build-osx/base.sh b/contrib/build-osx/base.sh index e7454f7a8..2c22ca9cf 100644 --- a/contrib/build-osx/base.sh +++ b/contrib/build-osx/base.sh @@ -21,7 +21,7 @@ function DoCodeSignMaybe { # ARGS: infoName fileOrDirName codesignIdentity identity="$3" deep="" if [ -z "$identity" ]; then - # we are ok with them not passing anything -- master script calls us always even if no identity is specified + # we are ok with them not passing anything; master script calls us unconditionally even if no identity is specified return fi if [ -d "$file" ]; then diff --git a/contrib/build-osx/make_osx b/contrib/build-osx/make_osx index ecccbef38..ff836fd62 100755 --- a/contrib/build-osx/make_osx +++ b/contrib/build-osx/make_osx @@ -16,6 +16,7 @@ export PYTHONHASHSEED=22 VERSION=`git describe --tags --dirty --always` which brew > /dev/null 2>&1 || fail "Please install brew from https://brew.sh/ to continue" +which xcodebuild > /dev/null 2>&1 || fail "Please install Xcode and xcode command line tools to continue" # Code Signing: See https://developer.apple.com/library/archive/documentation/Security/Conceptual/CodeSigningGuide/Procedures/Procedures.html APP_SIGN="" @@ -87,6 +88,14 @@ popd cp $BUILDDIR/secp256k1/.libs/libsecp256k1.0.dylib contrib/build-osx DoCodeSignMaybe "libsecp256k1" "contrib/build-osx/libsecp256k1.0.dylib" "$APP_SIGN" # If APP_SIGN is empty will be a noop +info "Building CalinsQRReader..." +d=contrib/CalinsQRReader +pushd $d +rm -fr build +xcodebuild || fail "Could not build CalinsQRReader" +popd +DoCodeSignMaybe "CalinsQRReader.app" "${d}/build/Release/CalinsQRReader.app" "$APP_SIGN" # If APP_SIGN is empty will be a noop + info "Installing requirements..." python3 -m pip install -Ir ./contrib/deterministic-build/requirements.txt --user && \ diff --git a/contrib/build-osx/osx.spec b/contrib/build-osx/osx.spec index 501df88ec..e48ba97cf 100644 --- a/contrib/build-osx/osx.spec +++ b/contrib/build-osx/osx.spec @@ -41,6 +41,9 @@ datas += collect_data_files('btchip') datas += collect_data_files('keepkeylib') datas += collect_data_files('ckcc') +# Add the QR Scanner helper app +datas += [(electrum + "contrib/CalinsQRReader/build/Release/CalinsQRReader.app", "./contrib/CalinsQRReader/build/Release/CalinsQRReader.app")] + # Add libusb so Trezor and Safe-T mini will work binaries = [(electrum + "contrib/build-osx/libusb-1.0.dylib", ".")] binaries += [(electrum + "contrib/build-osx/libsecp256k1.0.dylib", ".")] diff --git a/electrum/qrscanner.py b/electrum/qrscanner.py index 463d5ec6f..5d15aad9a 100644 --- a/electrum/qrscanner.py +++ b/electrum/qrscanner.py @@ -40,7 +40,7 @@ except BaseException: libzbar = None -def scan_barcode(device='', timeout=-1, display=True, threaded=False, try_again=True): +def scan_barcode_ctypes(device='', timeout=-1, display=True, threaded=False, try_again=True): if libzbar is None: raise RuntimeError("Cannot start QR scanner; zbar not available.") libzbar.zbar_symbol_get_data.restype = ctypes.c_char_p @@ -69,6 +69,29 @@ def scan_barcode(device='', timeout=-1, display=True, threaded=False, try_again= data = libzbar.zbar_symbol_get_data(symbol) return data.decode('utf8') +def scan_barcode_osx(*args_ignored, **kwargs_ignored): + import subprocess + # NOTE: This code needs to be modified if the positions of this file changes with respect to the helper app! + # This assumes the built macOS .app bundle which ends up putting the helper app in + # .app/contrib/CalinsQRReader/build/Release/CalinsQRReader.app. + root_ec_dir = os.path.abspath(os.path.dirname(__file__) + "/../") + prog = root_ec_dir + "/" + "contrib/CalinsQRReader/build/Release/CalinsQRReader.app/Contents/MacOS/CalinsQRReader" + if not os.path.exists(prog): + raise RuntimeError("Cannot start QR scanner; helper app not found.") + data = '' + try: + # This will run the "CalinsQRReader" helper app (which also gets bundled with the built .app) + # Just like the zbar implementation -- the main app will hang until the QR window returns a QR code + # (or is closed). Communication with the subprocess is done via stdout. + # See contrib/CalinsQRReader for the helper app source code. + with subprocess.Popen([prog], stdout=subprocess.PIPE) as p: + data = p.stdout.read().decode('utf-8').strip() + return data + except OSError as e: + raise RuntimeError("Cannot start camera helper app; {}".format(e.strerror)) + +scan_barcode = scan_barcode_osx if sys.platform == 'darwin' else scan_barcode_ctypes + def _find_system_cameras(): device_root = "/sys/class/video4linux" devices = {} # Name -> device From d7bf8826fc1e8c3692504edbe60523dd1327b8a0 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 29 Nov 2018 11:39:57 +0100 Subject: [PATCH 18/40] rename contrib/build-osx as contrib/osx. Move QRReader submodule there. --- .gitmodules | 4 ++-- README.rst | 2 +- contrib/{build-osx => osx}/README.md | 4 ++-- contrib/{build-osx => osx}/base.sh | 0 .../{build-osx => osx}/cdrkit-deterministic.patch | 0 contrib/{build-osx => osx}/make_osx | 12 ++++++------ contrib/{build-osx => osx}/osx.spec | 6 +++--- contrib/{build-osx => osx}/package.sh | 0 electrum/qrscanner.py | 4 ++-- 9 files changed, 16 insertions(+), 16 deletions(-) rename contrib/{build-osx => osx}/README.md (93%) rename contrib/{build-osx => osx}/base.sh (100%) rename contrib/{build-osx => osx}/cdrkit-deterministic.patch (100%) rename contrib/{build-osx => osx}/make_osx (91%) rename contrib/{build-osx => osx}/osx.spec (92%) rename contrib/{build-osx => osx}/package.sh (100%) diff --git a/.gitmodules b/.gitmodules index 34cfeafbd..c6788ecf0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,6 +4,6 @@ [submodule "contrib/deterministic-build/electrum-locale"] path = contrib/deterministic-build/electrum-locale url = https://github.com/spesmilo/electrum-locale -[submodule "contrib/CalinsQRReader"] - path = contrib/CalinsQRReader +[submodule "contrib/osx/CalinsQRReader"] + path = contrib/osx/CalinsQRReader url = https://github.com/spesmilo/CalinsQRReader diff --git a/README.rst b/README.rst index 3f67724b4..1bb140d86 100644 --- a/README.rst +++ b/README.rst @@ -101,7 +101,7 @@ This directory contains the python dependencies used by Electrum. Mac OS X / macOS -------- -See `contrib/build-osx/`. +See `contrib/osx/`. Windows ------- diff --git a/contrib/build-osx/README.md b/contrib/osx/README.md similarity index 93% rename from contrib/build-osx/README.md rename to contrib/osx/README.md index c1e96d90b..056d9fb84 100644 --- a/contrib/build-osx/README.md +++ b/contrib/osx/README.md @@ -14,7 +14,7 @@ Before starting, make sure that the Xcode command line tools are installed (e.g. cd electrum - ./contrib/build-osx/make_osx + ./contrib/osx/make_osx This creates a folder named Electrum.app. @@ -33,4 +33,4 @@ Copy the Electrum.app directory over and install the dependencies, e.g.: Then you can just invoke `package.sh` with the path to the app: cd electrum - ./contrib/build-osx/package.sh ~/Electrum.app/ \ No newline at end of file + ./contrib/osx/package.sh ~/Electrum.app/ \ No newline at end of file diff --git a/contrib/build-osx/base.sh b/contrib/osx/base.sh similarity index 100% rename from contrib/build-osx/base.sh rename to contrib/osx/base.sh diff --git a/contrib/build-osx/cdrkit-deterministic.patch b/contrib/osx/cdrkit-deterministic.patch similarity index 100% rename from contrib/build-osx/cdrkit-deterministic.patch rename to contrib/osx/cdrkit-deterministic.patch diff --git a/contrib/build-osx/make_osx b/contrib/osx/make_osx similarity index 91% rename from contrib/build-osx/make_osx rename to contrib/osx/make_osx index ff836fd62..b1cf4b724 100755 --- a/contrib/build-osx/make_osx +++ b/contrib/osx/make_osx @@ -72,8 +72,8 @@ cp ./contrib/deterministic-build/electrum-icons/icons_rc.py ./electrum/gui/qt info "Downloading libusb..." curl https://homebrew.bintray.com/bottles/libusb-1.0.22.el_capitan.bottle.tar.gz | \ tar xz --directory $BUILDDIR -cp $BUILDDIR/libusb/1.0.22/lib/libusb-1.0.dylib contrib/build-osx -DoCodeSignMaybe "libusb" "contrib/build-osx/libusb-1.0.dylib" "$APP_SIGN" # If APP_SIGN is empty will be a noop +cp $BUILDDIR/libusb/1.0.22/lib/libusb-1.0.dylib contrib/osx +DoCodeSignMaybe "libusb" "contrib/osx/libusb-1.0.dylib" "$APP_SIGN" # If APP_SIGN is empty will be a noop info "Building libsecp256k1" brew install autoconf automake libtool @@ -85,11 +85,11 @@ git clean -f -x -q ./configure --enable-module-recovery --enable-experimental --enable-module-ecdh --disable-jni make popd -cp $BUILDDIR/secp256k1/.libs/libsecp256k1.0.dylib contrib/build-osx -DoCodeSignMaybe "libsecp256k1" "contrib/build-osx/libsecp256k1.0.dylib" "$APP_SIGN" # If APP_SIGN is empty will be a noop +cp $BUILDDIR/secp256k1/.libs/libsecp256k1.0.dylib contrib/osx +DoCodeSignMaybe "libsecp256k1" "contrib/osx/libsecp256k1.0.dylib" "$APP_SIGN" # If APP_SIGN is empty will be a noop info "Building CalinsQRReader..." -d=contrib/CalinsQRReader +d=contrib/osx/CalinsQRReader pushd $d rm -fr build xcodebuild || fail "Could not build CalinsQRReader" @@ -117,7 +117,7 @@ for d in ~/Library/Python/ ~/.pyenv .; do done info "Building binary" -pyinstaller --noconfirm --ascii --clean --name $VERSION contrib/build-osx/osx.spec || fail "Could not build binary" +pyinstaller --noconfirm --ascii --clean --name $VERSION contrib/osx/osx.spec || fail "Could not build binary" info "Adding bitcoin URI types to Info.plist" plutil -insert 'CFBundleURLTypes' \ diff --git a/contrib/build-osx/osx.spec b/contrib/osx/osx.spec similarity index 92% rename from contrib/build-osx/osx.spec rename to contrib/osx/osx.spec index e48ba97cf..28ac336e4 100644 --- a/contrib/build-osx/osx.spec +++ b/contrib/osx/osx.spec @@ -42,11 +42,11 @@ datas += collect_data_files('keepkeylib') datas += collect_data_files('ckcc') # Add the QR Scanner helper app -datas += [(electrum + "contrib/CalinsQRReader/build/Release/CalinsQRReader.app", "./contrib/CalinsQRReader/build/Release/CalinsQRReader.app")] +datas += [(electrum + "contrib/osx/CalinsQRReader/build/Release/CalinsQRReader.app", "./contrib/osx/CalinsQRReader/build/Release/CalinsQRReader.app")] # Add libusb so Trezor and Safe-T mini will work -binaries = [(electrum + "contrib/build-osx/libusb-1.0.dylib", ".")] -binaries += [(electrum + "contrib/build-osx/libsecp256k1.0.dylib", ".")] +binaries = [(electrum + "contrib/osx/libusb-1.0.dylib", ".")] +binaries += [(electrum + "contrib/osx/libsecp256k1.0.dylib", ".")] # Workaround for "Retro Look": binaries += [b for b in collect_dynamic_libs('PyQt5') if 'macstyle' in b[0]] diff --git a/contrib/build-osx/package.sh b/contrib/osx/package.sh similarity index 100% rename from contrib/build-osx/package.sh rename to contrib/osx/package.sh diff --git a/electrum/qrscanner.py b/electrum/qrscanner.py index 5d15aad9a..ab1f1341e 100644 --- a/electrum/qrscanner.py +++ b/electrum/qrscanner.py @@ -73,9 +73,9 @@ def scan_barcode_osx(*args_ignored, **kwargs_ignored): import subprocess # NOTE: This code needs to be modified if the positions of this file changes with respect to the helper app! # This assumes the built macOS .app bundle which ends up putting the helper app in - # .app/contrib/CalinsQRReader/build/Release/CalinsQRReader.app. + # .app/contrib/osx/CalinsQRReader/build/Release/CalinsQRReader.app. root_ec_dir = os.path.abspath(os.path.dirname(__file__) + "/../") - prog = root_ec_dir + "/" + "contrib/CalinsQRReader/build/Release/CalinsQRReader.app/Contents/MacOS/CalinsQRReader" + prog = root_ec_dir + "/" + "contrib/osx/CalinsQRReader/build/Release/CalinsQRReader.app/Contents/MacOS/CalinsQRReader" if not os.path.exists(prog): raise RuntimeError("Cannot start QR scanner; helper app not found.") data = '' From f0a59f06cd25fbe4450f720693597967f1d150ba Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 29 Nov 2018 11:46:34 +0100 Subject: [PATCH 19/40] fix module path --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index c6788ecf0..34cfeafbd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,6 +4,6 @@ [submodule "contrib/deterministic-build/electrum-locale"] path = contrib/deterministic-build/electrum-locale url = https://github.com/spesmilo/electrum-locale -[submodule "contrib/osx/CalinsQRReader"] - path = contrib/osx/CalinsQRReader +[submodule "contrib/CalinsQRReader"] + path = contrib/CalinsQRReader url = https://github.com/spesmilo/CalinsQRReader From f4513c12ebd20fe224af14b1279bdfb520e6bba3 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 29 Nov 2018 11:47:02 +0100 Subject: [PATCH 20/40] follow-up --- .gitmodules | 2 +- contrib/{ => osx}/CalinsQRReader | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename contrib/{ => osx}/CalinsQRReader (100%) diff --git a/.gitmodules b/.gitmodules index 34cfeafbd..f95b1ebce 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,5 +5,5 @@ path = contrib/deterministic-build/electrum-locale url = https://github.com/spesmilo/electrum-locale [submodule "contrib/CalinsQRReader"] - path = contrib/CalinsQRReader + path = contrib/osx/CalinsQRReader url = https://github.com/spesmilo/CalinsQRReader diff --git a/contrib/CalinsQRReader b/contrib/osx/CalinsQRReader similarity index 100% rename from contrib/CalinsQRReader rename to contrib/osx/CalinsQRReader From 124d2e23b7f46863cafb47f2b69b6ba4e07d6791 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 29 Nov 2018 13:24:44 +0100 Subject: [PATCH 21/40] fix travis macOS build --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1c222c4fa..bfcef0231 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ jobs: python: false install: - git fetch --all --tags - script: ./contrib/build-osx/make_osx + script: ./contrib/osx/make_osx after_script: ls -lah dist && md5 dist/* after_success: true - stage: release check From ee287740a7eeed9b40cfdd85d967009bbbb882ee Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 29 Nov 2018 20:28:27 +0100 Subject: [PATCH 22/40] coldcard: fix p2pkh signing for new fw (1.1.0) PSBT was serialised incorrectly but old fw did not complain --- electrum/plugins/coldcard/coldcard.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/electrum/plugins/coldcard/coldcard.py b/electrum/plugins/coldcard/coldcard.py index afbb13843..96d80777f 100644 --- a/electrum/plugins/coldcard/coldcard.py +++ b/electrum/plugins/coldcard/coldcard.py @@ -118,6 +118,8 @@ class CKCCClient: or (self.dev.master_fingerprint != expected_xfp) or (self.dev.master_xpub != expected_xpub)): # probably indicating programing error, not hacking + print_error("[coldcard]", f"xpubs. reported by device: {self.dev.master_xpub}. " + f"stored in file: {expected_xpub}") raise RuntimeError("Expecting 0x%08x but that's not whats connected?!" % expected_xfp) @@ -454,9 +456,12 @@ class Coldcard_KeyStore(Hardware_KeyStore): # inputs section for txin in inputs: - utxo = txin['prev_tx'].outputs()[txin['prevout_n']] - spendable = txin['prev_tx'].serialize_output(utxo) - write_kv(PSBT_IN_WITNESS_UTXO, spendable) + if Transaction.is_segwit_input(txin): + utxo = txin['prev_tx'].outputs()[txin['prevout_n']] + spendable = txin['prev_tx'].serialize_output(utxo) + write_kv(PSBT_IN_WITNESS_UTXO, spendable) + else: + write_kv(PSBT_IN_NON_WITNESS_UTXO, str(txin['prev_tx'])) pubkeys, x_pubkeys = tx.get_sorted_pubkeys(txin) From 863ee984fee7931c94467c92934caf19f2ec1949 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 29 Nov 2018 20:47:26 +0100 Subject: [PATCH 23/40] wallet: cache NaN coin prices, clear cache on new history --- electrum/gui/kivy/main_window.py | 1 + electrum/gui/qt/main_window.py | 1 + electrum/wallet.py | 13 ++++++------- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index 73379bb22..811dda6f6 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -173,6 +173,7 @@ class ElectrumWindow(App): def on_history(self, d): Logger.info("on_history") + self.wallet.clear_coin_price_cache() self._trigger_update_history() def on_fee_histogram(self, *args): diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 11dad7b61..7c3f5e2e0 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -222,6 +222,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError): self.fetch_alias() def on_history(self, b): + self.wallet.clear_coin_price_cache() self.new_fx_history_signal.emit() def setup_exception_hook(self): diff --git a/electrum/wallet.py b/electrum/wallet.py index f3f3efe4c..a14bf4f92 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -182,7 +182,7 @@ class Abstract_Wallet(AddressSynchronizer): self.invoices = InvoiceStore(self.storage) self.contacts = Contacts(self.storage) - self.coin_price_cache = {} + self._coin_price_cache = {} def load_and_cleanup(self): self.load_keystore() @@ -1178,6 +1178,9 @@ class Abstract_Wallet(AddressSynchronizer): total_price += self.coin_price(ser.split(':')[0], price_func, ccy, v) return total_price / (input_value/Decimal(COIN)) + def clear_coin_price_cache(self): + self._coin_price_cache = {} + def coin_price(self, txid, price_func, ccy, txin_value): """ Acquisition price of a coin. @@ -1185,17 +1188,13 @@ class Abstract_Wallet(AddressSynchronizer): """ if txin_value is None: return Decimal('NaN') - # FIXME: this mutual recursion will be really slow and might even reach - # max recursion depth if there are no FX rates available as then - # nothing will be cached. cache_key = "{}:{}:{}".format(str(txid), str(ccy), str(txin_value)) - result = self.coin_price_cache.get(cache_key, None) + result = self._coin_price_cache.get(cache_key, None) if result is not None: return result if self.txi.get(txid, {}) != {}: result = self.average_price(txid, price_func, ccy) * txin_value/Decimal(COIN) - if not result.is_nan(): - self.coin_price_cache[cache_key] = result + self._coin_price_cache[cache_key] = result return result else: fiat_value = self.get_fiat_value(txid, ccy) From bddea809ecb66f310a27edf906f38776bf30e02a Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 30 Nov 2018 04:08:02 +0100 Subject: [PATCH 24/40] storage/blockchain: use os.replace --- electrum/blockchain.py | 6 +----- electrum/storage.py | 7 +------ 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/electrum/blockchain.py b/electrum/blockchain.py index 0f59211e3..018b3adbc 100644 --- a/electrum/blockchain.py +++ b/electrum/blockchain.py @@ -352,11 +352,7 @@ class Blockchain(util.PrintError): self._forkpoint_hash, parent._forkpoint_hash = parent._forkpoint_hash, hash_raw_header(bh2u(parent_data[:HEADER_SIZE])) self._prev_hash, parent._prev_hash = parent._prev_hash, self._prev_hash # parent's new name - try: - os.rename(child_old_name, parent.path()) - except OSError: - os.remove(parent.path()) - os.rename(child_old_name, parent.path()) + os.replace(child_old_name, parent.path()) self.update_size() parent.update_size() # update pointers diff --git a/electrum/storage.py b/electrum/storage.py index ad3de4c6a..d526000e2 100644 --- a/electrum/storage.py +++ b/electrum/storage.py @@ -122,12 +122,7 @@ class JsonDB(PrintError): os.fsync(f.fileno()) mode = os.stat(self.path).st_mode if os.path.exists(self.path) else stat.S_IREAD | stat.S_IWRITE - # perform atomic write on POSIX systems - try: - os.rename(temp_path, self.path) - except OSError: - os.remove(self.path) - os.rename(temp_path, self.path) + os.replace(temp_path, self.path) os.chmod(self.path, mode) self.print_error("saved", self.path) self.modified = False From 86e42a9081f6d0ca27dc93e8e7d6b2625ca8384f Mon Sep 17 00:00:00 2001 From: ThomasV Date: Fri, 30 Nov 2018 11:22:40 +0100 Subject: [PATCH 25/40] release notes for 3.3 --- RELEASE-NOTES | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 9350036e3..43486f914 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -1,3 +1,15 @@ +# Release 3.3 - (Hodler's Edition) + + * The network layer has been rewritten using asyncio. + * Follow blockchain that has the most work, not length. + * New wallet creation defaults to native segwit (bech32). + * RBF batching (option): If the wallet has an unconfirmed RBF + transaction, new payments will be added to that transaction, + instead of creating new transactions. + * OSX: support QR code scanner. + * Android APK: Use API 28, and do not use external storage. + + # Release 3.2.3 - (September 3, 2018) * hardware wallet: the Safe-T mini from Archos is now supported. From 1165d3f330c9be0318a262856af89938a1257641 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Fri, 30 Nov 2018 11:23:01 +0100 Subject: [PATCH 26/40] update version number --- electrum/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/electrum/version.py b/electrum/version.py index 53387af39..5866941f3 100644 --- a/electrum/version.py +++ b/electrum/version.py @@ -1,5 +1,5 @@ -ELECTRUM_VERSION = '3.2.3' # version of the client package -APK_VERSION = '3.2.3.1' # read by buildozer.spec +ELECTRUM_VERSION = '3.3.0' # version of the client package +APK_VERSION = '3.3.0.0' # read by buildozer.spec PROTOCOL_VERSION = '1.4' # protocol version requested From 73e2b09ba82fcc34f2762500b7ab3d7b839b5162 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 30 Nov 2018 16:36:37 +0100 Subject: [PATCH 27/40] blockchain: check best chain on disk is consistent with checkpoints had a corrupted mainnet datadir that had testnet blockchain_headers file (I had probably corrupted it myself but electrum could not recover from it) --- electrum/blockchain.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/electrum/blockchain.py b/electrum/blockchain.py index 018b3adbc..92a587236 100644 --- a/electrum/blockchain.py +++ b/electrum/blockchain.py @@ -86,11 +86,20 @@ blockchains_lock = threading.RLock() def read_blockchains(config: 'SimpleConfig'): - blockchains[constants.net.GENESIS] = Blockchain(config=config, - forkpoint=0, - parent=None, - forkpoint_hash=constants.net.GENESIS, - prev_hash=None) + best_chain = Blockchain(config=config, + forkpoint=0, + parent=None, + forkpoint_hash=constants.net.GENESIS, + prev_hash=None) + blockchains[constants.net.GENESIS] = best_chain + # consistency checks + if best_chain.height() > constants.net.max_checkpoint(): + header_after_cp = best_chain.read_header(constants.net.max_checkpoint()+1) + if not header_after_cp or not best_chain.can_connect(header_after_cp, check_height=False): + util.print_error("[blockchain] deleting best chain. cannot connect header after last cp to last cp.") + os.unlink(best_chain.path()) + best_chain.update_size() + # forks fdir = os.path.join(util.get_headers_dir(config), 'forks') util.make_dir(fdir) # files are named as: fork2_{forkpoint}_{prev_hash}_{first_hash} @@ -98,7 +107,7 @@ def read_blockchains(config: 'SimpleConfig'): l = sorted(l, key=lambda x: int(x.split('_')[1])) # sort by forkpoint def delete_chain(filename, reason): - util.print_error("[blockchain]", reason, filename) + util.print_error(f"[blockchain] deleting chain {filename}: {reason}") os.unlink(os.path.join(fdir, filename)) def instantiate_chain(filename): @@ -222,10 +231,10 @@ class Blockchain(util.PrintError): prev_hash=parent.get_hash(forkpoint-1)) open(self.path(), 'w+').close() self.save_header(header) - # put into global dict + # put into global dict. note that in some cases + # save_header might have already put it there but that's OK chain_id = self.get_id() with blockchains_lock: - assert chain_id not in blockchains, (chain_id, list(blockchains)) blockchains[chain_id] = self return self @@ -392,7 +401,7 @@ class Blockchain(util.PrintError): delta = header.get('block_height') - self.forkpoint data = bfh(serialize_header(header)) # headers are only _appended_ to the end: - assert delta == self.size() + assert delta == self.size(), (delta, self.size()) assert len(data) == HEADER_SIZE self.write(data, delta*HEADER_SIZE) self.swap_with_parent() From ed22f968f980ddc8d3bfcb203adf32f554493bcd Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 30 Nov 2018 17:18:06 +0100 Subject: [PATCH 28/40] text gui: fix network event handler --- electrum/gui/text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electrum/gui/text.py b/electrum/gui/text.py index 7d429ae01..ec5ba5c9b 100644 --- a/electrum/gui/text.py +++ b/electrum/gui/text.py @@ -91,7 +91,7 @@ class ElectrumGui: self.set_cursor(0) return s - def update(self, event): + def update(self, event, *args): self.update_history() if self.tab == 0: self.print_history() From fe6367cbcd91fc10664417feb88ec0ff8d3f74b2 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 30 Nov 2018 18:56:35 +0100 Subject: [PATCH 29/40] network: validate donation address for server --- electrum/network.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/electrum/network.py b/electrum/network.py index 85c8dbee1..9fe78a5e8 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -44,6 +44,7 @@ from .util import PrintError, print_error, log_exceptions, ignore_exceptions, bf from .bitcoin import COIN from . import constants from . import blockchain +from . import bitcoin from .blockchain import Blockchain, HEADER_SIZE from .interface import Interface, serialize_server, deserialize_server, RequestTimedOut from .version import PROTOCOL_VERSION @@ -321,7 +322,11 @@ class Network(PrintError): self.banner = await session.send_request('server.banner') self.notify('banner') async def get_donation_address(): - self.donation_address = await session.send_request('server.donation_address') + addr = await session.send_request('server.donation_address') + if not bitcoin.is_address(addr): + self.print_error(f"invalid donation address from server: {addr}") + addr = '' + self.donation_address = addr async def get_server_peers(): self.server_peers = parse_servers(await session.send_request('server.peers.subscribe')) self.notify('servers') From ec5f406f4904c1b53bdb55584da760141c502cad Mon Sep 17 00:00:00 2001 From: Janus Date: Fri, 30 Nov 2018 19:16:07 +0100 Subject: [PATCH 30/40] plugins: labels: dump response if malformed sync server response --- electrum/plugins/labels/labels.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/electrum/plugins/labels/labels.py b/electrum/plugins/labels/labels.py index 9405e70ed..3c5ff2068 100644 --- a/electrum/plugins/labels/labels.py +++ b/electrum/plugins/labels/labels.py @@ -73,7 +73,10 @@ class LabelsPlugin(BasePlugin): url = 'https://' + self.target_host + url async with make_aiohttp_session(self.proxy) as session: async with session.post(url, json=data) as result: - return await result.json() + try: + return await result.json() + except Exception as e: + raise Exception('Could not decode: ' + await result.text()) from e async def push_thread(self, wallet): wallet_data = self.wallets.get(wallet, None) From 74f6ac27af74b5e2a81bd98e3f883a946720fdef Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 30 Nov 2018 20:45:54 +0100 Subject: [PATCH 31/40] wizard/hw: cap transport string follow-up 32af83b7aede02f0b9bb3e8294ef2dc0481fb6de --- electrum/base_wizard.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index 84323226c..4f3909794 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -283,7 +283,9 @@ class BaseWizard(object): for name, info in devices: state = _("initialized") if info.initialized else _("wiped") label = info.label or _("An unnamed {}").format(name) - descr = f"{label} [{name}, {state}, {info.device.transport_ui_string}]" + try: transport_str = info.device.transport_ui_string[:20] + except: transport_str = 'unknown transport' + descr = f"{label} [{name}, {state}, {transport_str}]" choices.append(((name, info), descr)) msg = _('Select a device') + ':' self.choice_dialog(title=title, message=msg, choices=choices, run_next= lambda *args: self.on_device(*args, purpose=purpose)) From 2f4b9aa1f009dd2e3cdcb571d901e5ae0aa62689 Mon Sep 17 00:00:00 2001 From: Ken <41596906+preserveddarnell@users.noreply.github.com> Date: Sat, 1 Dec 2018 21:28:46 -0500 Subject: [PATCH 32/40] Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 1bb140d86..6dfb69940 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ Qt interface, install the Qt dependencies:: sudo apt-get install python3-pyqt5 If you downloaded the official package (tar.gz), you can run -Electrum from its root directory, without installing it on your +Electrum from its root directory without installing it on your system; all the python dependencies are included in the 'packages' directory. To run Electrum from its root directory, just do:: @@ -44,7 +44,7 @@ You can also install Electrum on your system, by running this command:: python3 -m pip install .[fast] This will download and install the Python dependencies used by -Electrum, instead of using the 'packages' directory. +Electrum instead of using the 'packages' directory. The 'fast' extra contains some optional dependencies that we think are often useful but they are not strictly needed. From d2374d62aad751733999f188495703e3522b01ef Mon Sep 17 00:00:00 2001 From: Calin Culianu Date: Sun, 2 Dec 2018 14:53:44 +0200 Subject: [PATCH 33/40] UI Pet Peeve: Make Coins Tab -> Details pop up a tx dialog that actually includes the tx description as seen in UTXOList (if available) --- electrum/gui/qt/utxo_list.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/electrum/gui/qt/utxo_list.py b/electrum/gui/qt/utxo_list.py index 5985d9c8c..64735de1a 100644 --- a/electrum/gui/qt/utxo_list.py +++ b/electrum/gui/qt/utxo_list.py @@ -71,7 +71,8 @@ class UTXOList(MyTreeWidget): txid = selected[0].split(':')[0] tx = self.wallet.transactions.get(txid) if tx: - menu.addAction(_("Details"), lambda: self.parent.show_transaction(tx)) + label = self.wallet.get_label(txid) + menu.addAction(_("Details"), lambda: self.parent.show_transaction(tx, label)) menu.exec_(self.viewport().mapToGlobal(position)) From 4386799fb0f02b785ed5d5c7a225d31a33eafc08 Mon Sep 17 00:00:00 2001 From: Calin Culianu Date: Sun, 2 Dec 2018 15:20:32 +0200 Subject: [PATCH 34/40] follow-up --- electrum/gui/qt/utxo_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electrum/gui/qt/utxo_list.py b/electrum/gui/qt/utxo_list.py index 64735de1a..0a6dc3349 100644 --- a/electrum/gui/qt/utxo_list.py +++ b/electrum/gui/qt/utxo_list.py @@ -71,7 +71,7 @@ class UTXOList(MyTreeWidget): txid = selected[0].split(':')[0] tx = self.wallet.transactions.get(txid) if tx: - label = self.wallet.get_label(txid) + label = self.wallet.get_label(txid) or None # Prefer None if empty (None hides the Description: field in the window) menu.addAction(_("Details"), lambda: self.parent.show_transaction(tx, label)) menu.exec_(self.viewport().mapToGlobal(position)) From ff454ab29dd374fb2998ef748ab3969e13e0f172 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 3 Dec 2018 12:46:12 +0100 Subject: [PATCH 35/40] cli restore: fix imported privkeys with password closes #4894 --- electrum/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electrum/commands.py b/electrum/commands.py index 40a8142cf..3acf79528 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -176,7 +176,7 @@ class Commands: storage.put('keystore', k.dump()) wallet = Imported_Wallet(storage) keys = keystore.get_private_keys(text) - good_inputs, bad_inputs = wallet.import_private_keys(keys, password) + good_inputs, bad_inputs = wallet.import_private_keys(keys, None) # FIXME tell user about bad_inputs if not good_inputs: raise Exception("None of the given privkeys can be imported") From 9350709f13bc7e3d79b8e0f1515a3fdba4f2cbff Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 3 Dec 2018 13:02:14 +0100 Subject: [PATCH 36/40] wallet creation: take care not to write plaintext keys to disk when creating imported privkey wallets the privkeys were written to disk unencrypted first, then overwritten with ciphertext --- electrum/base_wizard.py | 3 ++- electrum/commands.py | 3 ++- electrum/wallet.py | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index 4f3909794..7efd82297 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -200,7 +200,7 @@ class BaseWizard(object): self.storage.put('keystore', k.dump()) w = Imported_Wallet(self.storage) keys = keystore.get_private_keys(text) - good_inputs, bad_inputs = w.import_private_keys(keys, None) + good_inputs, bad_inputs = w.import_private_keys(keys, None, write_to_disk=False) self.keystores.append(w.keystore) else: return self.terminate() @@ -510,6 +510,7 @@ class BaseWizard(object): def on_password(self, password, *, encrypt_storage, storage_enc_version=STO_EV_USER_PW, encrypt_keystore): + assert not self.storage.file_exists(), "file was created too soon! plaintext keys might have been written to disk" self.storage.set_keystore_encryption(bool(password) and encrypt_keystore) if encrypt_storage: self.storage.set_password(password, enc_version=storage_enc_version) diff --git a/electrum/commands.py b/electrum/commands.py index 3acf79528..2192d992a 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -176,7 +176,7 @@ class Commands: storage.put('keystore', k.dump()) wallet = Imported_Wallet(storage) keys = keystore.get_private_keys(text) - good_inputs, bad_inputs = wallet.import_private_keys(keys, None) + good_inputs, bad_inputs = wallet.import_private_keys(keys, None, write_to_disk=False) # FIXME tell user about bad_inputs if not good_inputs: raise Exception("None of the given privkeys can be imported") @@ -191,6 +191,7 @@ class Commands: storage.put('wallet_type', 'standard') wallet = Wallet(storage) + assert not storage.file_exists(), "file was created too soon! plaintext keys might have been written to disk" wallet.update_password(old_pw=None, new_pw=password, encrypt_storage=encrypt_file) wallet.synchronize() diff --git a/electrum/wallet.py b/electrum/wallet.py index a14bf4f92..b38f441da 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -1379,8 +1379,8 @@ class Imported_Wallet(Simple_Wallet): def get_public_key(self, address): return self.addresses[address].get('pubkey') - def import_private_keys(self, keys: List[str], password: Optional[str]) -> Tuple[List[str], - List[Tuple[str, str]]]: + def import_private_keys(self, keys: List[str], password: Optional[str], + write_to_disk=True) -> Tuple[List[str], List[Tuple[str, str]]]: good_addr = [] # type: List[str] bad_keys = [] # type: List[Tuple[str, str]] for key in keys: @@ -1398,7 +1398,7 @@ class Imported_Wallet(Simple_Wallet): self.add_address(addr) self.save_keystore() self.save_addresses() - self.save_transactions(write=True) + self.save_transactions(write=write_to_disk) return good_addr, bad_keys def import_private_key(self, key: str, password: Optional[str]) -> str: From 5473320ce459b3076d60f71dab490ed3a07b86a5 Mon Sep 17 00:00:00 2001 From: Janus Date: Tue, 27 Nov 2018 21:32:55 +0100 Subject: [PATCH 37/40] qt: use QStandardItemModel --- electrum/contacts.py | 3 +- electrum/gui/qt/address_list.py | 82 +++--- electrum/gui/qt/contact_list.py | 80 +++--- electrum/gui/qt/history_list.py | 408 ++++++++++++++++++++---------- electrum/gui/qt/invoice_list.py | 28 +- electrum/gui/qt/main_window.py | 10 +- electrum/gui/qt/network_dialog.py | 3 +- electrum/gui/qt/request_list.py | 68 ++--- electrum/gui/qt/util.py | 232 +++++++++-------- electrum/gui/qt/utxo_list.py | 61 +++-- 10 files changed, 572 insertions(+), 403 deletions(-) diff --git a/electrum/contacts.py b/electrum/contacts.py index c09b59e22..49c8087f9 100644 --- a/electrum/contacts.py +++ b/electrum/contacts.py @@ -65,8 +65,9 @@ class Contacts(dict): def pop(self, key): if key in self.keys(): - dict.pop(self, key) + res = dict.pop(self, key) self.save() + return res def resolve(self, k): if bitcoin.is_address(k): diff --git a/electrum/gui/qt/address_list.py b/electrum/gui/qt/address_list.py index fce1004c2..019588102 100644 --- a/electrum/gui/qt/address_list.py +++ b/electrum/gui/qt/address_list.py @@ -31,13 +31,11 @@ from electrum.bitcoin import is_address from .util import * - -class AddressList(MyTreeWidget): +class AddressList(MyTreeView): filter_columns = [0, 1, 2, 3] # Type, Address, Label, Balance def __init__(self, parent=None): - MyTreeWidget.__init__(self, parent, self.create_menu, [], 2) - self.refresh_headers() + super().__init__(parent, self.create_menu, 2) self.setSelectionMode(QAbstractItemView.ExtendedSelection) self.setSortingEnabled(True) self.show_change = 0 @@ -50,6 +48,8 @@ class AddressList(MyTreeWidget): self.used_button.currentIndexChanged.connect(self.toggle_used) for t in [_('All'), _('Unused'), _('Funded'), _('Used')]: self.used_button.addItem(t) + self.setModel(QStandardItemModel(self)) + self.update() def get_toolbar_buttons(self): return QLabel(_("Filter:")), self.change_button, self.used_button @@ -82,18 +82,19 @@ class AddressList(MyTreeWidget): self.show_used = state self.update() - def on_update(self): + def update(self): self.wallet = self.parent.wallet - item = self.currentItem() - current_address = item.data(0, Qt.UserRole) if item else None + current_address = self.current_item_user_role(col=2) if self.show_change == 1: addr_list = self.wallet.get_receiving_addresses() elif self.show_change == 2: addr_list = self.wallet.get_change_addresses() else: addr_list = self.wallet.get_addresses() - self.clear() + self.model().clear() + self.refresh_headers() fx = self.parent.fx + set_address = None for address in addr_list: num = self.wallet.get_address_history_len(address) label = self.wallet.labels.get(address, '') @@ -111,61 +112,66 @@ class AddressList(MyTreeWidget): if fx and fx.get_fiat_address_config(): rate = fx.exchange_rate() fiat_balance = fx.value_str(balance, rate) - address_item = SortableTreeWidgetItem(['', address, label, balance_text, fiat_balance, "%d"%num]) + labels = ['', address, label, balance_text, fiat_balance, "%d"%num] + address_item = [QStandardItem(e) for e in labels] else: - address_item = SortableTreeWidgetItem(['', address, label, balance_text, "%d"%num]) + labels = ['', address, label, balance_text, "%d"%num] + address_item = [QStandardItem(e) for e in labels] # align text and set fonts - for i in range(address_item.columnCount()): - address_item.setTextAlignment(i, Qt.AlignVCenter) + for i, item in enumerate(address_item): + item.setTextAlignment(Qt.AlignVCenter) if i not in (0, 2): - address_item.setFont(i, QFont(MONOSPACE_FONT)) + item.setFont(QFont(MONOSPACE_FONT)) + item.setEditable(i in self.editable_columns) if fx and fx.get_fiat_address_config(): - address_item.setTextAlignment(4, Qt.AlignRight | Qt.AlignVCenter) + address_item[4].setTextAlignment(Qt.AlignRight | Qt.AlignVCenter) # setup column 0 if self.wallet.is_change(address): - address_item.setText(0, _('change')) - address_item.setBackground(0, ColorScheme.YELLOW.as_color(True)) + address_item[0].setText(_('change')) + address_item[0].setBackground(ColorScheme.YELLOW.as_color(True)) else: - address_item.setText(0, _('receiving')) - address_item.setBackground(0, ColorScheme.GREEN.as_color(True)) - address_item.setData(0, Qt.UserRole, address) # column 0; independent from address column + address_item[0].setText(_('receiving')) + address_item[0].setBackground(ColorScheme.GREEN.as_color(True)) + address_item[2].setData(address, Qt.UserRole) # setup column 1 if self.wallet.is_frozen(address): - address_item.setBackground(1, ColorScheme.BLUE.as_color(True)) + address_item[1].setBackground(ColorScheme.BLUE.as_color(True)) if self.wallet.is_beyond_limit(address): - address_item.setBackground(1, ColorScheme.RED.as_color(True)) + address_item[1].setBackground(ColorScheme.RED.as_color(True)) # add item - self.addChild(address_item) + count = self.model().rowCount() + self.model().insertRow(count, address_item) + address_idx = self.model().index(count, 2) if address == current_address: - self.setCurrentItem(address_item) + set_address = QPersistentModelIndex(address_idx) + self.set_current_idx(set_address) def create_menu(self, position): from electrum.wallet import Multisig_Wallet is_multisig = isinstance(self.wallet, Multisig_Wallet) can_delete = self.wallet.can_delete_address() - selected = self.selectedItems() + selected = self.selected_in_column(1) multi_select = len(selected) > 1 - addrs = [item.text(1) for item in selected] - if not addrs: - return + addrs = [self.model().itemFromIndex(item).text() for item in selected] if not multi_select: - item = self.itemAt(position) - col = self.currentColumn() + idx = self.indexAt(position) + col = idx.column() + item = self.model().itemFromIndex(idx) if not item: return addr = addrs[0] - if not is_address(addr): - item.setExpanded(not item.isExpanded()) - return menu = QMenu() if not multi_select: - column_title = self.headerItem().text(col) - copy_text = item.text(col) + addr_column_title = self.model().horizontalHeaderItem(2).text() + addr_idx = idx.sibling(idx.row(), 2) + + column_title = self.model().horizontalHeaderItem(col).text() + copy_text = self.model().itemFromIndex(idx).text() menu.addAction(_("Copy {}").format(column_title), lambda: self.parent.app.clipboard().setText(copy_text)) menu.addAction(_('Details'), lambda: self.parent.show_address(addr)) - if col in self.editable_columns: - menu.addAction(_("Edit {}").format(column_title), lambda: self.editItem(item, col)) + persistent = QPersistentModelIndex(addr_idx) + menu.addAction(_("Edit {}").format(addr_column_title), lambda p=persistent: self.edit(QModelIndex(p))) menu.addAction(_("Request payment"), lambda: self.parent.receive_at(addr)) if self.wallet.can_export(): menu.addAction(_("Private key"), lambda: self.parent.show_private_key(addr)) @@ -189,7 +195,3 @@ class AddressList(MyTreeWidget): run_hook('receive_menu', menu, addrs, self.wallet) menu.exec_(self.viewport().mapToGlobal(position)) - - def on_permit_edit(self, item, column): - # labels for headings, e.g. "receiving" or "used" should not be editable - return item.childCount() == 0 diff --git a/electrum/gui/qt/contact_list.py b/electrum/gui/qt/contact_list.py index d85c6df5c..e1915b1b6 100644 --- a/electrum/gui/qt/contact_list.py +++ b/electrum/gui/qt/contact_list.py @@ -34,67 +34,81 @@ from electrum.bitcoin import is_address from electrum.util import block_explorer_URL from electrum.plugin import run_hook -from .util import MyTreeWidget, import_meta_gui, export_meta_gui +from .util import MyTreeView, import_meta_gui, export_meta_gui -class ContactList(MyTreeWidget): +class ContactList(MyTreeView): filter_columns = [0, 1] # Key, Value def __init__(self, parent): - MyTreeWidget.__init__(self, parent, self.create_menu, [_('Name'), _('Address')], 0, [0]) + super().__init__(parent, self.create_menu, stretch_column=0, editable_columns=[0]) + self.setModel(QStandardItemModel(self)) self.setSelectionMode(QAbstractItemView.ExtendedSelection) self.setSortingEnabled(True) + self.update() - def on_permit_edit(self, item, column): - # openalias items shouldn't be editable - return item.text(1) != "openalias" + def on_edited(self, idx, user_role, text): + _type, prior_name = self.parent.contacts.pop(user_role) - def on_edited(self, item, column, prior): - if column == 0: # Remove old contact if renamed - self.parent.contacts.pop(prior) - self.parent.set_contact(item.text(0), item.text(1)) + # TODO when min Qt >= 5.11, use siblingAtColumn + col_1_sibling = idx.sibling(idx.row(), 1) + col_1_item = self.model().itemFromIndex(col_1_sibling) + + self.parent.set_contact(text, col_1_item.text()) def import_contacts(self): - import_meta_gui(self.parent, _('contacts'), self.parent.contacts.import_file, self.on_update) + import_meta_gui(self.parent, _('contacts'), self.parent.contacts.import_file, self.update) def export_contacts(self): export_meta_gui(self.parent, _('contacts'), self.parent.contacts.export_file) def create_menu(self, position): menu = QMenu() - selected = self.selectedItems() - if not selected: + selected = self.selected_in_column(0) + selected_keys = [] + for idx in selected: + sel_key = self.model().itemFromIndex(idx).data(Qt.UserRole) + selected_keys.append(sel_key) + idx = self.indexAt(position) + if not selected or not idx.isValid(): menu.addAction(_("New contact"), lambda: self.parent.new_contact_dialog()) menu.addAction(_("Import file"), lambda: self.import_contacts()) menu.addAction(_("Export file"), lambda: self.export_contacts()) else: - names = [item.text(0) for item in selected] - keys = [item.text(1) for item in selected] - column = self.currentColumn() - column_title = self.headerItem().text(column) - column_data = '\n'.join([item.text(column) for item in selected]) + column = idx.column() + column_title = self.model().horizontalHeaderItem(column).text() + column_data = '\n'.join(self.model().itemFromIndex(idx).text() for idx in selected) menu.addAction(_("Copy {}").format(column_title), lambda: self.parent.app.clipboard().setText(column_data)) if column in self.editable_columns: - item = self.currentItem() - menu.addAction(_("Edit {}").format(column_title), lambda: self.editItem(item, column)) - menu.addAction(_("Pay to"), lambda: self.parent.payto_contacts(keys)) - menu.addAction(_("Delete"), lambda: self.parent.delete_contacts(keys)) - URLs = [block_explorer_URL(self.config, 'addr', key) for key in filter(is_address, keys)] + item = self.model().itemFromIndex(idx) + if item.isEditable(): + # would not be editable if openalias + persistent = QPersistentModelIndex(idx) + menu.addAction(_("Edit {}").format(column_title), lambda p=persistent: self.edit(QModelIndex(p))) + menu.addAction(_("Pay to"), lambda: self.parent.payto_contacts(selected_keys)) + menu.addAction(_("Delete"), lambda: self.parent.delete_contacts(selected_keys)) + URLs = [block_explorer_URL(self.config, 'addr', key) for key in filter(is_address, selected_keys)] if URLs: menu.addAction(_("View on block explorer"), lambda: map(webbrowser.open, URLs)) - run_hook('create_contact_menu', menu, selected) + run_hook('create_contact_menu', menu, selected_keys) menu.exec_(self.viewport().mapToGlobal(position)) - def on_update(self): - item = self.currentItem() - current_key = item.data(0, Qt.UserRole) if item else None - self.clear() + def update(self): + current_key = self.current_item_user_role(col=0) + self.model().clear() + self.update_headers([_('Name'), _('Address')]) + set_current = None for key in sorted(self.parent.contacts.keys()): - _type, name = self.parent.contacts[key] - item = QTreeWidgetItem([name, key]) - item.setData(0, Qt.UserRole, key) - self.addTopLevelItem(item) + contact_type, name = self.parent.contacts[key] + items = [QStandardItem(x) for x in (name, key)] + items[0].setEditable(contact_type != 'openalias') + items[1].setEditable(False) + items[0].setData(key, Qt.UserRole) + row_count = self.model().rowCount() + self.model().insertRow(row_count, items) if key == current_key: - self.setCurrentItem(item) + idx = self.model().index(row_count, 0) + set_current = QPersistentModelIndex(idx) + self.set_current_idx(set_current) run_hook('update_contacts_tab', self) diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index 9d05ac74f..807484f9b 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -27,10 +27,11 @@ import webbrowser import datetime from datetime import date from typing import TYPE_CHECKING +from collections import OrderedDict from electrum.address_synchronizer import TX_HEIGHT_LOCAL from electrum.i18n import _ -from electrum.util import block_explorer_URL, profiler, print_error, TxMinedStatus +from electrum.util import block_explorer_URL, profiler, print_error, TxMinedStatus, Fiat from .util import * @@ -57,40 +58,111 @@ TX_ICONS = [ "confirmed.png", ] - -class HistoryList(MyTreeWidget, AcceptFileDragDrop): - filter_columns = [2, 3, 4] # Date, Description, Amount +class HistorySortModel(QSortFilterProxyModel): + def lessThan(self, source_left: QModelIndex, source_right: QModelIndex): + item1 = self.sourceModel().itemFromIndex(source_left) + item2 = self.sourceModel().itemFromIndex(source_right) + data1 = item1.data(HistoryList.SORT_ROLE) + data2 = item2.data(HistoryList.SORT_ROLE) + if data1 is not None and data2 is not None: + return data1 < data2 + return item1.text() < item2.text() + +class HistoryList(MyTreeView, AcceptFileDragDrop): + filter_columns = [1, 2, 3] # Date, Description, Amount TX_HASH_ROLE = Qt.UserRole - TX_VALUE_ROLE = Qt.UserRole + 1 + SORT_ROLE = Qt.UserRole + 1 + + def should_hide(self, proxy_row): + if self.start_timestamp and self.end_timestamp: + source_idx = self.proxy.mapToSource(self.proxy.index(proxy_row, 0)) + item = self.std_model.itemFromIndex(source_idx) + txid = item.data(self.TX_HASH_ROLE) + date = self.transactions[txid]['date'] + if date: + in_interval = self.start_timestamp <= date <= self.end_timestamp + if not in_interval: + return True + return False def __init__(self, parent=None): - MyTreeWidget.__init__(self, parent, self.create_menu, [], 3) + super().__init__(parent, self.create_menu, 2) + self.std_model = QStandardItemModel(self) + self.proxy = HistorySortModel(self) + self.proxy.setSourceModel(self.std_model) + self.setModel(self.proxy) + + self.txid_to_items = {} + self.transactions = OrderedDict() + self.summary = {} + self.blue_brush = QBrush(QColor("#1E1EFF")) + self.red_brush = QBrush(QColor("#BC1E1E")) + self.monospace_font = QFont(MONOSPACE_FONT) + self.default_color = self.parent.app.palette().text().color() + self.config = parent.config AcceptFileDragDrop.__init__(self, ".txn") - self.refresh_headers() - self.setColumnHidden(1, True) self.setSortingEnabled(True) - self.sortByColumn(0, Qt.AscendingOrder) self.start_timestamp = None self.end_timestamp = None self.years = [] self.create_toolbar_buttons() self.wallet = None + root = self.std_model.invisibleRootItem() + + self.wallet = self.parent.wallet # type: Abstract_Wallet + fx = self.parent.fx + r = self.wallet.get_full_history(domain=self.get_domain(), from_timestamp=None, to_timestamp=None, fx=fx) + self.transactions.update([(x['txid'], x) for x in r['transactions']]) + self.summary = r['summary'] + if not self.years and self.transactions: + start_date = next(iter(self.transactions.values())).get('date') or date.today() + end_date = next(iter(reversed(self.transactions.values()))).get('date') or date.today() + self.years = [str(i) for i in range(start_date.year, end_date.year + 1)] + self.period_combo.insertItems(1, self.years) + if fx: fx.history_used_spot = False + self.refresh_headers() + for tx_item in self.transactions.values(): + self.insert_tx(tx_item) + self.sortByColumn(0, Qt.AscendingOrder) + + #def on_activated(self, idx: QModelIndex): + # # TODO use siblingAtColumn when min Qt version is >=5.11 + # self.edit(idx.sibling(idx.row(), 2)) + def format_date(self, d): return str(datetime.date(d.year, d.month, d.day)) if d else _('None') def refresh_headers(self): - headers = ['', '', _('Date'), _('Description'), _('Amount'), _('Balance')] + headers = ['', _('Date'), _('Description'), _('Amount'), _('Balance')] fx = self.parent.fx if fx and fx.show_history(): headers.extend(['%s '%fx.ccy + _('Value')]) - self.editable_columns |= {6} + self.editable_columns |= {5} if fx.get_history_capital_gains_config(): headers.extend(['%s '%fx.ccy + _('Acquisition price')]) headers.extend(['%s '%fx.ccy + _('Capital Gains')]) else: - self.editable_columns -= {6} - self.update_headers(headers) + self.editable_columns -= {5} + col_count = self.std_model.columnCount() + diff = col_count-len(headers) + grew = False + if col_count > len(headers): + if diff == 2: + self.std_model.removeColumns(6, diff) + else: + assert diff in [1, 3] + self.std_model.removeColumns(5, diff) + for items in self.txid_to_items.values(): + while len(items) > col_count: + items.pop() + elif col_count < len(headers): + grew = True + self.std_model.clear() + self.txid_to_items.clear() + self.transactions.clear() + self.summary.clear() + self.update_headers(headers, self.std_model) def get_domain(self): '''Replaced in address_dialog.py''' @@ -111,13 +183,11 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): year = int(s) except: return - start_date = datetime.datetime(year, 1, 1) - end_date = datetime.datetime(year+1, 1, 1) - self.start_timestamp = time.mktime(start_date.timetuple()) - self.end_timestamp = time.mktime(end_date.timetuple()) + self.start_timestamp = start_date = datetime.datetime(year, 1, 1) + self.end_timestamp = end_date = datetime.datetime(year+1, 1, 1) self.start_button.setText(_('From') + ' ' + self.format_date(start_date)) self.end_button.setText(_('To') + ' ' + self.format_date(end_date)) - self.update() + self.hide_rows() def create_toolbar_buttons(self): self.period_combo = QComboBox() @@ -136,18 +206,18 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): def on_hide_toolbar(self): self.start_timestamp = None self.end_timestamp = None - self.update() + self.hide_rows() def save_toolbar_state(self, state, config): config.set_key('show_toolbar_history', state) def select_start_date(self): self.start_timestamp = self.select_date(self.start_button) - self.update() + self.hide_rows() def select_end_date(self): self.end_timestamp = self.select_date(self.end_button) - self.update() + self.hide_rows() def select_date(self, button): d = WindowModalDialog(self, _("Select date")) @@ -167,7 +237,7 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): return None date = d.date.toPyDate() button.setText(self.format_date(date)) - return time.mktime(date.timetuple()) + return datetime.datetime(date.year, date.month, date.day) def show_summary(self): h = self.summary @@ -215,104 +285,167 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): _("Perhaps some dependencies are missing...") + " (matplotlib?)") return try: - plt = plot_history(self.transactions) + plt = plot_history(list(self.transactions.values())) plt.show() except NothingToPlotException as e: self.parent.show_message(str(e)) + def insert_tx(self, tx_item): + fx = self.parent.fx + tx_hash = tx_item['txid'] + height = tx_item['height'] + conf = tx_item['confirmations'] + timestamp = tx_item['timestamp'] + value = tx_item['value'].value + balance = tx_item['balance'].value + label = tx_item['label'] + tx_mined_status = TxMinedStatus(height, conf, timestamp, None) + status, status_str = self.wallet.get_tx_status(tx_hash, tx_mined_status) + has_invoice = self.wallet.invoices.paid.get(tx_hash) + icon = self.icon_cache.get(":icons/" + TX_ICONS[status]) + v_str = self.parent.format_amount(value, is_diff=True, whitespaces=True) + balance_str = self.parent.format_amount(balance, whitespaces=True) + entry = ['', status_str, label, v_str, balance_str] + fiat_value = None + item = [QStandardItem(e) for e in entry] + item[3].setData(value, self.SORT_ROLE) + item[4].setData(balance, self.SORT_ROLE) + if has_invoice: + item[2].setIcon(self.icon_cache.get(":icons/seal")) + for i in range(len(entry)): + self.set_item_properties(item[i], i, tx_hash) + if value and value < 0: + item[2].setForeground(self.red_brush) + item[3].setForeground(self.red_brush) + self.txid_to_items[tx_hash] = item + self.update_item(tx_hash, self.parent.wallet.get_tx_height(tx_hash)) + source_row_idx = self.std_model.rowCount() + self.std_model.insertRow(source_row_idx, item) + new_idx = self.std_model.index(source_row_idx, 0) + history = self.parent.fx.show_history() + if history: + self.update_fiat(tx_hash, tx_item) + self.hide_row(self.proxy.mapFromSource(new_idx).row()) + + def set_item_properties(self, item, i, tx_hash): + if i>2: + item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter) + if i!=1: + item.setFont(self.monospace_font) + item.setEditable(i in self.editable_columns) + item.setData(tx_hash, self.TX_HASH_ROLE) + + def ensure_fields_available(self, items, idx, txid): + while len(items) < idx + 1: + row = list(self.transactions.keys()).index(txid) + qidx = self.std_model.index(row, len(items)) + assert qidx.isValid(), (self.std_model.columnCount(), idx) + item = self.std_model.itemFromIndex(qidx) + self.set_item_properties(item, len(items), txid) + items.append(item) + @profiler - def on_update(self): + def update(self): self.wallet = self.parent.wallet # type: Abstract_Wallet fx = self.parent.fx - r = self.wallet.get_full_history(domain=self.get_domain(), from_timestamp=self.start_timestamp, to_timestamp=self.end_timestamp, fx=fx) - self.transactions = r['transactions'] - self.summary = r['summary'] - if not self.years and self.transactions: - start_date = self.transactions[0].get('date') or date.today() - end_date = self.transactions[-1].get('date') or date.today() - self.years = [str(i) for i in range(start_date.year, end_date.year + 1)] - self.period_combo.insertItems(1, self.years) - item = self.currentItem() - current_tx = item.data(0, self.TX_HASH_ROLE) if item else None - self.clear() - if fx: fx.history_used_spot = False - blue_brush = QBrush(QColor("#1E1EFF")) - red_brush = QBrush(QColor("#BC1E1E")) - monospace_font = QFont(MONOSPACE_FONT) - for tx_item in self.transactions: - tx_hash = tx_item['txid'] - height = tx_item['height'] - conf = tx_item['confirmations'] - timestamp = tx_item['timestamp'] - value_sat = tx_item['value'].value - balance = tx_item['balance'].value - label = tx_item['label'] - tx_mined_status = TxMinedStatus(height, conf, timestamp, None) - status, status_str = self.wallet.get_tx_status(tx_hash, tx_mined_status) - has_invoice = self.wallet.invoices.paid.get(tx_hash) - icon = self.icon_cache.get(":icons/" + TX_ICONS[status]) - v_str = self.parent.format_amount(value_sat, is_diff=True, whitespaces=True) - balance_str = self.parent.format_amount(balance, whitespaces=True) - entry = ['', tx_hash, status_str, label, v_str, balance_str] - fiat_value = None - if value_sat is not None and fx and fx.show_history(): - fiat_value = tx_item['fiat_value'].value - value_str = fx.format_fiat(fiat_value) - entry.append(value_str) - # fixme: should use is_mine - if value_sat < 0: - entry.append(fx.format_fiat(tx_item['acquisition_price'].value)) - entry.append(fx.format_fiat(tx_item['capital_gain'].value)) - item = SortableTreeWidgetItem(entry) - item.setIcon(0, icon) - item.setToolTip(0, str(conf) + " confirmation" + ("s" if conf != 1 else "")) - if has_invoice: - item.setIcon(3, self.icon_cache.get(":icons/seal")) - for i in range(len(entry)): - if i>3: - item.setTextAlignment(i, Qt.AlignRight | Qt.AlignVCenter) - if i!=2: - item.setFont(i, monospace_font) - if value_sat and value_sat < 0: - item.setForeground(3, red_brush) - item.setForeground(4, red_brush) - if fiat_value is not None and not tx_item['fiat_default']: - item.setForeground(6, blue_brush) - # sort orders - item.setData(0, SortableTreeWidgetItem.DataRole, (status, conf)) - item.setData(4, SortableTreeWidgetItem.DataRole, value_sat) - item.setData(5, SortableTreeWidgetItem.DataRole, balance) - if fiat_value is not None: - item.setData(6, SortableTreeWidgetItem.DataRole, fiat_value) - if value_sat < 0: - item.setData(7, SortableTreeWidgetItem.DataRole, tx_item['acquisition_price'].value) - item.setData(8, SortableTreeWidgetItem.DataRole, tx_item['capital_gain'].value) - if tx_hash: - item.setData(0, self.TX_HASH_ROLE, tx_hash) - item.setData(0, self.TX_VALUE_ROLE, value_sat) - self.insertTopLevelItem(0, item) - if current_tx == tx_hash: - self.setCurrentItem(item) - - def on_edited(self, item, column, prior): - '''Called only when the text actually changes''' - key = item.data(0, self.TX_HASH_ROLE) - value_sat = item.data(0, self.TX_VALUE_ROLE) - text = item.text(column) + r = self.wallet.get_full_history(domain=self.get_domain(), from_timestamp=None, to_timestamp=None, fx=fx) + seen = set() + history = fx.show_history() + tx_list = list(self.transactions.values()) + if r['transactions'] == tx_list: + return + if r['transactions'][:-1] == tx_list: + print_error('history_list: one new transaction') + row = r['transactions'][-1] + txid = row['txid'] + if txid not in self.transactions: + self.transactions[txid] = row + self.transactions.move_to_end(txid, last=True) + self.insert_tx(row) + return + else: + print_error('history_list: tx added but txid is already in list (weird), txid: ', txid) + for idx, row in enumerate(r['transactions']): + txid = row['txid'] + seen.add(txid) + if txid not in self.transactions: + self.transactions[txid] = row + self.transactions.move_to_end(txid, last=True) + self.insert_tx(row) + continue + old = self.transactions[txid] + if old == row: + continue + self.update_item(txid, self.parent.wallet.get_tx_height(txid)) + if history: + self.update_fiat(txid, row) + balance_str = self.parent.format_amount(row['balance'].value, whitespaces=True) + self.txid_to_items[txid][4].setText(balance_str) + self.txid_to_items[txid][4].setData(row['balance'].value, self.SORT_ROLE) + old.clear() + old.update(**row) + removed = 0 + l = list(enumerate(self.transactions.keys())) + for idx, txid in l: + if txid not in seen: + del self.transactions[txid] + del self.txid_to_items[txid] + items = self.std_model.takeRow(idx - removed) + removed_txid = items[0].data(self.TX_HASH_ROLE) + assert removed_txid == txid, (idx, removed) + removed += 1 + self.apply_filter() + + def update_fiat(self, txid, row): + cap_gains = self.parent.fx.get_history_capital_gains_config() + items = self.txid_to_items[txid] + self.ensure_fields_available(items, 7 if cap_gains else 5, txid) + items[5].setForeground(self.blue_brush if not row['fiat_default'] and row['fiat_value'] else self.default_color) + value_str = self.parent.fx.format_fiat(row['fiat_value'].value) + items[5].setText(value_str) + items[5].setData(row['fiat_value'].value, self.SORT_ROLE) + # fixme: should use is_mine + if row['value'].value < 0 and cap_gains: + acq = row['acquisition_price'].value + items[6].setText(self.parent.fx.format_fiat(acq)) + items[6].setData(acq, self.SORT_ROLE) + cg = row['capital_gain'].value + items[7].setText(self.parent.fx.format_fiat(cg)) + items[7].setData(cg, self.SORT_ROLE) + + def update_on_new_fee_histogram(self): + pass + # TODO update unconfirmed tx'es + + def on_edited(self, index, user_role, text): + column = index.column() + index = self.proxy.mapToSource(index) + item = self.std_model.itemFromIndex(index) + key = item.data(self.TX_HASH_ROLE) # fixme - if column == 3: + if column == 2: self.parent.wallet.set_label(key, text) self.update_labels() self.parent.update_completions() - elif column == 6: - self.parent.wallet.set_fiat_value(key, self.parent.fx.ccy, text, self.parent.fx, value_sat) - self.on_update() - - def on_doubleclick(self, item, column): - if self.permit_edit(item, column): - super(HistoryList, self).on_doubleclick(item, column) + elif column == 5: + tx_item = self.transactions[key] + self.parent.wallet.set_fiat_value(key, self.parent.fx.ccy, text, self.parent.fx, tx_item['value'].value) + value = tx_item['value'].value + if value is not None: + fee = tx_item['fee'] + fiat_fields = self.parent.wallet.get_tx_item_fiat(key, value, self.parent.fx, fee.value if fee else None) + tx_item.update(fiat_fields) + self.update_fiat(key, tx_item) else: - tx_hash = item.data(0, self.TX_HASH_ROLE) + assert False + + def mouseDoubleClickEvent(self, event: QMouseEvent): + idx = self.indexAt(event.pos()) + item = self.std_model.itemFromIndex(self.proxy.mapToSource(idx)) + if not item or item.isEditable(): + super().mouseDoubleClickEvent(event) + elif item: + tx_hash = item.data(self.TX_HASH_ROLE) self.show_transaction(tx_hash) def show_transaction(self, tx_hash): @@ -323,13 +456,13 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): self.parent.show_transaction(tx, label) def update_labels(self): - root = self.invisibleRootItem() - child_count = root.childCount() + root = self.std_model.invisibleRootItem() + child_count = root.rowCount() for i in range(child_count): - item = root.child(i) - txid = item.data(0, self.TX_HASH_ROLE) + item = root.child(i, 2) + txid = item.data(self.TX_HASH_ROLE) label = self.wallet.get_label(txid) - item.setText(3, label) + item.setText(label) def update_item(self, tx_hash, tx_mined_status): if self.wallet is None: @@ -337,31 +470,30 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): conf = tx_mined_status.conf status, status_str = self.wallet.get_tx_status(tx_hash, tx_mined_status) icon = self.icon_cache.get(":icons/" + TX_ICONS[status]) - items = self.findItems(tx_hash, Qt.MatchExactly, column=1) - if items: - item = items[0] - item.setIcon(0, icon) - item.setData(0, SortableTreeWidgetItem.DataRole, (status, conf)) - item.setText(2, status_str) - - def create_menu(self, position): - self.selectedIndexes() - item = self.currentItem() - if not item: - return - column = self.currentColumn() - tx_hash = item.data(0, self.TX_HASH_ROLE) - if not tx_hash: + if tx_hash not in self.txid_to_items: return + items = self.txid_to_items[tx_hash] + items[0].setIcon(icon) + items[0].setToolTip(str(conf) + _(" confirmation" + ("s" if conf != 1 else ""))) + items[0].setData((status, conf), self.SORT_ROLE) + items[1].setText(status_str) + + def create_menu(self, position: QPoint): + org_idx: QModelIndex = self.indexAt(position) + idx = self.proxy.mapToSource(org_idx) + item: QStandardItem = self.std_model.itemFromIndex(idx) + assert item, 'create_menu: index not found in model' + tx_hash = idx.data(self.TX_HASH_ROLE) + column = idx.column() + assert tx_hash, "create_menu: no tx hash" tx = self.wallet.transactions.get(tx_hash) - if not tx: - return - if column is 0: - column_title = "ID" + assert tx, "create_menu: no tx" + if column == 0: + column_title = _('Transaction ID') column_data = tx_hash else: - column_title = self.headerItem().text(column) - column_data = item.text(column) + column_title = self.std_model.horizontalHeaderItem(column).text() + column_data = item.text() tx_URL = block_explorer_URL(self.config, 'tx', tx_hash) height = self.wallet.get_tx_height(tx_hash).height is_relevant, is_mine, v, fee = self.wallet.get_wallet_delta(tx) @@ -372,8 +504,10 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): menu.addAction(_("Remove"), lambda: self.remove_local_tx(tx_hash)) menu.addAction(_("Copy {}").format(column_title), lambda: self.parent.app.clipboard().setText(column_data)) for c in self.editable_columns: - menu.addAction(_("Edit {}").format(self.headerItem().text(c)), - lambda bound_c=c: self.editItem(item, bound_c)) + label = self.std_model.horizontalHeaderItem(c).text() + # TODO use siblingAtColumn when min Qt version is >=5.11 + persistent = QPersistentModelIndex(org_idx.sibling(org_idx.row(), c)) + menu.addAction(_("Edit {}").format(label), lambda p=persistent: self.edit(QModelIndex(p))) menu.addAction(_("Details"), lambda: self.show_transaction(tx_hash)) if is_unconfirmed and tx: # note: the current implementation of RBF *needs* the old tx fee @@ -442,7 +576,7 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): self.parent.show_message(_("Your wallet history has been successfully exported.")) def do_export_history(self, file_name, is_csv): - history = self.transactions + history = self.transactions.values() lines = [] if is_csv: for item in history: diff --git a/electrum/gui/qt/invoice_list.py b/electrum/gui/qt/invoice_list.py index 462aadd85..4789aa066 100644 --- a/electrum/gui/qt/invoice_list.py +++ b/electrum/gui/qt/invoice_list.py @@ -29,36 +29,40 @@ from electrum.util import format_time from .util import * -class InvoiceList(MyTreeWidget): +class InvoiceList(MyTreeView): filter_columns = [0, 1, 2, 3] # Date, Requestor, Description, Amount def __init__(self, parent): - MyTreeWidget.__init__(self, parent, self.create_menu, [_('Expires'), _('Requestor'), _('Description'), _('Amount'), _('Status')], 2) + super().__init__(parent, self.create_menu, 2) self.setSortingEnabled(True) - self.header().setSectionResizeMode(1, QHeaderView.Interactive) self.setColumnWidth(1, 200) + self.setModel(QStandardItemModel(self)) + self.update() - def on_update(self): + def update(self): inv_list = self.parent.invoices.unpaid_invoices() - self.clear() + self.model().clear() + self.update_headers([_('Expires'), _('Requestor'), _('Description'), _('Amount'), _('Status')]) + self.header().setSectionResizeMode(1, QHeaderView.Interactive) for pr in inv_list: key = pr.get_id() status = self.parent.invoices.get_status(key) requestor = pr.get_requestor() exp = pr.get_expiration_date() date_str = format_time(exp) if exp else _('Never') - item = QTreeWidgetItem([date_str, requestor, pr.memo, self.parent.format_amount(pr.get_amount(), whitespaces=True), pr_tooltips.get(status,'')]) - item.setIcon(4, self.icon_cache.get(pr_icons.get(status))) - item.setData(0, Qt.UserRole, key) - item.setFont(1, QFont(MONOSPACE_FONT)) - item.setFont(3, QFont(MONOSPACE_FONT)) + labels = [date_str, requestor, pr.memo, self.parent.format_amount(pr.get_amount(), whitespaces=True), pr_tooltips.get(status,'')] + item = [QStandardItem(e) for e in labels] + item[4].setIcon(self.icon_cache.get(pr_icons.get(status))) + item[0].setData(Qt.UserRole, key) + item[1].setFont(QFont(MONOSPACE_FONT)) + item[3].setFont(QFont(MONOSPACE_FONT)) self.addTopLevelItem(item) - self.setCurrentItem(self.topLevelItem(0)) + self.selectionModel().select(self.model().index(0,0), QItemSelectionModel.SelectCurrent) self.setVisible(len(inv_list)) self.parent.invoices_label.setVisible(len(inv_list)) def import_invoices(self): - import_meta_gui(self.parent, _('invoices'), self.parent.invoices.import_file, self.on_update) + import_meta_gui(self.parent, _('invoices'), self.parent.invoices.import_file, self.update) def export_invoices(self): export_meta_gui(self.parent, _('invoices'), self.parent.invoices.export_file) diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 7c3f5e2e0..db0e30657 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -353,8 +353,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError): if self.config.is_dynfee(): self.fee_slider.update() self.do_update_fee() - # todo: update only unconfirmed tx - self.history_list.update() + self.history_list.update_on_new_fee_histogram() else: self.print_error("unexpected network_qt signal:", event, args) @@ -379,9 +378,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError): def load_wallet(self, wallet): wallet.thread = TaskThread(self, self.on_error) self.update_recently_visited(wallet.storage.path) - # update(==init) all tabs; expensive for large wallets.. - # so delay it somewhat, hence __init__ can finish and the window can appear sooner - QTimer.singleShot(50, self.update_tabs) self.need_update.set() # Once GUI has been initialized check if we want to announce something since the callback has been called before the GUI was initialized # update menus @@ -1111,9 +1107,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError): self.from_label = QLabel(_('From')) grid.addWidget(self.from_label, 3, 0) - self.from_list = MyTreeWidget(self, self.from_list_menu, ['','']) - self.from_list.setHeaderHidden(True) - self.from_list.setMaximumHeight(80) + self.from_list = FromList(self, self.from_list_menu) grid.addWidget(self.from_list, 3, 1, 1, -1) self.set_pay_from([]) diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index 94ae77735..a1f2dace0 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -100,7 +100,6 @@ class NodesListWidget(QTreeWidget): def update(self, network: Network): self.clear() - self.addChild = self.addTopLevelItem chains = network.get_blockchains() n_chains = len(chains) for chain_id, interfaces in chains.items(): @@ -118,7 +117,7 @@ class NodesListWidget(QTreeWidget): item = QTreeWidgetItem([i.host + star, '%d'%i.tip]) item.setData(0, Qt.UserRole, 0) item.setData(1, Qt.UserRole, i.server) - x.addChild(item) + x.addTopLevelItem(item) if n_chains > 1: self.addTopLevelItem(x) x.setExpanded(True) diff --git a/electrum/gui/qt/request_list.py b/electrum/gui/qt/request_list.py index 19ec59703..8c6567fc8 100644 --- a/electrum/gui/qt/request_list.py +++ b/electrum/gui/qt/request_list.py @@ -23,43 +23,39 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from PyQt5.QtGui import * -from PyQt5.QtCore import * -from PyQt5.QtWidgets import QTreeWidgetItem, QMenu +from PyQt5.QtGui import QStandardItemModel, QStandardItem +from PyQt5.QtWidgets import QMenu +from PyQt5.QtCore import Qt from electrum.i18n import _ from electrum.util import format_time, age from electrum.plugin import run_hook from electrum.paymentrequest import PR_UNKNOWN -from .util import MyTreeWidget, pr_tooltips, pr_icons +from .util import MyTreeView, pr_tooltips, pr_icons - -class RequestList(MyTreeWidget): +class RequestList(MyTreeView): filter_columns = [0, 1, 2, 3, 4] # Date, Account, Address, Description, Amount def __init__(self, parent): - MyTreeWidget.__init__(self, parent, self.create_menu, [_('Date'), _('Address'), '', _('Description'), _('Amount'), _('Status')], 3) - self.currentItemChanged.connect(self.item_changed) - self.itemClicked.connect(self.item_changed) + super().__init__(parent, self.create_menu, 3, editable_columns=[]) + self.setModel(QStandardItemModel(self)) self.setSortingEnabled(True) self.setColumnWidth(0, 180) - self.hideColumn(1) + self.update() + self.selectionModel().currentRowChanged.connect(self.item_changed) - def item_changed(self, item): - if item is None: - return - if not item.isSelected(): - return - addr = str(item.text(1)) + def item_changed(self, idx): + # TODO use siblingAtColumn when min Qt version is >=5.11 + addr = self.model().itemFromIndex(idx.sibling(idx.row(), 1)).text() req = self.wallet.receive_requests.get(addr) if req is None: self.update() return expires = age(req['time'] + req['exp']) if req.get('exp') else _('Never') amount = req['amount'] - message = self.wallet.labels.get(addr, '') + message = req['memo'] self.parent.receive_address_e.setText(addr) self.parent.receive_message_e.setText(message) self.parent.receive_amount_e.setAmount(amount) @@ -68,7 +64,7 @@ class RequestList(MyTreeWidget): self.parent.expires_label.setText(expires) self.parent.new_request_button.setEnabled(True) - def on_update(self): + def update(self): self.wallet = self.parent.wallet # hide receive tab if no receive requests available b = len(self.wallet.receive_requests) > 0 @@ -86,8 +82,9 @@ class RequestList(MyTreeWidget): self.parent.set_receive_address(addr) self.parent.new_request_button.setEnabled(addr != current_address) - # clear the list and fill it again - self.clear() + self.model().clear() + self.update_headers([_('Date'), _('Address'), '', _('Description'), _('Amount'), _('Status')]) + self.hideColumn(1) # hide address column for req in self.wallet.get_sorted_requests(self.config): address = req['address'] if address not in domain: @@ -95,35 +92,40 @@ class RequestList(MyTreeWidget): timestamp = req.get('time', 0) amount = req.get('amount') expiration = req.get('exp', None) - message = req.get('memo', '') + message = req['memo'] date = format_time(timestamp) status = req.get('status') signature = req.get('sig') requestor = req.get('name', '') amount_str = self.parent.format_amount(amount) if amount else "" - item = QTreeWidgetItem([date, address, '', message, amount_str, pr_tooltips.get(status,'')]) + labels = [date, address, '', message, amount_str, pr_tooltips.get(status,'')] + items = [QStandardItem(e) for e in labels] + self.set_editability(items) if signature is not None: - item.setIcon(2, self.icon_cache.get(":icons/seal.png")) - item.setToolTip(2, 'signed by '+ requestor) + items[2].setIcon(self.icon_cache.get(":icons/seal.png")) + items[2].setToolTip('signed by '+ requestor) if status is not PR_UNKNOWN: - item.setIcon(6, self.icon_cache.get(pr_icons.get(status))) - self.addTopLevelItem(item) - + items[5].setIcon(self.icon_cache.get(pr_icons.get(status))) + items[3].setData(address, Qt.UserRole) + self.model().insertRow(self.model().rowCount(), items) def create_menu(self, position): - item = self.itemAt(position) + idx = self.indexAt(position) + # TODO use siblingAtColumn when min Qt version is >=5.11 + item = self.model().itemFromIndex(idx.sibling(idx.row(), 1)) if not item: return - addr = str(item.text(1)) + addr = item.text() req = self.wallet.receive_requests.get(addr) if req is None: self.update() return - column = self.currentColumn() - column_title = self.headerItem().text(column) - column_data = item.text(column) + column = idx.column() + column_title = self.model().horizontalHeaderItem(column).text() + column_data = item.text() menu = QMenu(self) - menu.addAction(_("Copy {}").format(column_title), lambda: self.parent.app.clipboard().setText(column_data)) + if column != 2: + menu.addAction(_("Copy {}").format(column_title), lambda: self.parent.app.clipboard().setText(column_data)) menu.addAction(_("Copy URI"), lambda: self.parent.view_and_paste('URI', '', self.parent.get_request_URI(addr))) menu.addAction(_("Save as BIP70 file"), lambda: self.parent.export_payment_request(addr)) menu.addAction(_("Delete"), lambda: self.parent.delete_payment_request(addr)) diff --git a/electrum/gui/qt/util.py b/electrum/gui/qt/util.py index b6db61c13..e320715a1 100644 --- a/electrum/gui/qt/util.py +++ b/electrum/gui/qt/util.py @@ -5,6 +5,7 @@ import platform import queue from functools import partial from typing import NamedTuple, Callable, Optional +from abc import abstractmethod from PyQt5.QtGui import * from PyQt5.QtCore import * @@ -398,20 +399,16 @@ class ElectrumItemDelegate(QStyledItemDelegate): def createEditor(self, parent, option, index): return self.parent().createEditor(parent, option, index) -class MyTreeWidget(QTreeWidget): +class MyTreeView(QTreeView): - def __init__(self, parent, create_menu, headers, stretch_column=None, - editable_columns=None): - QTreeWidget.__init__(self, parent) + def __init__(self, parent, create_menu, stretch_column=None, editable_columns=None): + super().__init__(parent) self.parent = parent self.config = self.parent.config self.stretch_column = stretch_column self.setContextMenuPolicy(Qt.CustomContextMenu) self.customContextMenuRequested.connect(create_menu) self.setUniformRowHeights(True) - # extend the syntax for consistency - self.addChild = self.addTopLevelItem - self.insertChild = self.insertTopLevelItem self.icon_cache = IconCache() @@ -424,127 +421,143 @@ class MyTreeWidget(QTreeWidget): editable_columns = set(editable_columns) self.editable_columns = editable_columns self.setItemDelegate(ElectrumItemDelegate(self)) - self.itemDoubleClicked.connect(self.on_doubleclick) - self.update_headers(headers) self.current_filter = "" self.setRootIsDecorated(False) # remove left margin self.toolbar_shown = False - def update_headers(self, headers): - self.setColumnCount(len(headers)) - self.setHeaderLabels(headers) + def set_editability(self, items): + for idx, i in enumerate(items): + i.setEditable(idx in self.editable_columns) + + def selected_in_column(self, column: int): + items = self.selectionModel().selectedIndexes() + return list(x for x in items if x.column() == column) + + def current_item_user_role(self, col) -> Optional[QStandardItem]: + idx = self.selectionModel().currentIndex() + idx = idx.sibling(idx.row(), col) + item = self.model().itemFromIndex(idx) + if item: + return item.data(Qt.UserRole) + + def set_current_idx(self, set_current: QPersistentModelIndex): + if set_current: + assert isinstance(set_current, QPersistentModelIndex) + assert set_current.isValid() + self.selectionModel().select(QModelIndex(set_current), QItemSelectionModel.SelectCurrent) + + def update_headers(self, headers, model=None): + if model is None: + model = self.model() + model.setHorizontalHeaderLabels(headers) self.header().setStretchLastSection(False) for col in range(len(headers)): sm = QHeaderView.Stretch if col == self.stretch_column else QHeaderView.ResizeToContents self.header().setSectionResizeMode(col, sm) - def editItem(self, item, column): - if column in self.editable_columns: - try: - self.editing_itemcol = (item, column, item.text(column)) - # Calling setFlags causes on_changed events for some reason - item.setFlags(item.flags() | Qt.ItemIsEditable) - QTreeWidget.editItem(self, item, column) - item.setFlags(item.flags() & ~Qt.ItemIsEditable) - except RuntimeError: - # (item) wrapped C/C++ object has been deleted - pass - def keyPressEvent(self, event): if event.key() in [ Qt.Key_F2, Qt.Key_Return ] and self.editor is None: - self.on_activated(self.currentItem(), self.currentColumn()) - else: - QTreeWidget.keyPressEvent(self, event) - - def permit_edit(self, item, column): - return (column in self.editable_columns - and self.on_permit_edit(item, column)) - - def on_permit_edit(self, item, column): - return True - - def on_doubleclick(self, item, column): - if self.permit_edit(item, column): - self.editItem(item, column) + self.on_activated(self.selectionModel().currentIndex()) + return + super().keyPressEvent(event) - def on_activated(self, item, column): + def on_activated(self, idx): # on 'enter' we show the menu - pt = self.visualItemRect(item).bottomLeft() + pt = self.visualRect(idx).bottomLeft() pt.setX(50) self.customContextMenuRequested.emit(pt) def createEditor(self, parent, option, index): self.editor = QStyledItemDelegate.createEditor(self.itemDelegate(), parent, option, index) - self.editor.editingFinished.connect(self.editing_finished) + persistent = QPersistentModelIndex(index) + user_role = index.data(Qt.UserRole) + assert user_role is not None + idx = QModelIndex(persistent) + index = self.proxy.mapToSource(idx) + item = self.std_model.itemFromIndex(index) + prior_text = item.text() + def editing_finished(): + # Long-time QT bug - pressing Enter to finish editing signals + # editingFinished twice. If the item changed the sequence is + # Enter key: editingFinished, on_change, editingFinished + # Mouse: on_change, editingFinished + # This mess is the cleanest way to ensure we make the + # on_edited callback with the updated item + if self.editor is None: + return + if self.editor.text() == prior_text: + self.editor = None # Unchanged - ignore any 2nd call + return + if item.text() == prior_text: + return # Buggy first call on Enter key, item not yet updated + if not idx.isValid(): + return + self.on_edited(idx, user_role, self.editor.text()) + self.editor = None + self.editor.editingFinished.connect(editing_finished) return self.editor - def editing_finished(self): - # Long-time QT bug - pressing Enter to finish editing signals - # editingFinished twice. If the item changed the sequence is - # Enter key: editingFinished, on_change, editingFinished - # Mouse: on_change, editingFinished - # This mess is the cleanest way to ensure we make the - # on_edited callback with the updated item - if self.editor: - (item, column, prior_text) = self.editing_itemcol - if self.editor.text() == prior_text: - self.editor = None # Unchanged - ignore any 2nd call - elif item.text(column) == prior_text: - pass # Buggy first call on Enter key, item not yet updated - else: - # What we want - the updated item - self.on_edited(*self.editing_itemcol) - self.editor = None - - # Now do any pending updates - if self.editor is None and self.pending_update: - self.pending_update = False - self.on_update() - - def on_edited(self, item, column, prior): - '''Called only when the text actually changes''' - key = item.data(0, Qt.UserRole) - text = item.text(column) - self.parent.wallet.set_label(key, text) + def edit(self, idx, trigger=QAbstractItemView.AllEditTriggers, event=None): + """ + this is to prevent: + edit: editing failed + from inside qt + """ + return super().edit(idx, trigger, event) + + def on_edited(self, idx: QModelIndex, user_role, text): + self.parent.wallet.set_label(user_role, text) self.parent.history_list.update_labels() self.parent.update_completions() - def update(self): - # Defer updates if editing - if self.editor: - self.pending_update = True - else: - self.setUpdatesEnabled(False) - scroll_pos = self.verticalScrollBar().value() - self.on_update() - self.setUpdatesEnabled(True) - # To paint the list before resetting the scroll position - self.parent.app.processEvents() - self.verticalScrollBar().setValue(scroll_pos) + def apply_filter(self): if self.current_filter: self.filter(self.current_filter) - def on_update(self): + @abstractmethod + def should_hide(self, row): + """ + row_num is for self.model(). So if there is a proxy, it is the row number + in that! + """ pass - def get_leaves(self, root): - child_count = root.childCount() - if child_count == 0: - yield root - for i in range(child_count): - item = root.child(i) - for x in self.get_leaves(item): - yield x + def hide_row(self, row_num): + """ + row_num is for self.model(). So if there is a proxy, it is the row number + in that! + """ + should_hide = self.should_hide(row_num) + if not self.current_filter and should_hide is None: + # no filters at all, neither date nor search + self.setRowHidden(row_num, QModelIndex(), False) + return + for column in self.filter_columns: + if isinstance(self.model(), QSortFilterProxyModel): + idx = self.model().mapToSource(self.model().index(row_num, column)) + item = self.model().sourceModel().itemFromIndex(idx) + else: + idx = self.model().index(row_num, column) + item = self.model().itemFromIndex(idx) + txt = item.text().lower() + if self.current_filter in txt: + # the filter matched, but the date filter might apply + self.setRowHidden(row_num, QModelIndex(), bool(should_hide)) + break + else: + # we did not find the filter in any columns, show the item + self.setRowHidden(row_num, QModelIndex(), True) def filter(self, p): - columns = self.__class__.filter_columns p = p.lower() self.current_filter = p - for item in self.get_leaves(self.invisibleRootItem()): - item.setHidden(all([item.text(column).lower().find(p) == -1 - for column in columns])) + self.hide_rows() + + def hide_rows(self): + for row in range(self.model().rowCount()): + self.hide_row(row) def create_toolbar(self, config=None): hbox = QHBoxLayout() @@ -790,22 +803,6 @@ def get_parent_main_window(widget): return widget return None -class SortableTreeWidgetItem(QTreeWidgetItem): - DataRole = Qt.UserRole + 100 - - def __lt__(self, other): - column = self.treeWidget().sortColumn() - if None not in [x.data(column, self.DataRole) for x in [self, other]]: - # We have set custom data to sort by - return self.data(column, self.DataRole) < other.data(column, self.DataRole) - try: - # Is the value something numeric? - return float(self.text(column)) < float(other.text(column)) - except ValueError: - # If not, we will just do string comparison - return self.text(column) < other.text(column) - - class IconCache: def __init__(self): @@ -821,6 +818,21 @@ def get_default_language(): name = QLocale.system().name() return name if name in languages else 'en_UK' +class FromList(QTreeWidget): + def __init__(self, parent, create_menu): + super().__init__(parent) + self.setHeaderHidden(True) + self.setMaximumHeight(300) + self.setContextMenuPolicy(Qt.CustomContextMenu) + self.customContextMenuRequested.connect(create_menu) + self.setUniformRowHeights(True) + # remove left margin + self.setRootIsDecorated(False) + self.setColumnCount(2) + self.header().setStretchLastSection(False) + sm = QHeaderView.ResizeToContents + self.header().setSectionResizeMode(0, sm) + self.header().setSectionResizeMode(1, sm) if __name__ == "__main__": app = QApplication([]) diff --git a/electrum/gui/qt/utxo_list.py b/electrum/gui/qt/utxo_list.py index 0a6dc3349..0b9d85508 100644 --- a/electrum/gui/qt/utxo_list.py +++ b/electrum/gui/qt/utxo_list.py @@ -23,49 +23,60 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from typing import Optional, List + from electrum.i18n import _ from .util import * - -class UTXOList(MyTreeWidget): - filter_columns = [0, 2] # Address, Label +class UTXOList(MyTreeView): + filter_columns = [0, 1] # Address, Label def __init__(self, parent=None): - MyTreeWidget.__init__(self, parent, self.create_menu, [ _('Address'), _('Label'), _('Amount'), _('Height'), _('Output point')], 1) + super().__init__(parent, self.create_menu, 1) + self.setModel(QStandardItemModel(self)) self.setSelectionMode(QAbstractItemView.ExtendedSelection) self.setSortingEnabled(True) + self.update() - def get_name(self, x): - return x.get('prevout_hash') + ":%d"%x.get('prevout_n') - - def on_update(self): + def update(self): self.wallet = self.parent.wallet - item = self.currentItem() - self.clear() - self.utxos = self.wallet.get_utxos() - for x in self.utxos: + utxos = self.wallet.get_utxos() + self.utxo_dict = {} + self.model().clear() + self.update_headers([ _('Address'), _('Label'), _('Amount'), _('Height'), _('Output point')]) + for idx, x in enumerate(utxos): address = x.get('address') height = x.get('height') - name = self.get_name(x) + name = x.get('prevout_hash') + ":%d"%x.get('prevout_n') + self.utxo_dict[name] = x label = self.wallet.get_label(x.get('prevout_hash')) amount = self.parent.format_amount(x['value'], whitespaces=True) - utxo_item = SortableTreeWidgetItem([address, label, amount, '%d'%height, name[0:10] + '...' + name[-2:]]) - utxo_item.setFont(0, QFont(MONOSPACE_FONT)) - utxo_item.setFont(2, QFont(MONOSPACE_FONT)) - utxo_item.setFont(4, QFont(MONOSPACE_FONT)) - utxo_item.setData(0, Qt.UserRole, name) + labels = [address, label, amount, '%d'%height, name[0:10] + '...' + name[-2:]] + utxo_item = [QStandardItem(x) for x in labels] + self.set_editability(utxo_item) + utxo_item[0].setFont(QFont(MONOSPACE_FONT)) + utxo_item[2].setFont(QFont(MONOSPACE_FONT)) + utxo_item[4].setFont(QFont(MONOSPACE_FONT)) + utxo_item[0].setData(name, Qt.UserRole) if self.wallet.is_frozen(address): - utxo_item.setBackground(0, ColorScheme.BLUE.as_color(True)) - self.addChild(utxo_item) + utxo_item[0].setBackground(ColorScheme.BLUE.as_color(True)) + self.model().insertRow(idx, utxo_item) + + def selected_column_0_user_roles(self) -> Optional[List[str]]: + if not self.model(): + return None + items = self.selected_in_column(0) + if not items: + return None + return [x.data(Qt.UserRole) for x in items] def create_menu(self, position): - selected = [x.data(0, Qt.UserRole) for x in self.selectedItems()] + selected = self.selected_column_0_user_roles() if not selected: return menu = QMenu() - coins = filter(lambda x: self.get_name(x) in selected, self.utxos) - + coins = (self.utxo_dict[name] for name in selected) menu.addAction(_("Spend"), lambda: self.parent.spend_coins(coins)) if len(selected) == 1: txid = selected[0].split(':')[0] @@ -75,7 +86,3 @@ class UTXOList(MyTreeWidget): menu.addAction(_("Details"), lambda: self.parent.show_transaction(tx, label)) menu.exec_(self.viewport().mapToGlobal(position)) - - def on_permit_edit(self, item, column): - # disable editing fields in this tab (labels) - return False From 72957f4d51265f2d0e61d90f92ba42a6e2f92726 Mon Sep 17 00:00:00 2001 From: Janus Date: Mon, 3 Dec 2018 15:33:51 +0100 Subject: [PATCH 38/40] qt_standardmodel: only use proxymodel when appropriate --- electrum/gui/qt/history_list.py | 10 ++++------ electrum/gui/qt/util.py | 26 +++++++++++++------------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index 807484f9b..f2feba9dd 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -75,8 +75,7 @@ class HistoryList(MyTreeView, AcceptFileDragDrop): def should_hide(self, proxy_row): if self.start_timestamp and self.end_timestamp: - source_idx = self.proxy.mapToSource(self.proxy.index(proxy_row, 0)) - item = self.std_model.itemFromIndex(source_idx) + item = self.item_from_coordinate(proxy_row, 0) txid = item.data(self.TX_HASH_ROLE) date = self.transactions[txid]['date'] if date: @@ -418,9 +417,8 @@ class HistoryList(MyTreeView, AcceptFileDragDrop): # TODO update unconfirmed tx'es def on_edited(self, index, user_role, text): - column = index.column() - index = self.proxy.mapToSource(index) - item = self.std_model.itemFromIndex(index) + row, column = index.row(), index.column() + item = self.item_from_coordinate(row, column) key = item.data(self.TX_HASH_ROLE) # fixme if column == 2: @@ -441,7 +439,7 @@ class HistoryList(MyTreeView, AcceptFileDragDrop): def mouseDoubleClickEvent(self, event: QMouseEvent): idx = self.indexAt(event.pos()) - item = self.std_model.itemFromIndex(self.proxy.mapToSource(idx)) + item = self.item_from_coordinate(idx.row(), idx.column()) if not item or item.isEditable(): super().mouseDoubleClickEvent(event) elif item: diff --git a/electrum/gui/qt/util.py b/electrum/gui/qt/util.py index e320715a1..0d386bdfc 100644 --- a/electrum/gui/qt/util.py +++ b/electrum/gui/qt/util.py @@ -468,15 +468,12 @@ class MyTreeView(QTreeView): pt.setX(50) self.customContextMenuRequested.emit(pt) - def createEditor(self, parent, option, index): + def createEditor(self, parent, option, idx): self.editor = QStyledItemDelegate.createEditor(self.itemDelegate(), - parent, option, index) - persistent = QPersistentModelIndex(index) - user_role = index.data(Qt.UserRole) + parent, option, idx) + item = self.item_from_coordinate(idx.row(), idx.column()) + user_role = item.data(Qt.UserRole) assert user_role is not None - idx = QModelIndex(persistent) - index = self.proxy.mapToSource(idx) - item = self.std_model.itemFromIndex(index) prior_text = item.text() def editing_finished(): # Long-time QT bug - pressing Enter to finish editing signals @@ -524,6 +521,14 @@ class MyTreeView(QTreeView): """ pass + def item_from_coordinate(self, row_num, column): + if isinstance(self.model(), QSortFilterProxyModel): + idx = self.model().mapToSource(self.model().index(row_num, column)) + return self.model().sourceModel().itemFromIndex(idx) + else: + idx = self.model().index(row_num, column) + return self.model().itemFromIndex(idx) + def hide_row(self, row_num): """ row_num is for self.model(). So if there is a proxy, it is the row number @@ -535,12 +540,7 @@ class MyTreeView(QTreeView): self.setRowHidden(row_num, QModelIndex(), False) return for column in self.filter_columns: - if isinstance(self.model(), QSortFilterProxyModel): - idx = self.model().mapToSource(self.model().index(row_num, column)) - item = self.model().sourceModel().itemFromIndex(idx) - else: - idx = self.model().index(row_num, column) - item = self.model().itemFromIndex(idx) + item = self.item_from_coordinate(row_num, column) txt = item.text().lower() if self.current_filter in txt: # the filter matched, but the date filter might apply From 0677ce6d52b8951df04f31030e52b805027623d2 Mon Sep 17 00:00:00 2001 From: Janus Date: Mon, 3 Dec 2018 15:54:21 +0100 Subject: [PATCH 39/40] qt: avoid app.palette().text().color(), doesn't work on dark style --- electrum/gui/qt/history_list.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index f2feba9dd..1ceb646b1 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -97,7 +97,6 @@ class HistoryList(MyTreeView, AcceptFileDragDrop): self.blue_brush = QBrush(QColor("#1E1EFF")) self.red_brush = QBrush(QColor("#BC1E1E")) self.monospace_font = QFont(MONOSPACE_FONT) - self.default_color = self.parent.app.palette().text().color() self.config = parent.config AcceptFileDragDrop.__init__(self, ".txn") self.setSortingEnabled(True) @@ -399,7 +398,8 @@ class HistoryList(MyTreeView, AcceptFileDragDrop): cap_gains = self.parent.fx.get_history_capital_gains_config() items = self.txid_to_items[txid] self.ensure_fields_available(items, 7 if cap_gains else 5, txid) - items[5].setForeground(self.blue_brush if not row['fiat_default'] and row['fiat_value'] else self.default_color) + if not row['fiat_default'] and row['fiat_value']: + items[5].setForeground(self.blue_brush) value_str = self.parent.fx.format_fiat(row['fiat_value'].value) items[5].setText(value_str) items[5].setData(row['fiat_value'].value, self.SORT_ROLE) From d69ef890c0f7ab8c1f0e34c4f33853d57337804e Mon Sep 17 00:00:00 2001 From: SomberNight Date: Mon, 3 Dec 2018 16:04:17 +0100 Subject: [PATCH 40/40] downgrade qdarkstyle for now see ColinDuquesnoy/QDarkStyleSheet#123 --- contrib/requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/requirements/requirements.txt b/contrib/requirements/requirements.txt index f4f458c3a..7d42df3f1 100644 --- a/contrib/requirements/requirements.txt +++ b/contrib/requirements/requirements.txt @@ -5,7 +5,7 @@ qrcode protobuf dnspython jsonrpclib-pelix -qdarkstyle<3.0 +qdarkstyle<2.6 aiorpcx>=0.9,<0.11 aiohttp aiohttp_socks