diff --git a/.gitmodules b/.gitmodules index 5a0f914f1..f95b1ebce 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "contrib/deterministic-build/electrum-locale"] path = contrib/deterministic-build/electrum-locale url = https://github.com/spesmilo/electrum-locale +[submodule "contrib/CalinsQRReader"] + path = contrib/osx/CalinsQRReader + url = https://github.com/spesmilo/CalinsQRReader diff --git a/.travis.yml b/.travis.yml index 1c222c4fa..bfcef0231 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ jobs: python: false install: - git fetch --all --tags - script: ./contrib/build-osx/make_osx + script: ./contrib/osx/make_osx after_script: ls -lah dist && md5 dist/* after_success: true - stage: release check diff --git a/README.rst b/README.rst index 3f67724b4..6dfb69940 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ Qt interface, install the Qt dependencies:: sudo apt-get install python3-pyqt5 If you downloaded the official package (tar.gz), you can run -Electrum from its root directory, without installing it on your +Electrum from its root directory without installing it on your system; all the python dependencies are included in the 'packages' directory. To run Electrum from its root directory, just do:: @@ -44,7 +44,7 @@ You can also install Electrum on your system, by running this command:: python3 -m pip install .[fast] This will download and install the Python dependencies used by -Electrum, instead of using the 'packages' directory. +Electrum instead of using the 'packages' directory. The 'fast' extra contains some optional dependencies that we think are often useful but they are not strictly needed. @@ -101,7 +101,7 @@ This directory contains the python dependencies used by Electrum. Mac OS X / macOS -------- -See `contrib/build-osx/`. +See `contrib/osx/`. Windows ------- diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 9350036e3..43486f914 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -1,3 +1,15 @@ +# Release 3.3 - (Hodler's Edition) + + * The network layer has been rewritten using asyncio. + * Follow blockchain that has the most work, not length. + * New wallet creation defaults to native segwit (bech32). + * RBF batching (option): If the wallet has an unconfirmed RBF + transaction, new payments will be added to that transaction, + instead of creating new transactions. + * OSX: support QR code scanner. + * Android APK: Use API 28, and do not use external storage. + + # Release 3.2.3 - (September 3, 2018) * hardware wallet: the Safe-T mini from Archos is now supported. diff --git a/contrib/build-osx/base.sh b/contrib/build-osx/base.sh deleted file mode 100644 index c5a5c0d69..000000000 --- a/contrib/build-osx/base.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -RED='\033[0;31m' -BLUE='\033[0,34m' -NC='\033[0m' # No Color -function info { - printf "\r💬 ${BLUE}INFO:${NC} ${1}\n" -} -function fail { - printf "\r🗯 ${RED}ERROR:${NC} ${1}\n" - exit 1 -} diff --git a/contrib/build-wine/docker/Dockerfile b/contrib/build-wine/docker/Dockerfile index 340bad00d..8ed7a5467 100644 --- a/contrib/build-wine/docker/Dockerfile +++ b/contrib/build-wine/docker/Dockerfile @@ -20,7 +20,7 @@ RUN dpkg --add-architecture i386 && \ wine-stable-i386:i386=3.0.1~bionic \ wine-stable:amd64=3.0.1~bionic \ winehq-stable:amd64=3.0.1~bionic \ - git=1:2.17.1-1ubuntu0.3 \ + git=1:2.17.1-1ubuntu0.4 \ p7zip-full=16.02+dfsg-6 \ make=4.1-9.1ubuntu1 \ mingw-w64=5.0.3-1 \ diff --git a/contrib/make_apk b/contrib/make_apk index 773aeab54..6940222cd 100755 --- a/contrib/make_apk +++ b/contrib/make_apk @@ -2,6 +2,8 @@ pushd ./electrum/gui/kivy/ +make theming + if [[ -n "$1" && "$1" == "release" ]] ; then echo -n Keystore Password: read -s password diff --git a/contrib/osx/CalinsQRReader b/contrib/osx/CalinsQRReader new file mode 160000 index 000000000..20189155a --- /dev/null +++ b/contrib/osx/CalinsQRReader @@ -0,0 +1 @@ +Subproject commit 20189155a461cf7fbad14357e58fbc8e7c964608 diff --git a/contrib/build-osx/README.md b/contrib/osx/README.md similarity index 93% rename from contrib/build-osx/README.md rename to contrib/osx/README.md index c1e96d90b..056d9fb84 100644 --- a/contrib/build-osx/README.md +++ b/contrib/osx/README.md @@ -14,7 +14,7 @@ Before starting, make sure that the Xcode command line tools are installed (e.g. cd electrum - ./contrib/build-osx/make_osx + ./contrib/osx/make_osx This creates a folder named Electrum.app. @@ -33,4 +33,4 @@ Copy the Electrum.app directory over and install the dependencies, e.g.: Then you can just invoke `package.sh` with the path to the app: cd electrum - ./contrib/build-osx/package.sh ~/Electrum.app/ \ No newline at end of file + ./contrib/osx/package.sh ~/Electrum.app/ \ No newline at end of file diff --git a/contrib/osx/base.sh b/contrib/osx/base.sh new file mode 100644 index 000000000..2c22ca9cf --- /dev/null +++ b/contrib/osx/base.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +RED='\033[0;31m' +BLUE='\033[0,34m' +YELLOW='\033[0;33m' +NC='\033[0m' # No Color +function info { + printf "\r💬 ${BLUE}INFO:${NC} ${1}\n" +} +function fail { + printf "\r🗯 ${RED}ERROR:${NC} ${1}\n" + exit 1 +} +function warn { + printf "\r⚠️ ${YELLOW}WARNING:${NC} ${1}\n" +} + +function DoCodeSignMaybe { # ARGS: infoName fileOrDirName codesignIdentity + infoName="$1" + file="$2" + identity="$3" + deep="" + if [ -z "$identity" ]; then + # we are ok with them not passing anything; master script calls us unconditionally even if no identity is specified + return + fi + if [ -d "$file" ]; then + deep="--deep" + fi + if [ -z "$infoName" ] || [ -z "$file" ] || [ -z "$identity" ] || [ ! -e "$file" ]; then + fail "Argument error to internal function DoCodeSignMaybe()" + fi + info "Code signing ${infoName}..." + codesign -f -v $deep -s "$identity" "$file" || fail "Could not code sign ${infoName}" +} diff --git a/contrib/build-osx/cdrkit-deterministic.patch b/contrib/osx/cdrkit-deterministic.patch similarity index 100% rename from contrib/build-osx/cdrkit-deterministic.patch rename to contrib/osx/cdrkit-deterministic.patch diff --git a/contrib/build-osx/make_osx b/contrib/osx/make_osx similarity index 61% rename from contrib/build-osx/make_osx rename to contrib/osx/make_osx index 599480e23..b1cf4b724 100755 --- a/contrib/build-osx/make_osx +++ b/contrib/osx/make_osx @@ -16,6 +16,25 @@ export PYTHONHASHSEED=22 VERSION=`git describe --tags --dirty --always` which brew > /dev/null 2>&1 || fail "Please install brew from https://brew.sh/ to continue" +which xcodebuild > /dev/null 2>&1 || fail "Please install Xcode and xcode command line tools to continue" + +# Code Signing: See https://developer.apple.com/library/archive/documentation/Security/Conceptual/CodeSigningGuide/Procedures/Procedures.html +APP_SIGN="" +if [ -n "$1" ]; then + # Test the identity is valid for signing by doing this hack. There is no other way to do this. + cp -f /bin/ls ./CODESIGN_TEST + codesign -s "$1" --dryrun -f ./CODESIGN_TEST > /dev/null 2>&1 + res=$? + rm -f ./CODESIGN_TEST + if ((res)); then + fail "Code signing identity \"$1\" appears to be invalid." + fi + unset res + APP_SIGN="$1" + info "Code signing enabled using identity \"$APP_SIGN\"" +else + warn "Code signing DISABLED. Specify a valid macOS Developer identity installed on the system as the first argument to this script to enable signing." +fi info "Installing Python $PYTHON_VERSION" export PATH="~/.pyenv/bin:~/.pyenv/shims:~/Library/Python/3.6/bin:$PATH" @@ -53,7 +72,8 @@ cp ./contrib/deterministic-build/electrum-icons/icons_rc.py ./electrum/gui/qt info "Downloading libusb..." curl https://homebrew.bintray.com/bottles/libusb-1.0.22.el_capitan.bottle.tar.gz | \ tar xz --directory $BUILDDIR -cp $BUILDDIR/libusb/1.0.22/lib/libusb-1.0.dylib contrib/build-osx +cp $BUILDDIR/libusb/1.0.22/lib/libusb-1.0.dylib contrib/osx +DoCodeSignMaybe "libusb" "contrib/osx/libusb-1.0.dylib" "$APP_SIGN" # If APP_SIGN is empty will be a noop info "Building libsecp256k1" brew install autoconf automake libtool @@ -65,7 +85,16 @@ git clean -f -x -q ./configure --enable-module-recovery --enable-experimental --enable-module-ecdh --disable-jni make popd -cp $BUILDDIR/secp256k1/.libs/libsecp256k1.0.dylib contrib/build-osx +cp $BUILDDIR/secp256k1/.libs/libsecp256k1.0.dylib contrib/osx +DoCodeSignMaybe "libsecp256k1" "contrib/osx/libsecp256k1.0.dylib" "$APP_SIGN" # If APP_SIGN is empty will be a noop + +info "Building CalinsQRReader..." +d=contrib/osx/CalinsQRReader +pushd $d +rm -fr build +xcodebuild || fail "Could not build CalinsQRReader" +popd +DoCodeSignMaybe "CalinsQRReader.app" "${d}/build/Release/CalinsQRReader.app" "$APP_SIGN" # If APP_SIGN is empty will be a noop info "Installing requirements..." @@ -88,7 +117,7 @@ for d in ~/Library/Python/ ~/.pyenv .; do done info "Building binary" -pyinstaller --noconfirm --ascii --clean --name $VERSION contrib/build-osx/osx.spec || fail "Could not build binary" +pyinstaller --noconfirm --ascii --clean --name $VERSION contrib/osx/osx.spec || fail "Could not build binary" info "Adding bitcoin URI types to Info.plist" plutil -insert 'CFBundleURLTypes' \ @@ -96,5 +125,14 @@ plutil -insert 'CFBundleURLTypes' \ -- dist/$PACKAGE.app/Contents/Info.plist \ || fail "Could not add keys to Info.plist. Make sure the program 'plutil' exists and is installed." +DoCodeSignMaybe "app bundle" "dist/${PACKAGE}.app" "$APP_SIGN" # If APP_SIGN is empty will be a noop + info "Creating .DMG" hdiutil create -fs HFS+ -volname $PACKAGE -srcfolder dist/$PACKAGE.app dist/electrum-$VERSION.dmg || fail "Could not create .DMG" + +DoCodeSignMaybe ".DMG" "dist/electrum-${VERSION}.dmg" "$APP_SIGN" # If APP_SIGN is empty will be a noop + +if [ -z "$APP_SIGN" ]; then + warn "App was built successfully but was not code signed. Users may get security warnings from macOS." + warn "Specify a valid code signing identity as the first argument to this script to enable code signing." +fi diff --git a/contrib/build-osx/osx.spec b/contrib/osx/osx.spec similarity index 91% rename from contrib/build-osx/osx.spec rename to contrib/osx/osx.spec index 862548165..ac9c07a0e 100644 --- a/contrib/build-osx/osx.spec +++ b/contrib/osx/osx.spec @@ -41,9 +41,12 @@ datas += collect_data_files('btchip') datas += collect_data_files('keepkeylib') datas += collect_data_files('ckcc') +# Add the QR Scanner helper app +datas += [(electrum + "contrib/osx/CalinsQRReader/build/Release/CalinsQRReader.app", "./contrib/osx/CalinsQRReader/build/Release/CalinsQRReader.app")] + # Add libusb so Trezor and Safe-T mini will work -binaries = [(electrum + "contrib/build-osx/libusb-1.0.dylib", ".")] -binaries += [(electrum + "contrib/build-osx/libsecp256k1.0.dylib", ".")] +binaries = [(electrum + "contrib/osx/libusb-1.0.dylib", ".")] +binaries += [(electrum + "contrib/osx/libsecp256k1.0.dylib", ".")] # Workaround for "Retro Look": binaries += [b for b in collect_dynamic_libs('PyQt5') if 'macstyle' in b[0]] diff --git a/contrib/build-osx/package.sh b/contrib/osx/package.sh similarity index 100% rename from contrib/build-osx/package.sh rename to contrib/osx/package.sh diff --git a/contrib/requirements/requirements.txt b/contrib/requirements/requirements.txt index f4f458c3a..7d42df3f1 100644 --- a/contrib/requirements/requirements.txt +++ b/contrib/requirements/requirements.txt @@ -5,7 +5,7 @@ qrcode protobuf dnspython jsonrpclib-pelix -qdarkstyle<3.0 +qdarkstyle<2.6 aiorpcx>=0.9,<0.11 aiohttp aiohttp_socks diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index e0d1ec241..909ed028d 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -717,12 +717,15 @@ class AddressSynchronizer(PrintError): return None if hasattr(tx, '_cached_fee'): return tx._cached_fee - is_relevant, is_mine, v, fee = self.get_wallet_delta(tx) - if fee is None: - txid = tx.txid() - fee = self.tx_fees.get(txid) - if fee is not None: - tx._cached_fee = fee + with self.lock, self.transaction_lock: + is_relevant, is_mine, v, fee = self.get_wallet_delta(tx) + if fee is None: + txid = tx.txid() + fee = self.tx_fees.get(txid) + # cache fees. if wallet is synced, cache all; + # otherwise only cache non-None, as None can still change while syncing + if self.up_to_date or fee is not None: + tx._cached_fee = fee return fee def get_addr_io(self, address): diff --git a/electrum/base_wizard.py b/electrum/base_wizard.py index 84323226c..7efd82297 100644 --- a/electrum/base_wizard.py +++ b/electrum/base_wizard.py @@ -200,7 +200,7 @@ class BaseWizard(object): self.storage.put('keystore', k.dump()) w = Imported_Wallet(self.storage) keys = keystore.get_private_keys(text) - good_inputs, bad_inputs = w.import_private_keys(keys, None) + good_inputs, bad_inputs = w.import_private_keys(keys, None, write_to_disk=False) self.keystores.append(w.keystore) else: return self.terminate() @@ -283,7 +283,9 @@ class BaseWizard(object): for name, info in devices: state = _("initialized") if info.initialized else _("wiped") label = info.label or _("An unnamed {}").format(name) - descr = f"{label} [{name}, {state}, {info.device.transport_ui_string}]" + try: transport_str = info.device.transport_ui_string[:20] + except: transport_str = 'unknown transport' + descr = f"{label} [{name}, {state}, {transport_str}]" choices.append(((name, info), descr)) msg = _('Select a device') + ':' self.choice_dialog(title=title, message=msg, choices=choices, run_next= lambda *args: self.on_device(*args, purpose=purpose)) @@ -508,6 +510,7 @@ class BaseWizard(object): def on_password(self, password, *, encrypt_storage, storage_enc_version=STO_EV_USER_PW, encrypt_keystore): + assert not self.storage.file_exists(), "file was created too soon! plaintext keys might have been written to disk" self.storage.set_keystore_encryption(bool(password) and encrypt_keystore) if encrypt_storage: self.storage.set_password(password, enc_version=storage_enc_version) diff --git a/electrum/blockchain.py b/electrum/blockchain.py index 9f1e0c018..92a587236 100644 --- a/electrum/blockchain.py +++ b/electrum/blockchain.py @@ -79,26 +79,81 @@ def hash_raw_header(header: str) -> str: return hash_encode(sha256d(bfh(header))) -blockchains = {} # type: Dict[int, Blockchain] -blockchains_lock = threading.Lock() - - -def read_blockchains(config: 'SimpleConfig') -> Dict[int, 'Blockchain']: - blockchains[0] = Blockchain(config, 0, None) +# key: blockhash hex at forkpoint +# the chain at some key is the best chain that includes the given hash +blockchains = {} # type: Dict[str, Blockchain] +blockchains_lock = threading.RLock() + + +def read_blockchains(config: 'SimpleConfig'): + best_chain = Blockchain(config=config, + forkpoint=0, + parent=None, + forkpoint_hash=constants.net.GENESIS, + prev_hash=None) + blockchains[constants.net.GENESIS] = best_chain + # consistency checks + if best_chain.height() > constants.net.max_checkpoint(): + header_after_cp = best_chain.read_header(constants.net.max_checkpoint()+1) + if not header_after_cp or not best_chain.can_connect(header_after_cp, check_height=False): + util.print_error("[blockchain] deleting best chain. cannot connect header after last cp to last cp.") + os.unlink(best_chain.path()) + best_chain.update_size() + # forks fdir = os.path.join(util.get_headers_dir(config), 'forks') util.make_dir(fdir) - l = filter(lambda x: x.startswith('fork_'), os.listdir(fdir)) - l = sorted(l, key = lambda x: int(x.split('_')[1])) - for filename in l: - forkpoint = int(filename.split('_')[2]) - parent_id = int(filename.split('_')[1]) - b = Blockchain(config, forkpoint, parent_id) - h = b.read_header(b.forkpoint) - if b.parent().can_connect(h, check_height=False): - blockchains[b.forkpoint] = b + # files are named as: fork2_{forkpoint}_{prev_hash}_{first_hash} + l = filter(lambda x: x.startswith('fork2_') and '.' not in x, os.listdir(fdir)) + l = sorted(l, key=lambda x: int(x.split('_')[1])) # sort by forkpoint + + def delete_chain(filename, reason): + util.print_error(f"[blockchain] deleting chain {filename}: {reason}") + os.unlink(os.path.join(fdir, filename)) + + def instantiate_chain(filename): + __, forkpoint, prev_hash, first_hash = filename.split('_') + forkpoint = int(forkpoint) + prev_hash = (64-len(prev_hash)) * "0" + prev_hash # left-pad with zeroes + first_hash = (64-len(first_hash)) * "0" + first_hash + # forks below the max checkpoint are not allowed + if forkpoint <= constants.net.max_checkpoint(): + delete_chain(filename, "deleting fork below max checkpoint") + return + # find parent (sorting by forkpoint guarantees it's already instantiated) + for parent in blockchains.values(): + if parent.check_hash(forkpoint - 1, prev_hash): + break else: - util.print_error("cannot connect", filename) - return blockchains + delete_chain(filename, "cannot find parent for chain") + return + b = Blockchain(config=config, + forkpoint=forkpoint, + parent=parent, + forkpoint_hash=first_hash, + prev_hash=prev_hash) + # consistency checks + h = b.read_header(b.forkpoint) + if first_hash != hash_header(h): + delete_chain(filename, "incorrect first hash for chain") + return + if not b.parent.can_connect(h, check_height=False): + delete_chain(filename, "cannot connect chain to parent") + return + chain_id = b.get_id() + assert first_hash == chain_id, (first_hash, chain_id) + blockchains[chain_id] = b + + for filename in l: + instantiate_chain(filename) + + +def get_best_chain() -> 'Blockchain': + return blockchains[constants.net.GENESIS] + +# block hash -> chain work; up to and including that block +_CHAINWORK_CACHE = { + "0000000000000000000000000000000000000000000000000000000000000000": 0, # virtual block at height -1 +} # type: Dict[str, int] class Blockchain(util.PrintError): @@ -106,15 +161,20 @@ class Blockchain(util.PrintError): Manages blockchain headers and their verification """ - def __init__(self, config: SimpleConfig, forkpoint: int, parent_id: Optional[int]): + def __init__(self, config: SimpleConfig, forkpoint: int, parent: Optional['Blockchain'], + forkpoint_hash: str, prev_hash: Optional[str]): + assert isinstance(forkpoint_hash, str) and len(forkpoint_hash) == 64, forkpoint_hash + assert (prev_hash is None) or (isinstance(prev_hash, str) and len(prev_hash) == 64), prev_hash + # assert (parent is None) == (forkpoint == 0) + if 0 < forkpoint <= constants.net.max_checkpoint(): + raise Exception(f"cannot fork below max checkpoint. forkpoint: {forkpoint}") self.config = config - self.forkpoint = forkpoint - self.checkpoints = constants.net.CHECKPOINTS - self.parent_id = parent_id - assert parent_id != forkpoint + self.forkpoint = forkpoint # height of first header + self.parent = parent + self._forkpoint_hash = forkpoint_hash # blockhash at forkpoint. "first hash" + self._prev_hash = prev_hash # blockhash immediately before forkpoint self.lock = threading.RLock() - with self.lock: - self.update_size() + self.update_size() def with_lock(func): def func_wrapper(self, *args, **kwargs): @@ -122,12 +182,13 @@ class Blockchain(util.PrintError): return func(self, *args, **kwargs) return func_wrapper - def parent(self) -> 'Blockchain': - return blockchains[self.parent_id] + @property + def checkpoints(self): + return constants.net.CHECKPOINTS def get_max_child(self) -> Optional[int]: with blockchains_lock: chains = list(blockchains.values()) - children = list(filter(lambda y: y.parent_id==self.forkpoint, chains)) + children = list(filter(lambda y: y.parent==self, chains)) return max([x.forkpoint for x in children]) if children else None def get_max_forkpoint(self) -> int: @@ -137,11 +198,12 @@ class Blockchain(util.PrintError): mc = self.get_max_child() return mc if mc is not None else self.forkpoint + @with_lock def get_branch_size(self) -> int: return self.height() - self.get_max_forkpoint() + 1 def get_name(self) -> str: - return self.get_hash(self.get_max_forkpoint()).lstrip('00')[0:10] + return self.get_hash(self.get_max_forkpoint()).lstrip('0')[0:10] def check_header(self, header: dict) -> bool: header_hash = hash_header(header) @@ -159,24 +221,38 @@ class Blockchain(util.PrintError): return False def fork(parent, header: dict) -> 'Blockchain': + if not parent.can_connect(header, check_height=False): + raise Exception("forking header does not connect to parent chain") forkpoint = header.get('block_height') - self = Blockchain(parent.config, forkpoint, parent.forkpoint) + self = Blockchain(config=parent.config, + forkpoint=forkpoint, + parent=parent, + forkpoint_hash=hash_header(header), + prev_hash=parent.get_hash(forkpoint-1)) open(self.path(), 'w+').close() self.save_header(header) + # put into global dict. note that in some cases + # save_header might have already put it there but that's OK + chain_id = self.get_id() + with blockchains_lock: + blockchains[chain_id] = self return self + @with_lock def height(self) -> int: return self.forkpoint + self.size() - 1 + @with_lock def size(self) -> int: - with self.lock: - return self._size + return self._size + @with_lock def update_size(self) -> None: p = self.path() self._size = os.path.getsize(p)//HEADER_SIZE if os.path.exists(p) else 0 - def verify_header(self, header: dict, prev_hash: str, target: int, expected_header_hash: str=None) -> None: + @classmethod + def verify_header(cls, header: dict, prev_hash: str, target: int, expected_header_hash: str=None) -> None: _hash = hash_header(header) if expected_header_hash and expected_header_hash != _hash: raise Exception("hash mismatches with expected: {} vs {}".format(expected_header_hash, _hash)) @@ -184,7 +260,7 @@ class Blockchain(util.PrintError): raise Exception("prev hash mismatch: %s vs %s" % (prev_hash, header.get('prev_block_hash'))) if constants.net.TESTNET: return - bits = self.target_to_bits(target) + bits = cls.target_to_bits(target) if bits != header.get('bits'): raise Exception("bits mismatch: %s vs %s" % (bits, header.get('bits'))) block_hash_as_num = int.from_bytes(bfh(_hash), byteorder='big') @@ -207,21 +283,26 @@ class Blockchain(util.PrintError): self.verify_header(header, prev_hash, target, expected_header_hash) prev_hash = hash_header(header) + @with_lock def path(self): d = util.get_headers_dir(self.config) - if self.parent_id is None: + if self.parent is None: filename = 'blockchain_headers' else: - basename = 'fork_%d_%d' % (self.parent_id, self.forkpoint) + assert self.forkpoint > 0, self.forkpoint + prev_hash = self._prev_hash.lstrip('0') + first_hash = self._forkpoint_hash.lstrip('0') + basename = f'fork2_{self.forkpoint}_{prev_hash}_{first_hash}' filename = os.path.join('forks', basename) return os.path.join(d, filename) @with_lock def save_chunk(self, index: int, chunk: bytes): + assert index >= 0, index chunk_within_checkpoint_region = index < len(self.checkpoints) # chunks in checkpoint region are the responsibility of the 'main chain' - if chunk_within_checkpoint_region and self.parent_id is not None: - main_chain = blockchains[0] + if chunk_within_checkpoint_region and self.parent is not None: + main_chain = get_best_chain() main_chain.save_chunk(index, chunk) return @@ -236,18 +317,36 @@ class Blockchain(util.PrintError): self.write(chunk, delta_bytes, truncate) self.swap_with_parent() - @with_lock def swap_with_parent(self) -> None: - if self.parent_id is None: - return - parent_branch_size = self.parent().height() - self.forkpoint + 1 - if parent_branch_size >= self.size(): - return - self.print_error("swap", self.forkpoint, self.parent_id) - parent_id = self.parent_id - forkpoint = self.forkpoint - parent = self.parent() + parent_lock = self.parent.lock if self.parent is not None else threading.Lock() + with parent_lock, self.lock, blockchains_lock: # this order should not deadlock + # do the swap; possibly multiple ones + cnt = 0 + while self._swap_with_parent(): + cnt += 1 + if cnt > len(blockchains): # make sure we are making progress + raise Exception(f'swapping fork with parent too many times: {cnt}') + + def _swap_with_parent(self) -> bool: + """Check if this chain became stronger than its parent, and swap + the underlying files if so. The Blockchain instances will keep + 'containing' the same headers, but their ids change and so + they will be stored in different files.""" + if self.parent is None: + return False + if self.parent.get_chainwork() >= self.get_chainwork(): + return False + self.print_error("swap", self.forkpoint, self.parent.forkpoint) + parent_branch_size = self.parent.height() - self.forkpoint + 1 + forkpoint = self.forkpoint # type: Optional[int] + parent = self.parent # type: Optional[Blockchain] + child_old_id = self.get_id() + parent_old_id = parent.get_id() + # swap files + # child takes parent's name + # parent's new name will be something new (not child's old name) self.assert_headers_file_available(self.path()) + child_old_name = self.path() with open(self.path(), 'rb') as f: my_data = f.read() self.assert_headers_file_available(parent.path()) @@ -256,24 +355,24 @@ class Blockchain(util.PrintError): parent_data = f.read(parent_branch_size*HEADER_SIZE) self.write(parent_data, 0) parent.write(my_data, (forkpoint - parent.forkpoint)*HEADER_SIZE) - # store file path - with blockchains_lock: chains = list(blockchains.values()) - for b in chains: - b.old_path = b.path() # swap parameters - self.parent_id = parent.parent_id; parent.parent_id = parent_id - self.forkpoint = parent.forkpoint; parent.forkpoint = forkpoint - self._size = parent._size; parent._size = parent_branch_size - # move files - for b in chains: - if b in [self, parent]: continue - if b.old_path != b.path(): - self.print_error("renaming", b.old_path, b.path()) - os.rename(b.old_path, b.path()) + self.parent, parent.parent = parent.parent, self # type: Optional[Blockchain], Optional[Blockchain] + self.forkpoint, parent.forkpoint = parent.forkpoint, self.forkpoint + self._forkpoint_hash, parent._forkpoint_hash = parent._forkpoint_hash, hash_raw_header(bh2u(parent_data[:HEADER_SIZE])) + self._prev_hash, parent._prev_hash = parent._prev_hash, self._prev_hash + # parent's new name + os.replace(child_old_name, parent.path()) + self.update_size() + parent.update_size() # update pointers - with blockchains_lock: - blockchains[self.forkpoint] = self - blockchains[parent.forkpoint] = parent + blockchains.pop(child_old_id, None) + blockchains.pop(parent_old_id, None) + blockchains[self.get_id()] = self + blockchains[parent.get_id()] = parent + return True + + def get_id(self) -> str: + return self._forkpoint_hash def assert_headers_file_available(self, path): if os.path.exists(path): @@ -283,36 +382,36 @@ class Blockchain(util.PrintError): else: raise FileNotFoundError('Cannot find headers file but headers_dir is there. Should be at {}'.format(path)) + @with_lock def write(self, data: bytes, offset: int, truncate: bool=True) -> None: filename = self.path() - with self.lock: - self.assert_headers_file_available(filename) - with open(filename, 'rb+') as f: - if truncate and offset != self._size * HEADER_SIZE: - f.seek(offset) - f.truncate() + self.assert_headers_file_available(filename) + with open(filename, 'rb+') as f: + if truncate and offset != self._size * HEADER_SIZE: f.seek(offset) - f.write(data) - f.flush() - os.fsync(f.fileno()) - self.update_size() + f.truncate() + f.seek(offset) + f.write(data) + f.flush() + os.fsync(f.fileno()) + self.update_size() @with_lock def save_header(self, header: dict) -> None: delta = header.get('block_height') - self.forkpoint data = bfh(serialize_header(header)) # headers are only _appended_ to the end: - assert delta == self.size() + assert delta == self.size(), (delta, self.size()) assert len(data) == HEADER_SIZE self.write(data, delta*HEADER_SIZE) self.swap_with_parent() + @with_lock def read_header(self, height: int) -> Optional[dict]: - assert self.parent_id != self.forkpoint if height < 0: return if height < self.forkpoint: - return self.parent().read_header(height) + return self.parent.read_header(height) if height > self.height(): return delta = height - self.forkpoint @@ -372,16 +471,18 @@ class Blockchain(util.PrintError): new_target = self.bits_to_target(self.target_to_bits(new_target)) return new_target - def bits_to_target(self, bits: int) -> int: + @classmethod + def bits_to_target(cls, bits: int) -> int: bitsN = (bits >> 24) & 0xff - if not (bitsN >= 0x03 and bitsN <= 0x1d): + if not (0x03 <= bitsN <= 0x1d): raise Exception("First part of bits should be in [0x03, 0x1d]") bitsBase = bits & 0xffffff - if not (bitsBase >= 0x8000 and bitsBase <= 0x7fffff): + if not (0x8000 <= bitsBase <= 0x7fffff): raise Exception("Second part of bits should be in [0x8000, 0x7fffff]") return bitsBase << (8 * (bitsN-3)) - def target_to_bits(self, target: int) -> int: + @classmethod + def target_to_bits(cls, target: int) -> int: c = ("%064x" % target)[2:] while c[:2] == '00' and len(c) > 6: c = c[2:] @@ -391,6 +492,40 @@ class Blockchain(util.PrintError): bitsBase >>= 8 return bitsN << 24 | bitsBase + def chainwork_of_header_at_height(self, height: int) -> int: + """work done by single header at given height""" + chunk_idx = height // 2016 - 1 + target = self.get_target(chunk_idx) + work = ((2 ** 256 - target - 1) // (target + 1)) + 1 + return work + + @with_lock + def get_chainwork(self, height=None) -> int: + if height is None: + height = max(0, self.height()) + if constants.net.TESTNET: + # On testnet/regtest, difficulty works somewhat different. + # It's out of scope to properly implement that. + return height + last_retarget = height // 2016 * 2016 - 1 + cached_height = last_retarget + while _CHAINWORK_CACHE.get(self.get_hash(cached_height)) is None: + if cached_height <= -1: + break + cached_height -= 2016 + assert cached_height >= -1, cached_height + running_total = _CHAINWORK_CACHE[self.get_hash(cached_height)] + while cached_height < last_retarget: + cached_height += 2016 + work_in_single_header = self.chainwork_of_header_at_height(cached_height) + work_in_chunk = 2016 * work_in_single_header + running_total += work_in_chunk + _CHAINWORK_CACHE[self.get_hash(cached_height)] = running_total + cached_height += 2016 + work_in_single_header = self.chainwork_of_header_at_height(cached_height) + work_in_last_partial_chunk = (height % 2016 + 1) * work_in_single_header + return running_total + work_in_last_partial_chunk + def can_connect(self, header: dict, check_height: bool=True) -> bool: if header is None: return False @@ -417,6 +552,7 @@ class Blockchain(util.PrintError): return True def connect_chunk(self, idx: int, hexdata: str) -> bool: + assert idx >= 0, idx try: data = bfh(hexdata) self.verify_chunk(idx, data) @@ -424,7 +560,7 @@ class Blockchain(util.PrintError): self.save_chunk(idx, data) return True except BaseException as e: - self.print_error('verify_chunk %d failed'%idx, str(e)) + self.print_error(f'verify_chunk idx {idx} failed: {repr(e)}') return False def get_checkpoints(self): diff --git a/electrum/commands.py b/electrum/commands.py index 40a8142cf..2192d992a 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -176,7 +176,7 @@ class Commands: storage.put('keystore', k.dump()) wallet = Imported_Wallet(storage) keys = keystore.get_private_keys(text) - good_inputs, bad_inputs = wallet.import_private_keys(keys, password) + good_inputs, bad_inputs = wallet.import_private_keys(keys, None, write_to_disk=False) # FIXME tell user about bad_inputs if not good_inputs: raise Exception("None of the given privkeys can be imported") @@ -191,6 +191,7 @@ class Commands: storage.put('wallet_type', 'standard') wallet = Wallet(storage) + assert not storage.file_exists(), "file was created too soon! plaintext keys might have been written to disk" wallet.update_password(old_pw=None, new_pw=password, encrypt_storage=encrypt_file) wallet.synchronize() diff --git a/electrum/contacts.py b/electrum/contacts.py index c09b59e22..49c8087f9 100644 --- a/electrum/contacts.py +++ b/electrum/contacts.py @@ -65,8 +65,9 @@ class Contacts(dict): def pop(self, key): if key in self.keys(): - dict.pop(self, key) + res = dict.pop(self, key) self.save() + return res def resolve(self, k): if bitcoin.is_address(k): diff --git a/electrum/crypto.py b/electrum/crypto.py index 038caafce..7df21156b 100644 --- a/electrum/crypto.py +++ b/electrum/crypto.py @@ -32,6 +32,7 @@ from typing import Union import pyaes from .util import assert_bytes, InvalidPassword, to_bytes, to_string +from .i18n import _ try: @@ -90,37 +91,103 @@ def aes_decrypt_with_iv(key: bytes, iv: bytes, data: bytes) -> bytes: raise InvalidPassword() -def EncodeAES(secret: bytes, msg: bytes) -> bytes: +def EncodeAES_base64(secret: bytes, msg: bytes) -> bytes: """Returns base64 encoded ciphertext.""" + e = EncodeAES_bytes(secret, msg) + return base64.b64encode(e) + + +def EncodeAES_bytes(secret: bytes, msg: bytes) -> bytes: assert_bytes(msg) iv = bytes(os.urandom(16)) ct = aes_encrypt_with_iv(secret, iv, msg) - e = iv + ct - return base64.b64encode(e) + return iv + ct + +def DecodeAES_base64(secret: bytes, ciphertext_b64: Union[bytes, str]) -> bytes: + ciphertext = bytes(base64.b64decode(ciphertext_b64)) + return DecodeAES_bytes(secret, ciphertext) -def DecodeAES(secret: bytes, ciphertext_b64: Union[bytes, str]) -> bytes: - e = bytes(base64.b64decode(ciphertext_b64)) - iv, e = e[:16], e[16:] + +def DecodeAES_bytes(secret: bytes, ciphertext: bytes) -> bytes: + assert_bytes(ciphertext) + iv, e = ciphertext[:16], ciphertext[16:] s = aes_decrypt_with_iv(secret, iv, e) return s -def pw_encode(data: str, password: Union[bytes, str]) -> str: +PW_HASH_VERSION_LATEST = 2 +KNOWN_PW_HASH_VERSIONS = (1, 2) +assert PW_HASH_VERSION_LATEST in KNOWN_PW_HASH_VERSIONS + + +class UnexpectedPasswordHashVersion(InvalidPassword): + def __init__(self, version): + self.version = version + + def __str__(self): + return "{unexpected}: {version}\n{please_update}".format( + unexpected=_("Unexpected password hash version"), + version=self.version, + please_update=_('You are most likely using an outdated version of Electrum. Please update.')) + + +def _hash_password(password: Union[bytes, str], *, version: int, salt: bytes) -> bytes: + pw = to_bytes(password, 'utf8') + if version == 1: + return sha256d(pw) + elif version == 2: + if not isinstance(salt, bytes) or len(salt) < 16: + raise Exception('too weak salt', salt) + return hashlib.pbkdf2_hmac(hash_name='sha256', + password=pw, + salt=b'ELECTRUM_PW_HASH_V2'+salt, + iterations=50_000) + else: + assert version not in KNOWN_PW_HASH_VERSIONS + raise UnexpectedPasswordHashVersion(version) + + +def pw_encode(data: str, password: Union[bytes, str, None], *, version: int) -> str: if not password: return data - secret = sha256d(password) - return EncodeAES(secret, to_bytes(data, "utf8")).decode('utf8') + if version not in KNOWN_PW_HASH_VERSIONS: + raise UnexpectedPasswordHashVersion(version) + # derive key from password + if version == 1: + salt = b'' + elif version == 2: + salt = bytes(os.urandom(16)) + else: + assert False, version + secret = _hash_password(password, version=version, salt=salt) + # encrypt given data + e = EncodeAES_bytes(secret, to_bytes(data, "utf8")) + # return base64(salt + encrypted data) + ciphertext = salt + e + ciphertext_b64 = base64.b64encode(ciphertext) + return ciphertext_b64.decode('utf8') -def pw_decode(data: str, password: Union[bytes, str]) -> str: +def pw_decode(data: str, password: Union[bytes, str, None], *, version: int) -> str: if password is None: return data - secret = sha256d(password) + if version not in KNOWN_PW_HASH_VERSIONS: + raise UnexpectedPasswordHashVersion(version) + data_bytes = bytes(base64.b64decode(data)) + # derive key from password + if version == 1: + salt = b'' + elif version == 2: + salt, data_bytes = data_bytes[:16], data_bytes[16:] + else: + assert False, version + secret = _hash_password(password, version=version, salt=salt) + # decrypt given data try: - d = to_string(DecodeAES(secret, data), "utf8") - except Exception: - raise InvalidPassword() + d = to_string(DecodeAES_bytes(secret, data_bytes), "utf8") + except Exception as e: + raise InvalidPassword() from e return d diff --git a/electrum/exchange_rate.py b/electrum/exchange_rate.py index ae57cab73..4f3e9cb65 100644 --- a/electrum/exchange_rate.py +++ b/electrum/exchange_rate.py @@ -464,9 +464,13 @@ class FxThread(ThreadJob): d = get_exchanges_by_ccy(history) return d.get(ccy, []) + @staticmethod + def remove_thousands_separator(text): + return text.replace(',', '') # FIXME use THOUSAND_SEPARATOR in util + def ccy_amount_str(self, amount, commas): prec = CCY_PRECISIONS.get(self.ccy, 2) - fmt_str = "{:%s.%df}" % ("," if commas else "", max(0, prec)) + fmt_str = "{:%s.%df}" % ("," if commas else "", max(0, prec)) # FIXME use util.THOUSAND_SEPARATOR and util.DECIMAL_POINT try: rounded_amount = round(amount, prec) except decimal.InvalidOperation: diff --git a/electrum/gui/kivy/Makefile b/electrum/gui/kivy/Makefile index 7e87afa6b..868a774a0 100644 --- a/electrum/gui/kivy/Makefile +++ b/electrum/gui/kivy/Makefile @@ -11,7 +11,7 @@ prepare: @cp tools/buildozer.spec ../../../buildozer.spec # copy electrum to main.py @cp ../../../run_electrum ../../../main.py - @-if [ ! -d "../../.buildozer" ];then \ + @-if [ ! -d "../../../.buildozer" ];then \ cd ../../..; buildozer android debug;\ cp -f electrum/gui/kivy/tools/blacklist.txt .buildozer/android/platform/python-for-android/src/blacklist.txt;\ rm -rf ./.buildozer/android/platform/python-for-android/dist;\ diff --git a/electrum/gui/kivy/Readme.md b/electrum/gui/kivy/Readme.md index cbafb938f..f394ddf13 100644 --- a/electrum/gui/kivy/Readme.md +++ b/electrum/gui/kivy/Readme.md @@ -3,147 +3,62 @@ The Kivy GUI is used with Electrum on Android devices. To generate an APK file, follow these instructions. -Recommended env: Ubuntu 18.04 +## Android binary with Docker -## 1. Preliminaries +This assumes an Ubuntu host, but it should not be too hard to adapt to another +similar system. The docker commands should be executed in the project's root +folder. -Make sure the current user can write `/opt` (e.g. `sudo chown username: /opt`). +1. Install Docker -We assume that you already got Electrum to run from source on this machine, -hence have e.g. `git`, `python3-pip` and `python3-setuptools`. + ``` + $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + $ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + $ sudo apt-get update + $ sudo apt-get install -y docker-ce + ``` -## 2. Install kivy +2. Build image -Install kivy for python3 as described [here](https://kivy.org/docs/installation/installation-linux.html). -So for example: -```sh -sudo add-apt-repository ppa:kivy-team/kivy -sudo apt-get install python3-kivy -``` - - -## 3. Install python-for-android (p4a) -p4a is used to package Electrum, Python, SDL and a bootstrap Java app into an APK file. -We need some functionality not in p4a master, so for the time being we have our own fork. - -Something like this should work: - -```sh -cd /opt -git clone https://github.com/kivy/python-for-android -cd python-for-android -git remote add sombernight https://github.com/SomberNight/python-for-android -git fetch --all -git checkout f74226666af69f9915afaee9ef9292db85a6c617 -``` + ``` + $ sudo docker build -t electrum-android-builder-img electrum/gui/kivy/tools + ``` -## 4. Install buildozer -4.1 Buildozer is a frontend to p4a. Luckily we don't need to patch it: +3. Prepare pure python dependencies -```sh -cd /opt -git clone https://github.com/kivy/buildozer -cd buildozer -sudo python3 setup.py install -``` + ``` + $ sudo ./contrib/make_packages + ``` -4.2 Install additional dependencies: +4. Build binaries -```sh -sudo apt-get install python-pip -``` - -(from [buildozer docs](https://buildozer.readthedocs.io/en/latest/installation.html#targeting-android)) -```sh -sudo pip install --upgrade cython==0.21 -sudo dpkg --add-architecture i386 -sudo apt-get update -sudo apt-get install build-essential ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 -``` - -4.3 Download Android NDK -```sh -cd /opt -wget https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip -unzip android-ndk-r14b-linux-x86_64.zip -``` - -## 5. Some more dependencies - -```sh -python3 -m pip install colorama appdirs sh jinja2 cython==0.29 -sudo apt-get install autotools-dev autoconf libtool pkg-config python3.7 -``` - - -## 6. Create the UI Atlas -In the `electrum/gui/kivy` directory of Electrum, run `make theming`. - -## 7. Download Electrum dependencies -```sh -sudo contrib/make_packages -``` + ``` + $ sudo docker run -it --rm \ + --name electrum-android-builder-cont \ + -v $PWD:/home/user/wspace/electrum \ + -v ~/.keystore:/home/user/.keystore \ + --workdir /home/user/wspace/electrum \ + electrum-android-builder-img \ + ./contrib/make_apk + ``` + This mounts the project dir inside the container, + and so the modifications will affect it, e.g. `.buildozer` folder + will be created. -## 8. Try building the APK and fail +5. The generated binary is in `./bin`. -### 1. Try and fail: -```sh -contrib/make_apk -``` -Symlink android tools: +## FAQ -```sh -ln -sf ~/.buildozer/android/platform/android-sdk-24/tools ~/.buildozer/android/platform/android-sdk-24/tools.save -``` +### I changed something but I don't see any differences on the phone. What did I do wrong? +You probably need to clear the cache: `rm -rf .buildozer/android/platform/build/{build,dists}` -### 2. Try and fail: -```sh -contrib/make_apk +### How do I get an interactive shell inside docker? ``` - -During this build attempt, buildozer downloaded some tools, -e.g. those needed in the next step. - -## 9. Update the Android SDK build tools - -### Method 1: Using the GUI - - Start the Android SDK manager in GUI mode: - - ~/.buildozer/android/platform/android-sdk-24/tools/android - - Check the latest SDK available and install it - ("Android SDK Tools" and "Android SDK Platform-tools"). - Close the SDK manager. Repeat until there is no newer version. - - Reopen the SDK manager, and install the latest build tools - ("Android SDK Build-tools"), 28.0.3 at the time of writing. - - Install "Android 9">"SDK Platform". - Install "Android Support Repository" from the SDK manager (under "Extras"). - -### Method 2: Using the command line: - - Repeat the following command until there is nothing to install: - - ~/.buildozer/android/platform/android-sdk-24/tools/android update sdk -u -t tools,platform-tools - - Install Build Tools, android API 19 and Android Support Library: - - ~/.buildozer/android/platform/android-sdk-24/tools/android update sdk -u -t build-tools-28.0.3,android-28,extra-android-m2repository - - (FIXME: build-tools is not getting installed?! use GUI for now.) - -## 10. Build the APK - -```sh -contrib/make_apk +$ sudo docker run -it --rm \ + -v $PWD:/home/user/wspace/electrum \ + --workdir /home/user/wspace/electrum \ + electrum-android-builder-img ``` - -# FAQ - -## I changed something but I don't see any differences on the phone. What did I do wrong? -You probably need to clear the cache: `rm -rf .buildozer/android/platform/build/{build,dists}` diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index c34e5ef9f..811dda6f6 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -126,10 +126,12 @@ class ElectrumWindow(App): chains = self.network.get_blockchains() def cb(name): with blockchain.blockchains_lock: blockchain_items = list(blockchain.blockchains.items()) - for index, b in blockchain_items: + for chain_id, b in blockchain_items: if name == b.get_name(): - self.network.run_from_another_thread(self.network.follow_chain_given_id(index)) - names = [blockchain.blockchains[b].get_name() for b in chains] + self.network.run_from_another_thread(self.network.follow_chain_given_id(chain_id)) + chain_objects = [blockchain.blockchains.get(chain_id) for chain_id in chains] + chain_objects = filter(lambda b: b is not None, chain_objects) + names = [b.get_name() for b in chain_objects] if len(names) > 1: cur_chain = self.network.blockchain().get_name() ChoiceDialog(_('Choose your chain'), names, cur_chain, cb).open() @@ -171,6 +173,7 @@ class ElectrumWindow(App): def on_history(self, d): Logger.info("on_history") + self.wallet.clear_coin_price_cache() self._trigger_update_history() def on_fee_histogram(self, *args): diff --git a/electrum/gui/kivy/tools/Dockerfile b/electrum/gui/kivy/tools/Dockerfile new file mode 100644 index 000000000..0300d8620 --- /dev/null +++ b/electrum/gui/kivy/tools/Dockerfile @@ -0,0 +1,142 @@ +# based on https://github.com/kivy/python-for-android/blob/master/Dockerfile + +FROM ubuntu:18.04 + +ENV ANDROID_HOME="/opt/android" + +RUN apt -y update -qq \ + && apt -y install -qq --no-install-recommends curl unzip git python3-pip python3-setuptools \ + && apt -y autoremove \ + && apt -y clean + + +ENV ANDROID_NDK_HOME="${ANDROID_HOME}/android-ndk" +ENV ANDROID_NDK_VERSION="14b" +ENV ANDROID_NDK_HOME_V="${ANDROID_NDK_HOME}-r${ANDROID_NDK_VERSION}" + +# get the latest version from https://developer.android.com/ndk/downloads/index.html +ENV ANDROID_NDK_ARCHIVE="android-ndk-r${ANDROID_NDK_VERSION}-linux-x86_64.zip" +ENV ANDROID_NDK_DL_URL="https://dl.google.com/android/repository/${ANDROID_NDK_ARCHIVE}" + +# download and install Android NDK +RUN curl --location --progress-bar \ + "${ANDROID_NDK_DL_URL}" \ + --output "${ANDROID_NDK_ARCHIVE}" \ + && mkdir --parents "${ANDROID_NDK_HOME_V}" \ + && unzip -q "${ANDROID_NDK_ARCHIVE}" -d "${ANDROID_HOME}" \ + && ln -sfn "${ANDROID_NDK_HOME_V}" "${ANDROID_NDK_HOME}" \ + && rm -rf "${ANDROID_NDK_ARCHIVE}" + + +ENV ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk" + +# get the latest version from https://developer.android.com/studio/index.html +ENV ANDROID_SDK_TOOLS_VERSION="4333796" +ENV ANDROID_SDK_TOOLS_ARCHIVE="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip" +ENV ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}" + +# download and install Android SDK +RUN curl --location --progress-bar \ + "${ANDROID_SDK_TOOLS_DL_URL}" \ + --output "${ANDROID_SDK_TOOLS_ARCHIVE}" \ + && mkdir --parents "${ANDROID_SDK_HOME}" \ + && unzip -q "${ANDROID_SDK_TOOLS_ARCHIVE}" -d "${ANDROID_SDK_HOME}" \ + && rm -rf "${ANDROID_SDK_TOOLS_ARCHIVE}" + +# update Android SDK, install Android API, Build Tools... +RUN mkdir --parents "${ANDROID_SDK_HOME}/.android/" \ + && echo '### User Sources for Android SDK Manager' \ + > "${ANDROID_SDK_HOME}/.android/repositories.cfg" + +# accept Android licenses (JDK necessary!) +RUN apt -y update -qq \ + && apt -y install -qq --no-install-recommends openjdk-8-jdk \ + && apt -y autoremove \ + && apt -y clean +RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" --licenses > /dev/null + +# download platforms, API, build tools +RUN "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-24" && \ + "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-28" && \ + "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;28.0.3" && \ + "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "extras;android;m2repository" && \ + chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager" + + +ENV USER="user" +ENV HOME_DIR="/home/${USER}" +ENV WORK_DIR="${HOME_DIR}/wspace" \ + PATH="${HOME_DIR}/.local/bin:${PATH}" + +# install system dependencies +RUN apt -y update -qq \ + && apt -y install -qq --no-install-recommends \ + python virtualenv python-pip wget lbzip2 patch sudo \ + software-properties-common + +# install kivy +RUN add-apt-repository ppa:kivy-team/kivy \ + && apt -y update -qq \ + && apt -y install -qq --no-install-recommends python3-kivy \ + && apt -y autoremove \ + && apt -y clean +RUN python3 -m pip install image + +# build dependencies +# https://buildozer.readthedocs.io/en/latest/installation.html#android-on-ubuntu-16-04-64bit +RUN dpkg --add-architecture i386 \ + && apt -y update -qq \ + && apt -y install -qq --no-install-recommends \ + build-essential ccache git python2.7 python2.7-dev \ + libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \ + libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 \ + zip zlib1g-dev zlib1g:i386 \ + && apt -y autoremove \ + && apt -y clean + +# specific recipes dependencies (e.g. libffi requires autoreconf binary) +RUN apt -y update -qq \ + && apt -y install -qq --no-install-recommends \ + autoconf automake cmake gettext libltdl-dev libtool pkg-config \ + python3.7 \ + && apt -y autoremove \ + && apt -y clean + + +# prepare non root env +RUN useradd --create-home --shell /bin/bash ${USER} + +# with sudo access and no password +RUN usermod -append --groups sudo ${USER} +RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + + +WORKDIR ${WORK_DIR} + +# user needs ownership/write access to these directories +RUN chown --recursive ${USER} ${WORK_DIR} ${ANDROID_SDK_HOME} +RUN chown ${USER} /opt +USER ${USER} + + +RUN pip install --upgrade cython==0.29 +RUN python3 -m pip install --upgrade cython==0.29 + +# install buildozer +RUN cd /opt \ + && git clone https://github.com/kivy/buildozer \ + && cd buildozer \ + && python3 -m pip install -e . + +# install python-for-android +RUN cd /opt \ + && git clone https://github.com/kivy/python-for-android \ + && cd python-for-android \ + && git remote add sombernight https://github.com/SomberNight/python-for-android \ + && git fetch --all \ + && git checkout f74226666af69f9915afaee9ef9292db85a6c617 \ + && python3 -m pip install -e . + +# build env vars +ENV USE_SDK_WRAPPER=1 +ENV GRADLE_OPTS="-Xmx1536M -Dorg.gradle.jvmargs='-Xmx1536M'" diff --git a/electrum/gui/kivy/tools/buildozer.spec b/electrum/gui/kivy/tools/buildozer.spec index dca6daab8..2c4e77454 100644 --- a/electrum/gui/kivy/tools/buildozer.spec +++ b/electrum/gui/kivy/tools/buildozer.spec @@ -70,10 +70,10 @@ android.ndk = 14b android.private_storage = True # (str) Android NDK directory (if empty, it will be automatically downloaded.) -android.ndk_path = /opt/android-ndk-r14b +android.ndk_path = /opt/android/android-ndk # (str) Android SDK directory (if empty, it will be automatically downloaded.) -#android.sdk_path = +android.sdk_path = /opt/android/android-sdk # (str) Android entry point, default is ok for Kivy-based app #android.entrypoint = org.renpy.android.PythonActivity diff --git a/electrum/gui/qt/__init__.py b/electrum/gui/qt/__init__.py index 9da968f30..33ecd2920 100644 --- a/electrum/gui/qt/__init__.py +++ b/electrum/gui/qt/__init__.py @@ -87,7 +87,7 @@ class ElectrumGui(PrintError): @profiler def __init__(self, config, daemon, plugins): - set_language(config.get('language')) + set_language(config.get('language', get_default_language())) # Uncomment this call to verify objects are being properly # GC-ed when windows are closed #network.add_jobs([DebugMem([Abstract_Wallet, SPV, Synchronizer, diff --git a/electrum/gui/qt/address_list.py b/electrum/gui/qt/address_list.py index fce1004c2..019588102 100644 --- a/electrum/gui/qt/address_list.py +++ b/electrum/gui/qt/address_list.py @@ -31,13 +31,11 @@ from electrum.bitcoin import is_address from .util import * - -class AddressList(MyTreeWidget): +class AddressList(MyTreeView): filter_columns = [0, 1, 2, 3] # Type, Address, Label, Balance def __init__(self, parent=None): - MyTreeWidget.__init__(self, parent, self.create_menu, [], 2) - self.refresh_headers() + super().__init__(parent, self.create_menu, 2) self.setSelectionMode(QAbstractItemView.ExtendedSelection) self.setSortingEnabled(True) self.show_change = 0 @@ -50,6 +48,8 @@ class AddressList(MyTreeWidget): self.used_button.currentIndexChanged.connect(self.toggle_used) for t in [_('All'), _('Unused'), _('Funded'), _('Used')]: self.used_button.addItem(t) + self.setModel(QStandardItemModel(self)) + self.update() def get_toolbar_buttons(self): return QLabel(_("Filter:")), self.change_button, self.used_button @@ -82,18 +82,19 @@ class AddressList(MyTreeWidget): self.show_used = state self.update() - def on_update(self): + def update(self): self.wallet = self.parent.wallet - item = self.currentItem() - current_address = item.data(0, Qt.UserRole) if item else None + current_address = self.current_item_user_role(col=2) if self.show_change == 1: addr_list = self.wallet.get_receiving_addresses() elif self.show_change == 2: addr_list = self.wallet.get_change_addresses() else: addr_list = self.wallet.get_addresses() - self.clear() + self.model().clear() + self.refresh_headers() fx = self.parent.fx + set_address = None for address in addr_list: num = self.wallet.get_address_history_len(address) label = self.wallet.labels.get(address, '') @@ -111,61 +112,66 @@ class AddressList(MyTreeWidget): if fx and fx.get_fiat_address_config(): rate = fx.exchange_rate() fiat_balance = fx.value_str(balance, rate) - address_item = SortableTreeWidgetItem(['', address, label, balance_text, fiat_balance, "%d"%num]) + labels = ['', address, label, balance_text, fiat_balance, "%d"%num] + address_item = [QStandardItem(e) for e in labels] else: - address_item = SortableTreeWidgetItem(['', address, label, balance_text, "%d"%num]) + labels = ['', address, label, balance_text, "%d"%num] + address_item = [QStandardItem(e) for e in labels] # align text and set fonts - for i in range(address_item.columnCount()): - address_item.setTextAlignment(i, Qt.AlignVCenter) + for i, item in enumerate(address_item): + item.setTextAlignment(Qt.AlignVCenter) if i not in (0, 2): - address_item.setFont(i, QFont(MONOSPACE_FONT)) + item.setFont(QFont(MONOSPACE_FONT)) + item.setEditable(i in self.editable_columns) if fx and fx.get_fiat_address_config(): - address_item.setTextAlignment(4, Qt.AlignRight | Qt.AlignVCenter) + address_item[4].setTextAlignment(Qt.AlignRight | Qt.AlignVCenter) # setup column 0 if self.wallet.is_change(address): - address_item.setText(0, _('change')) - address_item.setBackground(0, ColorScheme.YELLOW.as_color(True)) + address_item[0].setText(_('change')) + address_item[0].setBackground(ColorScheme.YELLOW.as_color(True)) else: - address_item.setText(0, _('receiving')) - address_item.setBackground(0, ColorScheme.GREEN.as_color(True)) - address_item.setData(0, Qt.UserRole, address) # column 0; independent from address column + address_item[0].setText(_('receiving')) + address_item[0].setBackground(ColorScheme.GREEN.as_color(True)) + address_item[2].setData(address, Qt.UserRole) # setup column 1 if self.wallet.is_frozen(address): - address_item.setBackground(1, ColorScheme.BLUE.as_color(True)) + address_item[1].setBackground(ColorScheme.BLUE.as_color(True)) if self.wallet.is_beyond_limit(address): - address_item.setBackground(1, ColorScheme.RED.as_color(True)) + address_item[1].setBackground(ColorScheme.RED.as_color(True)) # add item - self.addChild(address_item) + count = self.model().rowCount() + self.model().insertRow(count, address_item) + address_idx = self.model().index(count, 2) if address == current_address: - self.setCurrentItem(address_item) + set_address = QPersistentModelIndex(address_idx) + self.set_current_idx(set_address) def create_menu(self, position): from electrum.wallet import Multisig_Wallet is_multisig = isinstance(self.wallet, Multisig_Wallet) can_delete = self.wallet.can_delete_address() - selected = self.selectedItems() + selected = self.selected_in_column(1) multi_select = len(selected) > 1 - addrs = [item.text(1) for item in selected] - if not addrs: - return + addrs = [self.model().itemFromIndex(item).text() for item in selected] if not multi_select: - item = self.itemAt(position) - col = self.currentColumn() + idx = self.indexAt(position) + col = idx.column() + item = self.model().itemFromIndex(idx) if not item: return addr = addrs[0] - if not is_address(addr): - item.setExpanded(not item.isExpanded()) - return menu = QMenu() if not multi_select: - column_title = self.headerItem().text(col) - copy_text = item.text(col) + addr_column_title = self.model().horizontalHeaderItem(2).text() + addr_idx = idx.sibling(idx.row(), 2) + + column_title = self.model().horizontalHeaderItem(col).text() + copy_text = self.model().itemFromIndex(idx).text() menu.addAction(_("Copy {}").format(column_title), lambda: self.parent.app.clipboard().setText(copy_text)) menu.addAction(_('Details'), lambda: self.parent.show_address(addr)) - if col in self.editable_columns: - menu.addAction(_("Edit {}").format(column_title), lambda: self.editItem(item, col)) + persistent = QPersistentModelIndex(addr_idx) + menu.addAction(_("Edit {}").format(addr_column_title), lambda p=persistent: self.edit(QModelIndex(p))) menu.addAction(_("Request payment"), lambda: self.parent.receive_at(addr)) if self.wallet.can_export(): menu.addAction(_("Private key"), lambda: self.parent.show_private_key(addr)) @@ -189,7 +195,3 @@ class AddressList(MyTreeWidget): run_hook('receive_menu', menu, addrs, self.wallet) menu.exec_(self.viewport().mapToGlobal(position)) - - def on_permit_edit(self, item, column): - # labels for headings, e.g. "receiving" or "used" should not be editable - return item.childCount() == 0 diff --git a/electrum/gui/qt/contact_list.py b/electrum/gui/qt/contact_list.py index d85c6df5c..e1915b1b6 100644 --- a/electrum/gui/qt/contact_list.py +++ b/electrum/gui/qt/contact_list.py @@ -34,67 +34,81 @@ from electrum.bitcoin import is_address from electrum.util import block_explorer_URL from electrum.plugin import run_hook -from .util import MyTreeWidget, import_meta_gui, export_meta_gui +from .util import MyTreeView, import_meta_gui, export_meta_gui -class ContactList(MyTreeWidget): +class ContactList(MyTreeView): filter_columns = [0, 1] # Key, Value def __init__(self, parent): - MyTreeWidget.__init__(self, parent, self.create_menu, [_('Name'), _('Address')], 0, [0]) + super().__init__(parent, self.create_menu, stretch_column=0, editable_columns=[0]) + self.setModel(QStandardItemModel(self)) self.setSelectionMode(QAbstractItemView.ExtendedSelection) self.setSortingEnabled(True) + self.update() - def on_permit_edit(self, item, column): - # openalias items shouldn't be editable - return item.text(1) != "openalias" + def on_edited(self, idx, user_role, text): + _type, prior_name = self.parent.contacts.pop(user_role) - def on_edited(self, item, column, prior): - if column == 0: # Remove old contact if renamed - self.parent.contacts.pop(prior) - self.parent.set_contact(item.text(0), item.text(1)) + # TODO when min Qt >= 5.11, use siblingAtColumn + col_1_sibling = idx.sibling(idx.row(), 1) + col_1_item = self.model().itemFromIndex(col_1_sibling) + + self.parent.set_contact(text, col_1_item.text()) def import_contacts(self): - import_meta_gui(self.parent, _('contacts'), self.parent.contacts.import_file, self.on_update) + import_meta_gui(self.parent, _('contacts'), self.parent.contacts.import_file, self.update) def export_contacts(self): export_meta_gui(self.parent, _('contacts'), self.parent.contacts.export_file) def create_menu(self, position): menu = QMenu() - selected = self.selectedItems() - if not selected: + selected = self.selected_in_column(0) + selected_keys = [] + for idx in selected: + sel_key = self.model().itemFromIndex(idx).data(Qt.UserRole) + selected_keys.append(sel_key) + idx = self.indexAt(position) + if not selected or not idx.isValid(): menu.addAction(_("New contact"), lambda: self.parent.new_contact_dialog()) menu.addAction(_("Import file"), lambda: self.import_contacts()) menu.addAction(_("Export file"), lambda: self.export_contacts()) else: - names = [item.text(0) for item in selected] - keys = [item.text(1) for item in selected] - column = self.currentColumn() - column_title = self.headerItem().text(column) - column_data = '\n'.join([item.text(column) for item in selected]) + column = idx.column() + column_title = self.model().horizontalHeaderItem(column).text() + column_data = '\n'.join(self.model().itemFromIndex(idx).text() for idx in selected) menu.addAction(_("Copy {}").format(column_title), lambda: self.parent.app.clipboard().setText(column_data)) if column in self.editable_columns: - item = self.currentItem() - menu.addAction(_("Edit {}").format(column_title), lambda: self.editItem(item, column)) - menu.addAction(_("Pay to"), lambda: self.parent.payto_contacts(keys)) - menu.addAction(_("Delete"), lambda: self.parent.delete_contacts(keys)) - URLs = [block_explorer_URL(self.config, 'addr', key) for key in filter(is_address, keys)] + item = self.model().itemFromIndex(idx) + if item.isEditable(): + # would not be editable if openalias + persistent = QPersistentModelIndex(idx) + menu.addAction(_("Edit {}").format(column_title), lambda p=persistent: self.edit(QModelIndex(p))) + menu.addAction(_("Pay to"), lambda: self.parent.payto_contacts(selected_keys)) + menu.addAction(_("Delete"), lambda: self.parent.delete_contacts(selected_keys)) + URLs = [block_explorer_URL(self.config, 'addr', key) for key in filter(is_address, selected_keys)] if URLs: menu.addAction(_("View on block explorer"), lambda: map(webbrowser.open, URLs)) - run_hook('create_contact_menu', menu, selected) + run_hook('create_contact_menu', menu, selected_keys) menu.exec_(self.viewport().mapToGlobal(position)) - def on_update(self): - item = self.currentItem() - current_key = item.data(0, Qt.UserRole) if item else None - self.clear() + def update(self): + current_key = self.current_item_user_role(col=0) + self.model().clear() + self.update_headers([_('Name'), _('Address')]) + set_current = None for key in sorted(self.parent.contacts.keys()): - _type, name = self.parent.contacts[key] - item = QTreeWidgetItem([name, key]) - item.setData(0, Qt.UserRole, key) - self.addTopLevelItem(item) + contact_type, name = self.parent.contacts[key] + items = [QStandardItem(x) for x in (name, key)] + items[0].setEditable(contact_type != 'openalias') + items[1].setEditable(False) + items[0].setData(key, Qt.UserRole) + row_count = self.model().rowCount() + self.model().insertRow(row_count, items) if key == current_key: - self.setCurrentItem(item) + idx = self.model().index(row_count, 0) + set_current = QPersistentModelIndex(idx) + self.set_current_idx(set_current) run_hook('update_contacts_tab', self) diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index eb410d953..1ceb646b1 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -27,10 +27,11 @@ import webbrowser import datetime from datetime import date from typing import TYPE_CHECKING +from collections import OrderedDict from electrum.address_synchronizer import TX_HEIGHT_LOCAL from electrum.i18n import _ -from electrum.util import block_explorer_URL, profiler, print_error, TxMinedStatus +from electrum.util import block_explorer_URL, profiler, print_error, TxMinedStatus, Fiat from .util import * @@ -57,38 +58,109 @@ TX_ICONS = [ "confirmed.png", ] - -class HistoryList(MyTreeWidget, AcceptFileDragDrop): - filter_columns = [2, 3, 4] # Date, Description, Amount +class HistorySortModel(QSortFilterProxyModel): + def lessThan(self, source_left: QModelIndex, source_right: QModelIndex): + item1 = self.sourceModel().itemFromIndex(source_left) + item2 = self.sourceModel().itemFromIndex(source_right) + data1 = item1.data(HistoryList.SORT_ROLE) + data2 = item2.data(HistoryList.SORT_ROLE) + if data1 is not None and data2 is not None: + return data1 < data2 + return item1.text() < item2.text() + +class HistoryList(MyTreeView, AcceptFileDragDrop): + filter_columns = [1, 2, 3] # Date, Description, Amount + TX_HASH_ROLE = Qt.UserRole + SORT_ROLE = Qt.UserRole + 1 + + def should_hide(self, proxy_row): + if self.start_timestamp and self.end_timestamp: + item = self.item_from_coordinate(proxy_row, 0) + txid = item.data(self.TX_HASH_ROLE) + date = self.transactions[txid]['date'] + if date: + in_interval = self.start_timestamp <= date <= self.end_timestamp + if not in_interval: + return True + return False def __init__(self, parent=None): - MyTreeWidget.__init__(self, parent, self.create_menu, [], 3) + super().__init__(parent, self.create_menu, 2) + self.std_model = QStandardItemModel(self) + self.proxy = HistorySortModel(self) + self.proxy.setSourceModel(self.std_model) + self.setModel(self.proxy) + + self.txid_to_items = {} + self.transactions = OrderedDict() + self.summary = {} + self.blue_brush = QBrush(QColor("#1E1EFF")) + self.red_brush = QBrush(QColor("#BC1E1E")) + self.monospace_font = QFont(MONOSPACE_FONT) + self.config = parent.config AcceptFileDragDrop.__init__(self, ".txn") - self.refresh_headers() - self.setColumnHidden(1, True) self.setSortingEnabled(True) - self.sortByColumn(0, Qt.AscendingOrder) self.start_timestamp = None self.end_timestamp = None self.years = [] self.create_toolbar_buttons() self.wallet = None + root = self.std_model.invisibleRootItem() + + self.wallet = self.parent.wallet # type: Abstract_Wallet + fx = self.parent.fx + r = self.wallet.get_full_history(domain=self.get_domain(), from_timestamp=None, to_timestamp=None, fx=fx) + self.transactions.update([(x['txid'], x) for x in r['transactions']]) + self.summary = r['summary'] + if not self.years and self.transactions: + start_date = next(iter(self.transactions.values())).get('date') or date.today() + end_date = next(iter(reversed(self.transactions.values()))).get('date') or date.today() + self.years = [str(i) for i in range(start_date.year, end_date.year + 1)] + self.period_combo.insertItems(1, self.years) + if fx: fx.history_used_spot = False + self.refresh_headers() + for tx_item in self.transactions.values(): + self.insert_tx(tx_item) + self.sortByColumn(0, Qt.AscendingOrder) + + #def on_activated(self, idx: QModelIndex): + # # TODO use siblingAtColumn when min Qt version is >=5.11 + # self.edit(idx.sibling(idx.row(), 2)) + def format_date(self, d): return str(datetime.date(d.year, d.month, d.day)) if d else _('None') def refresh_headers(self): - headers = ['', '', _('Date'), _('Description'), _('Amount'), _('Balance')] + headers = ['', _('Date'), _('Description'), _('Amount'), _('Balance')] fx = self.parent.fx if fx and fx.show_history(): headers.extend(['%s '%fx.ccy + _('Value')]) - self.editable_columns |= {6} + self.editable_columns |= {5} if fx.get_history_capital_gains_config(): headers.extend(['%s '%fx.ccy + _('Acquisition price')]) headers.extend(['%s '%fx.ccy + _('Capital Gains')]) else: - self.editable_columns -= {6} - self.update_headers(headers) + self.editable_columns -= {5} + col_count = self.std_model.columnCount() + diff = col_count-len(headers) + grew = False + if col_count > len(headers): + if diff == 2: + self.std_model.removeColumns(6, diff) + else: + assert diff in [1, 3] + self.std_model.removeColumns(5, diff) + for items in self.txid_to_items.values(): + while len(items) > col_count: + items.pop() + elif col_count < len(headers): + grew = True + self.std_model.clear() + self.txid_to_items.clear() + self.transactions.clear() + self.summary.clear() + self.update_headers(headers, self.std_model) def get_domain(self): '''Replaced in address_dialog.py''' @@ -109,13 +181,11 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): year = int(s) except: return - start_date = datetime.datetime(year, 1, 1) - end_date = datetime.datetime(year+1, 1, 1) - self.start_timestamp = time.mktime(start_date.timetuple()) - self.end_timestamp = time.mktime(end_date.timetuple()) + self.start_timestamp = start_date = datetime.datetime(year, 1, 1) + self.end_timestamp = end_date = datetime.datetime(year+1, 1, 1) self.start_button.setText(_('From') + ' ' + self.format_date(start_date)) self.end_button.setText(_('To') + ' ' + self.format_date(end_date)) - self.update() + self.hide_rows() def create_toolbar_buttons(self): self.period_combo = QComboBox() @@ -134,18 +204,18 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): def on_hide_toolbar(self): self.start_timestamp = None self.end_timestamp = None - self.update() + self.hide_rows() def save_toolbar_state(self, state, config): config.set_key('show_toolbar_history', state) def select_start_date(self): self.start_timestamp = self.select_date(self.start_button) - self.update() + self.hide_rows() def select_end_date(self): self.end_timestamp = self.select_date(self.end_button) - self.update() + self.hide_rows() def select_date(self, button): d = WindowModalDialog(self, _("Select date")) @@ -165,7 +235,7 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): return None date = d.date.toPyDate() button.setText(self.format_date(date)) - return time.mktime(date.timetuple()) + return datetime.datetime(date.year, date.month, date.day) def show_summary(self): h = self.summary @@ -213,94 +283,167 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): _("Perhaps some dependencies are missing...") + " (matplotlib?)") return try: - plt = plot_history(self.transactions) + plt = plot_history(list(self.transactions.values())) plt.show() except NothingToPlotException as e: self.parent.show_message(str(e)) + def insert_tx(self, tx_item): + fx = self.parent.fx + tx_hash = tx_item['txid'] + height = tx_item['height'] + conf = tx_item['confirmations'] + timestamp = tx_item['timestamp'] + value = tx_item['value'].value + balance = tx_item['balance'].value + label = tx_item['label'] + tx_mined_status = TxMinedStatus(height, conf, timestamp, None) + status, status_str = self.wallet.get_tx_status(tx_hash, tx_mined_status) + has_invoice = self.wallet.invoices.paid.get(tx_hash) + icon = self.icon_cache.get(":icons/" + TX_ICONS[status]) + v_str = self.parent.format_amount(value, is_diff=True, whitespaces=True) + balance_str = self.parent.format_amount(balance, whitespaces=True) + entry = ['', status_str, label, v_str, balance_str] + fiat_value = None + item = [QStandardItem(e) for e in entry] + item[3].setData(value, self.SORT_ROLE) + item[4].setData(balance, self.SORT_ROLE) + if has_invoice: + item[2].setIcon(self.icon_cache.get(":icons/seal")) + for i in range(len(entry)): + self.set_item_properties(item[i], i, tx_hash) + if value and value < 0: + item[2].setForeground(self.red_brush) + item[3].setForeground(self.red_brush) + self.txid_to_items[tx_hash] = item + self.update_item(tx_hash, self.parent.wallet.get_tx_height(tx_hash)) + source_row_idx = self.std_model.rowCount() + self.std_model.insertRow(source_row_idx, item) + new_idx = self.std_model.index(source_row_idx, 0) + history = self.parent.fx.show_history() + if history: + self.update_fiat(tx_hash, tx_item) + self.hide_row(self.proxy.mapFromSource(new_idx).row()) + + def set_item_properties(self, item, i, tx_hash): + if i>2: + item.setTextAlignment(Qt.AlignRight | Qt.AlignVCenter) + if i!=1: + item.setFont(self.monospace_font) + item.setEditable(i in self.editable_columns) + item.setData(tx_hash, self.TX_HASH_ROLE) + + def ensure_fields_available(self, items, idx, txid): + while len(items) < idx + 1: + row = list(self.transactions.keys()).index(txid) + qidx = self.std_model.index(row, len(items)) + assert qidx.isValid(), (self.std_model.columnCount(), idx) + item = self.std_model.itemFromIndex(qidx) + self.set_item_properties(item, len(items), txid) + items.append(item) + @profiler - def on_update(self): + def update(self): self.wallet = self.parent.wallet # type: Abstract_Wallet fx = self.parent.fx - r = self.wallet.get_full_history(domain=self.get_domain(), from_timestamp=self.start_timestamp, to_timestamp=self.end_timestamp, fx=fx) - self.transactions = r['transactions'] - self.summary = r['summary'] - if not self.years and self.transactions: - start_date = self.transactions[0].get('date') or date.today() - end_date = self.transactions[-1].get('date') or date.today() - self.years = [str(i) for i in range(start_date.year, end_date.year + 1)] - self.period_combo.insertItems(1, self.years) - item = self.currentItem() - current_tx = item.data(0, Qt.UserRole) if item else None - self.clear() - if fx: fx.history_used_spot = False - blue_brush = QBrush(QColor("#1E1EFF")) - red_brush = QBrush(QColor("#BC1E1E")) - monospace_font = QFont(MONOSPACE_FONT) - for tx_item in self.transactions: - tx_hash = tx_item['txid'] - height = tx_item['height'] - conf = tx_item['confirmations'] - timestamp = tx_item['timestamp'] - value = tx_item['value'].value - balance = tx_item['balance'].value - label = tx_item['label'] - tx_mined_status = TxMinedStatus(height, conf, timestamp, None) - status, status_str = self.wallet.get_tx_status(tx_hash, tx_mined_status) - has_invoice = self.wallet.invoices.paid.get(tx_hash) - icon = self.icon_cache.get(":icons/" + TX_ICONS[status]) - v_str = self.parent.format_amount(value, is_diff=True, whitespaces=True) - balance_str = self.parent.format_amount(balance, whitespaces=True) - entry = ['', tx_hash, status_str, label, v_str, balance_str] - fiat_value = None - if value is not None and fx and fx.show_history(): - fiat_value = tx_item['fiat_value'].value - value_str = fx.format_fiat(fiat_value) - entry.append(value_str) - # fixme: should use is_mine - if value < 0: - entry.append(fx.format_fiat(tx_item['acquisition_price'].value)) - entry.append(fx.format_fiat(tx_item['capital_gain'].value)) - item = SortableTreeWidgetItem(entry) - item.setIcon(0, icon) - item.setToolTip(0, str(conf) + " confirmation" + ("s" if conf != 1 else "")) - item.setData(0, SortableTreeWidgetItem.DataRole, (status, conf)) - if has_invoice: - item.setIcon(3, self.icon_cache.get(":icons/seal")) - for i in range(len(entry)): - if i>3: - item.setTextAlignment(i, Qt.AlignRight | Qt.AlignVCenter) - if i!=2: - item.setFont(i, monospace_font) - if value and value < 0: - item.setForeground(3, red_brush) - item.setForeground(4, red_brush) - if fiat_value and not tx_item['fiat_default']: - item.setForeground(6, blue_brush) - if tx_hash: - item.setData(0, Qt.UserRole, tx_hash) - self.insertTopLevelItem(0, item) - if current_tx == tx_hash: - self.setCurrentItem(item) - - def on_edited(self, item, column, prior): - '''Called only when the text actually changes''' - key = item.data(0, Qt.UserRole) - text = item.text(column) + r = self.wallet.get_full_history(domain=self.get_domain(), from_timestamp=None, to_timestamp=None, fx=fx) + seen = set() + history = fx.show_history() + tx_list = list(self.transactions.values()) + if r['transactions'] == tx_list: + return + if r['transactions'][:-1] == tx_list: + print_error('history_list: one new transaction') + row = r['transactions'][-1] + txid = row['txid'] + if txid not in self.transactions: + self.transactions[txid] = row + self.transactions.move_to_end(txid, last=True) + self.insert_tx(row) + return + else: + print_error('history_list: tx added but txid is already in list (weird), txid: ', txid) + for idx, row in enumerate(r['transactions']): + txid = row['txid'] + seen.add(txid) + if txid not in self.transactions: + self.transactions[txid] = row + self.transactions.move_to_end(txid, last=True) + self.insert_tx(row) + continue + old = self.transactions[txid] + if old == row: + continue + self.update_item(txid, self.parent.wallet.get_tx_height(txid)) + if history: + self.update_fiat(txid, row) + balance_str = self.parent.format_amount(row['balance'].value, whitespaces=True) + self.txid_to_items[txid][4].setText(balance_str) + self.txid_to_items[txid][4].setData(row['balance'].value, self.SORT_ROLE) + old.clear() + old.update(**row) + removed = 0 + l = list(enumerate(self.transactions.keys())) + for idx, txid in l: + if txid not in seen: + del self.transactions[txid] + del self.txid_to_items[txid] + items = self.std_model.takeRow(idx - removed) + removed_txid = items[0].data(self.TX_HASH_ROLE) + assert removed_txid == txid, (idx, removed) + removed += 1 + self.apply_filter() + + def update_fiat(self, txid, row): + cap_gains = self.parent.fx.get_history_capital_gains_config() + items = self.txid_to_items[txid] + self.ensure_fields_available(items, 7 if cap_gains else 5, txid) + if not row['fiat_default'] and row['fiat_value']: + items[5].setForeground(self.blue_brush) + value_str = self.parent.fx.format_fiat(row['fiat_value'].value) + items[5].setText(value_str) + items[5].setData(row['fiat_value'].value, self.SORT_ROLE) + # fixme: should use is_mine + if row['value'].value < 0 and cap_gains: + acq = row['acquisition_price'].value + items[6].setText(self.parent.fx.format_fiat(acq)) + items[6].setData(acq, self.SORT_ROLE) + cg = row['capital_gain'].value + items[7].setText(self.parent.fx.format_fiat(cg)) + items[7].setData(cg, self.SORT_ROLE) + + def update_on_new_fee_histogram(self): + pass + # TODO update unconfirmed tx'es + + def on_edited(self, index, user_role, text): + row, column = index.row(), index.column() + item = self.item_from_coordinate(row, column) + key = item.data(self.TX_HASH_ROLE) # fixme - if column == 3: + if column == 2: self.parent.wallet.set_label(key, text) self.update_labels() self.parent.update_completions() - elif column == 6: - self.parent.wallet.set_fiat_value(key, self.parent.fx.ccy, text) - self.on_update() - - def on_doubleclick(self, item, column): - if self.permit_edit(item, column): - super(HistoryList, self).on_doubleclick(item, column) + elif column == 5: + tx_item = self.transactions[key] + self.parent.wallet.set_fiat_value(key, self.parent.fx.ccy, text, self.parent.fx, tx_item['value'].value) + value = tx_item['value'].value + if value is not None: + fee = tx_item['fee'] + fiat_fields = self.parent.wallet.get_tx_item_fiat(key, value, self.parent.fx, fee.value if fee else None) + tx_item.update(fiat_fields) + self.update_fiat(key, tx_item) else: - tx_hash = item.data(0, Qt.UserRole) + assert False + + def mouseDoubleClickEvent(self, event: QMouseEvent): + idx = self.indexAt(event.pos()) + item = self.item_from_coordinate(idx.row(), idx.column()) + if not item or item.isEditable(): + super().mouseDoubleClickEvent(event) + elif item: + tx_hash = item.data(self.TX_HASH_ROLE) self.show_transaction(tx_hash) def show_transaction(self, tx_hash): @@ -311,13 +454,13 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): self.parent.show_transaction(tx, label) def update_labels(self): - root = self.invisibleRootItem() - child_count = root.childCount() + root = self.std_model.invisibleRootItem() + child_count = root.rowCount() for i in range(child_count): - item = root.child(i) - txid = item.data(0, Qt.UserRole) + item = root.child(i, 2) + txid = item.data(self.TX_HASH_ROLE) label = self.wallet.get_label(txid) - item.setText(3, label) + item.setText(label) def update_item(self, tx_hash, tx_mined_status): if self.wallet is None: @@ -325,31 +468,30 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): conf = tx_mined_status.conf status, status_str = self.wallet.get_tx_status(tx_hash, tx_mined_status) icon = self.icon_cache.get(":icons/" + TX_ICONS[status]) - items = self.findItems(tx_hash, Qt.MatchExactly, column=1) - if items: - item = items[0] - item.setIcon(0, icon) - item.setData(0, SortableTreeWidgetItem.DataRole, (status, conf)) - item.setText(2, status_str) - - def create_menu(self, position): - self.selectedIndexes() - item = self.currentItem() - if not item: - return - column = self.currentColumn() - tx_hash = item.data(0, Qt.UserRole) - if not tx_hash: + if tx_hash not in self.txid_to_items: return + items = self.txid_to_items[tx_hash] + items[0].setIcon(icon) + items[0].setToolTip(str(conf) + _(" confirmation" + ("s" if conf != 1 else ""))) + items[0].setData((status, conf), self.SORT_ROLE) + items[1].setText(status_str) + + def create_menu(self, position: QPoint): + org_idx: QModelIndex = self.indexAt(position) + idx = self.proxy.mapToSource(org_idx) + item: QStandardItem = self.std_model.itemFromIndex(idx) + assert item, 'create_menu: index not found in model' + tx_hash = idx.data(self.TX_HASH_ROLE) + column = idx.column() + assert tx_hash, "create_menu: no tx hash" tx = self.wallet.transactions.get(tx_hash) - if not tx: - return - if column is 0: - column_title = "ID" + assert tx, "create_menu: no tx" + if column == 0: + column_title = _('Transaction ID') column_data = tx_hash else: - column_title = self.headerItem().text(column) - column_data = item.text(column) + column_title = self.std_model.horizontalHeaderItem(column).text() + column_data = item.text() tx_URL = block_explorer_URL(self.config, 'tx', tx_hash) height = self.wallet.get_tx_height(tx_hash).height is_relevant, is_mine, v, fee = self.wallet.get_wallet_delta(tx) @@ -360,8 +502,10 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): menu.addAction(_("Remove"), lambda: self.remove_local_tx(tx_hash)) menu.addAction(_("Copy {}").format(column_title), lambda: self.parent.app.clipboard().setText(column_data)) for c in self.editable_columns: - menu.addAction(_("Edit {}").format(self.headerItem().text(c)), - lambda bound_c=c: self.editItem(item, bound_c)) + label = self.std_model.horizontalHeaderItem(c).text() + # TODO use siblingAtColumn when min Qt version is >=5.11 + persistent = QPersistentModelIndex(org_idx.sibling(org_idx.row(), c)) + menu.addAction(_("Edit {}").format(label), lambda p=persistent: self.edit(QModelIndex(p))) menu.addAction(_("Details"), lambda: self.show_transaction(tx_hash)) if is_unconfirmed and tx: # note: the current implementation of RBF *needs* the old tx fee @@ -430,7 +574,7 @@ class HistoryList(MyTreeWidget, AcceptFileDragDrop): self.parent.show_message(_("Your wallet history has been successfully exported.")) def do_export_history(self, file_name, is_csv): - history = self.transactions + history = self.transactions.values() lines = [] if is_csv: for item in history: diff --git a/electrum/gui/qt/installwizard.py b/electrum/gui/qt/installwizard.py index dc4ce28e4..ffd188671 100644 --- a/electrum/gui/qt/installwizard.py +++ b/electrum/gui/qt/installwizard.py @@ -432,7 +432,7 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard): return slayout.is_ext def pw_layout(self, msg, kind, force_disable_encrypt_cb): - playout = PasswordLayout(None, msg, kind, self.next_button, + playout = PasswordLayout(msg=msg, kind=kind, OK_button=self.next_button, force_disable_encrypt_cb=force_disable_encrypt_cb) playout.encrypt_cb.setChecked(True) self.exec_layout(playout.layout()) @@ -446,7 +446,7 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard): @wizard_dialog def request_storage_encryption(self, run_next): - playout = PasswordLayoutForHW(None, MSG_HW_STORAGE_ENCRYPTION, PW_NEW, self.next_button) + playout = PasswordLayoutForHW(MSG_HW_STORAGE_ENCRYPTION) playout.encrypt_cb.setChecked(True) self.exec_layout(playout.layout()) return playout.encrypt_cb.isChecked() diff --git a/electrum/gui/qt/invoice_list.py b/electrum/gui/qt/invoice_list.py index 462aadd85..4789aa066 100644 --- a/electrum/gui/qt/invoice_list.py +++ b/electrum/gui/qt/invoice_list.py @@ -29,36 +29,40 @@ from electrum.util import format_time from .util import * -class InvoiceList(MyTreeWidget): +class InvoiceList(MyTreeView): filter_columns = [0, 1, 2, 3] # Date, Requestor, Description, Amount def __init__(self, parent): - MyTreeWidget.__init__(self, parent, self.create_menu, [_('Expires'), _('Requestor'), _('Description'), _('Amount'), _('Status')], 2) + super().__init__(parent, self.create_menu, 2) self.setSortingEnabled(True) - self.header().setSectionResizeMode(1, QHeaderView.Interactive) self.setColumnWidth(1, 200) + self.setModel(QStandardItemModel(self)) + self.update() - def on_update(self): + def update(self): inv_list = self.parent.invoices.unpaid_invoices() - self.clear() + self.model().clear() + self.update_headers([_('Expires'), _('Requestor'), _('Description'), _('Amount'), _('Status')]) + self.header().setSectionResizeMode(1, QHeaderView.Interactive) for pr in inv_list: key = pr.get_id() status = self.parent.invoices.get_status(key) requestor = pr.get_requestor() exp = pr.get_expiration_date() date_str = format_time(exp) if exp else _('Never') - item = QTreeWidgetItem([date_str, requestor, pr.memo, self.parent.format_amount(pr.get_amount(), whitespaces=True), pr_tooltips.get(status,'')]) - item.setIcon(4, self.icon_cache.get(pr_icons.get(status))) - item.setData(0, Qt.UserRole, key) - item.setFont(1, QFont(MONOSPACE_FONT)) - item.setFont(3, QFont(MONOSPACE_FONT)) + labels = [date_str, requestor, pr.memo, self.parent.format_amount(pr.get_amount(), whitespaces=True), pr_tooltips.get(status,'')] + item = [QStandardItem(e) for e in labels] + item[4].setIcon(self.icon_cache.get(pr_icons.get(status))) + item[0].setData(Qt.UserRole, key) + item[1].setFont(QFont(MONOSPACE_FONT)) + item[3].setFont(QFont(MONOSPACE_FONT)) self.addTopLevelItem(item) - self.setCurrentItem(self.topLevelItem(0)) + self.selectionModel().select(self.model().index(0,0), QItemSelectionModel.SelectCurrent) self.setVisible(len(inv_list)) self.parent.invoices_label.setVisible(len(inv_list)) def import_invoices(self): - import_meta_gui(self.parent, _('invoices'), self.parent.invoices.import_file, self.on_update) + import_meta_gui(self.parent, _('invoices'), self.parent.invoices.import_file, self.update) def export_invoices(self): export_meta_gui(self.parent, _('invoices'), self.parent.invoices.export_file) diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 11dad7b61..db0e30657 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -222,6 +222,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError): self.fetch_alias() def on_history(self, b): + self.wallet.clear_coin_price_cache() self.new_fx_history_signal.emit() def setup_exception_hook(self): @@ -352,8 +353,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError): if self.config.is_dynfee(): self.fee_slider.update() self.do_update_fee() - # todo: update only unconfirmed tx - self.history_list.update() + self.history_list.update_on_new_fee_histogram() else: self.print_error("unexpected network_qt signal:", event, args) @@ -378,9 +378,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError): def load_wallet(self, wallet): wallet.thread = TaskThread(self, self.on_error) self.update_recently_visited(wallet.storage.path) - # update(==init) all tabs; expensive for large wallets.. - # so delay it somewhat, hence __init__ can finish and the window can appear sooner - QTimer.singleShot(50, self.update_tabs) self.need_update.set() # Once GUI has been initialized check if we want to announce something since the callback has been called before the GUI was initialized # update menus @@ -1110,9 +1107,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError): self.from_label = QLabel(_('From')) grid.addWidget(self.from_label, 3, 0) - self.from_list = MyTreeWidget(self, self.from_list_menu, ['','']) - self.from_list.setHeaderHidden(True) - self.from_list.setMaximumHeight(80) + self.from_list = FromList(self, self.from_list_menu) grid.addWidget(self.from_list, 3, 1, 1, -1) self.set_pay_from([]) diff --git a/electrum/gui/qt/network_dialog.py b/electrum/gui/qt/network_dialog.py index bef853830..a1f2dace0 100644 --- a/electrum/gui/qt/network_dialog.py +++ b/electrum/gui/qt/network_dialog.py @@ -82,8 +82,8 @@ class NodesListWidget(QTreeWidget): server = item.data(1, Qt.UserRole) menu.addAction(_("Use as server"), lambda: self.parent.follow_server(server)) else: - index = item.data(1, Qt.UserRole) - menu.addAction(_("Follow this branch"), lambda: self.parent.follow_branch(index)) + chain_id = item.data(1, Qt.UserRole) + menu.addAction(_("Follow this branch"), lambda: self.parent.follow_branch(chain_id)) menu.exec_(self.viewport().mapToGlobal(position)) def keyPressEvent(self, event): @@ -100,25 +100,25 @@ class NodesListWidget(QTreeWidget): def update(self, network: Network): self.clear() - self.addChild = self.addTopLevelItem chains = network.get_blockchains() n_chains = len(chains) - for k, items in chains.items(): - b = blockchain.blockchains[k] + for chain_id, interfaces in chains.items(): + b = blockchain.blockchains.get(chain_id) + if b is None: continue name = b.get_name() - if n_chains >1: + if n_chains > 1: x = QTreeWidgetItem([name + '@%d'%b.get_max_forkpoint(), '%d'%b.height()]) x.setData(0, Qt.UserRole, 1) - x.setData(1, Qt.UserRole, b.forkpoint) + x.setData(1, Qt.UserRole, b.get_id()) else: x = self - for i in items: + for i in interfaces: star = ' *' if i == network.interface else '' item = QTreeWidgetItem([i.host + star, '%d'%i.tip]) item.setData(0, Qt.UserRole, 0) item.setData(1, Qt.UserRole, i.server) - x.addChild(item) - if n_chains>1: + x.addTopLevelItem(item) + if n_chains > 1: self.addTopLevelItem(x) x.setExpanded(True) @@ -410,8 +410,8 @@ class NetworkChoiceLayout(object): self.set_protocol(p) self.set_server() - def follow_branch(self, index): - self.network.run_from_another_thread(self.network.follow_chain_given_id(index)) + def follow_branch(self, chain_id): + self.network.run_from_another_thread(self.network.follow_chain_given_id(chain_id)) self.update() def follow_server(self, server): diff --git a/electrum/gui/qt/password_dialog.py b/electrum/gui/qt/password_dialog.py index 66b3f51b9..3202618ec 100644 --- a/electrum/gui/qt/password_dialog.py +++ b/electrum/gui/qt/password_dialog.py @@ -60,7 +60,7 @@ class PasswordLayout(object): titles = [_("Enter Password"), _("Change Password"), _("Enter Passphrase")] - def __init__(self, wallet, msg, kind, OK_button, force_disable_encrypt_cb=False): + def __init__(self, msg, kind, OK_button, wallet=None, force_disable_encrypt_cb=False): self.wallet = wallet self.pw = QLineEdit() @@ -169,12 +169,9 @@ class PasswordLayout(object): class PasswordLayoutForHW(object): - def __init__(self, wallet, msg, kind, OK_button): + def __init__(self, msg, wallet=None): self.wallet = wallet - self.kind = kind - self.OK_button = OK_button - vbox = QVBoxLayout() label = QLabel(msg + "\n") label.setWordWrap(True) @@ -254,9 +251,11 @@ class ChangePasswordDialogForSW(ChangePasswordDialogBase): else: msg = _('Your wallet is password protected and encrypted.') msg += ' ' + _('Use this dialog to change your password.') - self.playout = PasswordLayout( - wallet, msg, PW_CHANGE, OK_button, - force_disable_encrypt_cb=not wallet.can_have_keystore_encryption()) + self.playout = PasswordLayout(msg=msg, + kind=PW_CHANGE, + OK_button=OK_button, + wallet=wallet, + force_disable_encrypt_cb=not wallet.can_have_keystore_encryption()) def run(self): if not self.exec_(): @@ -276,7 +275,7 @@ class ChangePasswordDialogForHW(ChangePasswordDialogBase): msg = _('Your wallet file is encrypted.') msg += '\n' + _('Note: If you enable this setting, you will need your hardware device to open your wallet.') msg += '\n' + _('Use this dialog to toggle encryption.') - self.playout = PasswordLayoutForHW(wallet, msg, PW_CHANGE, OK_button) + self.playout = PasswordLayoutForHW(msg) def run(self): if not self.exec_(): diff --git a/electrum/gui/qt/request_list.py b/electrum/gui/qt/request_list.py index 19ec59703..8c6567fc8 100644 --- a/electrum/gui/qt/request_list.py +++ b/electrum/gui/qt/request_list.py @@ -23,43 +23,39 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from PyQt5.QtGui import * -from PyQt5.QtCore import * -from PyQt5.QtWidgets import QTreeWidgetItem, QMenu +from PyQt5.QtGui import QStandardItemModel, QStandardItem +from PyQt5.QtWidgets import QMenu +from PyQt5.QtCore import Qt from electrum.i18n import _ from electrum.util import format_time, age from electrum.plugin import run_hook from electrum.paymentrequest import PR_UNKNOWN -from .util import MyTreeWidget, pr_tooltips, pr_icons +from .util import MyTreeView, pr_tooltips, pr_icons - -class RequestList(MyTreeWidget): +class RequestList(MyTreeView): filter_columns = [0, 1, 2, 3, 4] # Date, Account, Address, Description, Amount def __init__(self, parent): - MyTreeWidget.__init__(self, parent, self.create_menu, [_('Date'), _('Address'), '', _('Description'), _('Amount'), _('Status')], 3) - self.currentItemChanged.connect(self.item_changed) - self.itemClicked.connect(self.item_changed) + super().__init__(parent, self.create_menu, 3, editable_columns=[]) + self.setModel(QStandardItemModel(self)) self.setSortingEnabled(True) self.setColumnWidth(0, 180) - self.hideColumn(1) + self.update() + self.selectionModel().currentRowChanged.connect(self.item_changed) - def item_changed(self, item): - if item is None: - return - if not item.isSelected(): - return - addr = str(item.text(1)) + def item_changed(self, idx): + # TODO use siblingAtColumn when min Qt version is >=5.11 + addr = self.model().itemFromIndex(idx.sibling(idx.row(), 1)).text() req = self.wallet.receive_requests.get(addr) if req is None: self.update() return expires = age(req['time'] + req['exp']) if req.get('exp') else _('Never') amount = req['amount'] - message = self.wallet.labels.get(addr, '') + message = req['memo'] self.parent.receive_address_e.setText(addr) self.parent.receive_message_e.setText(message) self.parent.receive_amount_e.setAmount(amount) @@ -68,7 +64,7 @@ class RequestList(MyTreeWidget): self.parent.expires_label.setText(expires) self.parent.new_request_button.setEnabled(True) - def on_update(self): + def update(self): self.wallet = self.parent.wallet # hide receive tab if no receive requests available b = len(self.wallet.receive_requests) > 0 @@ -86,8 +82,9 @@ class RequestList(MyTreeWidget): self.parent.set_receive_address(addr) self.parent.new_request_button.setEnabled(addr != current_address) - # clear the list and fill it again - self.clear() + self.model().clear() + self.update_headers([_('Date'), _('Address'), '', _('Description'), _('Amount'), _('Status')]) + self.hideColumn(1) # hide address column for req in self.wallet.get_sorted_requests(self.config): address = req['address'] if address not in domain: @@ -95,35 +92,40 @@ class RequestList(MyTreeWidget): timestamp = req.get('time', 0) amount = req.get('amount') expiration = req.get('exp', None) - message = req.get('memo', '') + message = req['memo'] date = format_time(timestamp) status = req.get('status') signature = req.get('sig') requestor = req.get('name', '') amount_str = self.parent.format_amount(amount) if amount else "" - item = QTreeWidgetItem([date, address, '', message, amount_str, pr_tooltips.get(status,'')]) + labels = [date, address, '', message, amount_str, pr_tooltips.get(status,'')] + items = [QStandardItem(e) for e in labels] + self.set_editability(items) if signature is not None: - item.setIcon(2, self.icon_cache.get(":icons/seal.png")) - item.setToolTip(2, 'signed by '+ requestor) + items[2].setIcon(self.icon_cache.get(":icons/seal.png")) + items[2].setToolTip('signed by '+ requestor) if status is not PR_UNKNOWN: - item.setIcon(6, self.icon_cache.get(pr_icons.get(status))) - self.addTopLevelItem(item) - + items[5].setIcon(self.icon_cache.get(pr_icons.get(status))) + items[3].setData(address, Qt.UserRole) + self.model().insertRow(self.model().rowCount(), items) def create_menu(self, position): - item = self.itemAt(position) + idx = self.indexAt(position) + # TODO use siblingAtColumn when min Qt version is >=5.11 + item = self.model().itemFromIndex(idx.sibling(idx.row(), 1)) if not item: return - addr = str(item.text(1)) + addr = item.text() req = self.wallet.receive_requests.get(addr) if req is None: self.update() return - column = self.currentColumn() - column_title = self.headerItem().text(column) - column_data = item.text(column) + column = idx.column() + column_title = self.model().horizontalHeaderItem(column).text() + column_data = item.text() menu = QMenu(self) - menu.addAction(_("Copy {}").format(column_title), lambda: self.parent.app.clipboard().setText(column_data)) + if column != 2: + menu.addAction(_("Copy {}").format(column_title), lambda: self.parent.app.clipboard().setText(column_data)) menu.addAction(_("Copy URI"), lambda: self.parent.view_and_paste('URI', '', self.parent.get_request_URI(addr))) menu.addAction(_("Save as BIP70 file"), lambda: self.parent.export_payment_request(addr)) menu.addAction(_("Delete"), lambda: self.parent.delete_payment_request(addr)) diff --git a/electrum/gui/qt/util.py b/electrum/gui/qt/util.py index 5e1912a3e..0d386bdfc 100644 --- a/electrum/gui/qt/util.py +++ b/electrum/gui/qt/util.py @@ -5,12 +5,13 @@ import platform import queue from functools import partial from typing import NamedTuple, Callable, Optional +from abc import abstractmethod from PyQt5.QtGui import * from PyQt5.QtCore import * from PyQt5.QtWidgets import * -from electrum.i18n import _ +from electrum.i18n import _, languages from electrum.util import FileImportFailed, FileExportFailed from electrum.paymentrequest import PR_UNPAID, PR_PAID, PR_EXPIRED @@ -398,20 +399,16 @@ class ElectrumItemDelegate(QStyledItemDelegate): def createEditor(self, parent, option, index): return self.parent().createEditor(parent, option, index) -class MyTreeWidget(QTreeWidget): +class MyTreeView(QTreeView): - def __init__(self, parent, create_menu, headers, stretch_column=None, - editable_columns=None): - QTreeWidget.__init__(self, parent) + def __init__(self, parent, create_menu, stretch_column=None, editable_columns=None): + super().__init__(parent) self.parent = parent self.config = self.parent.config self.stretch_column = stretch_column self.setContextMenuPolicy(Qt.CustomContextMenu) self.customContextMenuRequested.connect(create_menu) self.setUniformRowHeights(True) - # extend the syntax for consistency - self.addChild = self.addTopLevelItem - self.insertChild = self.insertTopLevelItem self.icon_cache = IconCache() @@ -424,127 +421,143 @@ class MyTreeWidget(QTreeWidget): editable_columns = set(editable_columns) self.editable_columns = editable_columns self.setItemDelegate(ElectrumItemDelegate(self)) - self.itemDoubleClicked.connect(self.on_doubleclick) - self.update_headers(headers) self.current_filter = "" self.setRootIsDecorated(False) # remove left margin self.toolbar_shown = False - def update_headers(self, headers): - self.setColumnCount(len(headers)) - self.setHeaderLabels(headers) + def set_editability(self, items): + for idx, i in enumerate(items): + i.setEditable(idx in self.editable_columns) + + def selected_in_column(self, column: int): + items = self.selectionModel().selectedIndexes() + return list(x for x in items if x.column() == column) + + def current_item_user_role(self, col) -> Optional[QStandardItem]: + idx = self.selectionModel().currentIndex() + idx = idx.sibling(idx.row(), col) + item = self.model().itemFromIndex(idx) + if item: + return item.data(Qt.UserRole) + + def set_current_idx(self, set_current: QPersistentModelIndex): + if set_current: + assert isinstance(set_current, QPersistentModelIndex) + assert set_current.isValid() + self.selectionModel().select(QModelIndex(set_current), QItemSelectionModel.SelectCurrent) + + def update_headers(self, headers, model=None): + if model is None: + model = self.model() + model.setHorizontalHeaderLabels(headers) self.header().setStretchLastSection(False) for col in range(len(headers)): sm = QHeaderView.Stretch if col == self.stretch_column else QHeaderView.ResizeToContents self.header().setSectionResizeMode(col, sm) - def editItem(self, item, column): - if column in self.editable_columns: - try: - self.editing_itemcol = (item, column, item.text(column)) - # Calling setFlags causes on_changed events for some reason - item.setFlags(item.flags() | Qt.ItemIsEditable) - QTreeWidget.editItem(self, item, column) - item.setFlags(item.flags() & ~Qt.ItemIsEditable) - except RuntimeError: - # (item) wrapped C/C++ object has been deleted - pass - def keyPressEvent(self, event): if event.key() in [ Qt.Key_F2, Qt.Key_Return ] and self.editor is None: - self.on_activated(self.currentItem(), self.currentColumn()) - else: - QTreeWidget.keyPressEvent(self, event) - - def permit_edit(self, item, column): - return (column in self.editable_columns - and self.on_permit_edit(item, column)) - - def on_permit_edit(self, item, column): - return True - - def on_doubleclick(self, item, column): - if self.permit_edit(item, column): - self.editItem(item, column) + self.on_activated(self.selectionModel().currentIndex()) + return + super().keyPressEvent(event) - def on_activated(self, item, column): + def on_activated(self, idx): # on 'enter' we show the menu - pt = self.visualItemRect(item).bottomLeft() + pt = self.visualRect(idx).bottomLeft() pt.setX(50) self.customContextMenuRequested.emit(pt) - def createEditor(self, parent, option, index): + def createEditor(self, parent, option, idx): self.editor = QStyledItemDelegate.createEditor(self.itemDelegate(), - parent, option, index) - self.editor.editingFinished.connect(self.editing_finished) + parent, option, idx) + item = self.item_from_coordinate(idx.row(), idx.column()) + user_role = item.data(Qt.UserRole) + assert user_role is not None + prior_text = item.text() + def editing_finished(): + # Long-time QT bug - pressing Enter to finish editing signals + # editingFinished twice. If the item changed the sequence is + # Enter key: editingFinished, on_change, editingFinished + # Mouse: on_change, editingFinished + # This mess is the cleanest way to ensure we make the + # on_edited callback with the updated item + if self.editor is None: + return + if self.editor.text() == prior_text: + self.editor = None # Unchanged - ignore any 2nd call + return + if item.text() == prior_text: + return # Buggy first call on Enter key, item not yet updated + if not idx.isValid(): + return + self.on_edited(idx, user_role, self.editor.text()) + self.editor = None + self.editor.editingFinished.connect(editing_finished) return self.editor - def editing_finished(self): - # Long-time QT bug - pressing Enter to finish editing signals - # editingFinished twice. If the item changed the sequence is - # Enter key: editingFinished, on_change, editingFinished - # Mouse: on_change, editingFinished - # This mess is the cleanest way to ensure we make the - # on_edited callback with the updated item - if self.editor: - (item, column, prior_text) = self.editing_itemcol - if self.editor.text() == prior_text: - self.editor = None # Unchanged - ignore any 2nd call - elif item.text(column) == prior_text: - pass # Buggy first call on Enter key, item not yet updated - else: - # What we want - the updated item - self.on_edited(*self.editing_itemcol) - self.editor = None - - # Now do any pending updates - if self.editor is None and self.pending_update: - self.pending_update = False - self.on_update() - - def on_edited(self, item, column, prior): - '''Called only when the text actually changes''' - key = item.data(0, Qt.UserRole) - text = item.text(column) - self.parent.wallet.set_label(key, text) + def edit(self, idx, trigger=QAbstractItemView.AllEditTriggers, event=None): + """ + this is to prevent: + edit: editing failed + from inside qt + """ + return super().edit(idx, trigger, event) + + def on_edited(self, idx: QModelIndex, user_role, text): + self.parent.wallet.set_label(user_role, text) self.parent.history_list.update_labels() self.parent.update_completions() - def update(self): - # Defer updates if editing - if self.editor: - self.pending_update = True - else: - self.setUpdatesEnabled(False) - scroll_pos = self.verticalScrollBar().value() - self.on_update() - self.setUpdatesEnabled(True) - # To paint the list before resetting the scroll position - self.parent.app.processEvents() - self.verticalScrollBar().setValue(scroll_pos) + def apply_filter(self): if self.current_filter: self.filter(self.current_filter) - def on_update(self): + @abstractmethod + def should_hide(self, row): + """ + row_num is for self.model(). So if there is a proxy, it is the row number + in that! + """ pass - def get_leaves(self, root): - child_count = root.childCount() - if child_count == 0: - yield root - for i in range(child_count): - item = root.child(i) - for x in self.get_leaves(item): - yield x + def item_from_coordinate(self, row_num, column): + if isinstance(self.model(), QSortFilterProxyModel): + idx = self.model().mapToSource(self.model().index(row_num, column)) + return self.model().sourceModel().itemFromIndex(idx) + else: + idx = self.model().index(row_num, column) + return self.model().itemFromIndex(idx) + + def hide_row(self, row_num): + """ + row_num is for self.model(). So if there is a proxy, it is the row number + in that! + """ + should_hide = self.should_hide(row_num) + if not self.current_filter and should_hide is None: + # no filters at all, neither date nor search + self.setRowHidden(row_num, QModelIndex(), False) + return + for column in self.filter_columns: + item = self.item_from_coordinate(row_num, column) + txt = item.text().lower() + if self.current_filter in txt: + # the filter matched, but the date filter might apply + self.setRowHidden(row_num, QModelIndex(), bool(should_hide)) + break + else: + # we did not find the filter in any columns, show the item + self.setRowHidden(row_num, QModelIndex(), True) def filter(self, p): - columns = self.__class__.filter_columns p = p.lower() self.current_filter = p - for item in self.get_leaves(self.invisibleRootItem()): - item.setHidden(all([item.text(column).lower().find(p) == -1 - for column in columns])) + self.hide_rows() + + def hide_rows(self): + for row in range(self.model().rowCount()): + self.hide_row(row) def create_toolbar(self, config=None): hbox = QHBoxLayout() @@ -790,22 +803,6 @@ def get_parent_main_window(widget): return widget return None -class SortableTreeWidgetItem(QTreeWidgetItem): - DataRole = Qt.UserRole + 1 - - def __lt__(self, other): - column = self.treeWidget().sortColumn() - if None not in [x.data(column, self.DataRole) for x in [self, other]]: - # We have set custom data to sort by - return self.data(column, self.DataRole) < other.data(column, self.DataRole) - try: - # Is the value something numeric? - return float(self.text(column)) < float(other.text(column)) - except ValueError: - # If not, we will just do string comparison - return self.text(column) < other.text(column) - - class IconCache: def __init__(self): @@ -817,6 +814,26 @@ class IconCache: return self.__cache[file_name] +def get_default_language(): + name = QLocale.system().name() + return name if name in languages else 'en_UK' + +class FromList(QTreeWidget): + def __init__(self, parent, create_menu): + super().__init__(parent) + self.setHeaderHidden(True) + self.setMaximumHeight(300) + self.setContextMenuPolicy(Qt.CustomContextMenu) + self.customContextMenuRequested.connect(create_menu) + self.setUniformRowHeights(True) + # remove left margin + self.setRootIsDecorated(False) + self.setColumnCount(2) + self.header().setStretchLastSection(False) + sm = QHeaderView.ResizeToContents + self.header().setSectionResizeMode(0, sm) + self.header().setSectionResizeMode(1, sm) + if __name__ == "__main__": app = QApplication([]) t = WaitingDialog(None, 'testing ...', lambda: [time.sleep(1)], lambda x: QMessageBox.information(None, 'done', "done")) diff --git a/electrum/gui/qt/utxo_list.py b/electrum/gui/qt/utxo_list.py index 5985d9c8c..0b9d85508 100644 --- a/electrum/gui/qt/utxo_list.py +++ b/electrum/gui/qt/utxo_list.py @@ -23,58 +23,66 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from typing import Optional, List + from electrum.i18n import _ from .util import * - -class UTXOList(MyTreeWidget): - filter_columns = [0, 2] # Address, Label +class UTXOList(MyTreeView): + filter_columns = [0, 1] # Address, Label def __init__(self, parent=None): - MyTreeWidget.__init__(self, parent, self.create_menu, [ _('Address'), _('Label'), _('Amount'), _('Height'), _('Output point')], 1) + super().__init__(parent, self.create_menu, 1) + self.setModel(QStandardItemModel(self)) self.setSelectionMode(QAbstractItemView.ExtendedSelection) self.setSortingEnabled(True) + self.update() - def get_name(self, x): - return x.get('prevout_hash') + ":%d"%x.get('prevout_n') - - def on_update(self): + def update(self): self.wallet = self.parent.wallet - item = self.currentItem() - self.clear() - self.utxos = self.wallet.get_utxos() - for x in self.utxos: + utxos = self.wallet.get_utxos() + self.utxo_dict = {} + self.model().clear() + self.update_headers([ _('Address'), _('Label'), _('Amount'), _('Height'), _('Output point')]) + for idx, x in enumerate(utxos): address = x.get('address') height = x.get('height') - name = self.get_name(x) + name = x.get('prevout_hash') + ":%d"%x.get('prevout_n') + self.utxo_dict[name] = x label = self.wallet.get_label(x.get('prevout_hash')) amount = self.parent.format_amount(x['value'], whitespaces=True) - utxo_item = SortableTreeWidgetItem([address, label, amount, '%d'%height, name[0:10] + '...' + name[-2:]]) - utxo_item.setFont(0, QFont(MONOSPACE_FONT)) - utxo_item.setFont(2, QFont(MONOSPACE_FONT)) - utxo_item.setFont(4, QFont(MONOSPACE_FONT)) - utxo_item.setData(0, Qt.UserRole, name) + labels = [address, label, amount, '%d'%height, name[0:10] + '...' + name[-2:]] + utxo_item = [QStandardItem(x) for x in labels] + self.set_editability(utxo_item) + utxo_item[0].setFont(QFont(MONOSPACE_FONT)) + utxo_item[2].setFont(QFont(MONOSPACE_FONT)) + utxo_item[4].setFont(QFont(MONOSPACE_FONT)) + utxo_item[0].setData(name, Qt.UserRole) if self.wallet.is_frozen(address): - utxo_item.setBackground(0, ColorScheme.BLUE.as_color(True)) - self.addChild(utxo_item) + utxo_item[0].setBackground(ColorScheme.BLUE.as_color(True)) + self.model().insertRow(idx, utxo_item) + + def selected_column_0_user_roles(self) -> Optional[List[str]]: + if not self.model(): + return None + items = self.selected_in_column(0) + if not items: + return None + return [x.data(Qt.UserRole) for x in items] def create_menu(self, position): - selected = [x.data(0, Qt.UserRole) for x in self.selectedItems()] + selected = self.selected_column_0_user_roles() if not selected: return menu = QMenu() - coins = filter(lambda x: self.get_name(x) in selected, self.utxos) - + coins = (self.utxo_dict[name] for name in selected) menu.addAction(_("Spend"), lambda: self.parent.spend_coins(coins)) if len(selected) == 1: txid = selected[0].split(':')[0] tx = self.wallet.transactions.get(txid) if tx: - menu.addAction(_("Details"), lambda: self.parent.show_transaction(tx)) + label = self.wallet.get_label(txid) or None # Prefer None if empty (None hides the Description: field in the window) + menu.addAction(_("Details"), lambda: self.parent.show_transaction(tx, label)) menu.exec_(self.viewport().mapToGlobal(position)) - - def on_permit_edit(self, item, column): - # disable editing fields in this tab (labels) - return False diff --git a/electrum/gui/text.py b/electrum/gui/text.py index 7d429ae01..ec5ba5c9b 100644 --- a/electrum/gui/text.py +++ b/electrum/gui/text.py @@ -91,7 +91,7 @@ class ElectrumGui: self.set_cursor(0) return s - def update(self, event): + def update(self, event, *args): self.update_history() if self.tab == 0: self.print_history() diff --git a/electrum/interface.py b/electrum/interface.py index 68ede7554..99e2349e4 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -28,7 +28,7 @@ import ssl import sys import traceback import asyncio -from typing import Tuple, Union, List, TYPE_CHECKING +from typing import Tuple, Union, List, TYPE_CHECKING, Optional from collections import defaultdict import aiorpcx @@ -140,14 +140,14 @@ def serialize_server(host: str, port: Union[str, int], protocol: str) -> str: class Interface(PrintError): verbosity_filter = 'i' - def __init__(self, network: 'Network', server: str, config_path, proxy: dict): + def __init__(self, network: 'Network', server: str, proxy: Optional[dict]): self.ready = asyncio.Future() self.got_disconnected = asyncio.Future() self.server = server self.host, self.port, self.protocol = deserialize_server(self.server) self.port = int(self.port) - self.config_path = config_path - self.cert_path = os.path.join(self.config_path, 'certs', self.host) + assert network.config.path + self.cert_path = os.path.join(network.config.path, 'certs', self.host) self.blockchain = None self._requested_chunks = set() self.network = network @@ -281,7 +281,7 @@ class Interface(PrintError): assert self.tip_header chain = blockchain.check_header(self.tip_header) if not chain: - self.blockchain = blockchain.blockchains[0] + self.blockchain = blockchain.get_best_chain() else: self.blockchain = chain assert self.blockchain is not None @@ -502,7 +502,7 @@ class Interface(PrintError): # bad_header connects to good_header; bad_header itself is NOT in self.blockchain. bh = self.blockchain.height() - assert bh >= good + assert bh >= good, (bh, good) if bh == good: height = good + 1 self.print_error("catching up from {}".format(height)) @@ -510,53 +510,12 @@ class Interface(PrintError): # this is a new fork we don't yet have height = bad + 1 - branch = blockchain.blockchains.get(bad) - if branch is not None: - # Conflict!! As our fork handling is not completely general, - # we need to delete another fork to save this one. - # Note: This could be a potential DOS vector against Electrum. - # However, mining blocks that satisfy the difficulty requirements - # is assumed to be expensive; especially as forks below the max - # checkpoint are ignored. - self.print_error("new fork at bad height {}. conflict!!".format(bad)) - assert self.blockchain != branch - ismocking = type(branch) is dict - if ismocking: - self.print_error("TODO replace blockchain") - return 'fork_conflict', height - self.print_error('forkpoint conflicts with existing fork', branch.path()) - self._raise_if_fork_conflicts_with_default_server(branch) - await self._disconnect_from_interfaces_on_conflicting_blockchain(branch) - branch.write(b'', 0) - branch.save_header(bad_header) - self.blockchain = branch - return 'fork_conflict', height - else: - # No conflict. Just save the new fork. - self.print_error("new fork at bad height {}. NO conflict.".format(bad)) - forkfun = self.blockchain.fork if 'mock' not in bad_header else bad_header['mock']['fork'] - b = forkfun(bad_header) - with blockchain.blockchains_lock: - assert bad not in blockchain.blockchains, (bad, list(blockchain.blockchains)) - blockchain.blockchains[bad] = b - self.blockchain = b - assert b.forkpoint == bad - return 'fork_noconflict', height - - def _raise_if_fork_conflicts_with_default_server(self, chain_to_delete: Blockchain) -> None: - main_interface = self.network.interface - if not main_interface: return - if main_interface == self: return - chain_of_default_server = main_interface.blockchain - if not chain_of_default_server: return - if chain_to_delete == chain_of_default_server: - raise GracefulDisconnect('refusing to overwrite blockchain of default server') - - async def _disconnect_from_interfaces_on_conflicting_blockchain(self, chain: Blockchain) -> None: - ifaces = await self.network.disconnect_from_interfaces_on_given_blockchain(chain) - if not ifaces: return - servers = [interface.server for interface in ifaces] - self.print_error("forcing disconnect of other interfaces: {}".format(servers)) + self.print_error(f"new fork at bad height {bad}") + forkfun = self.blockchain.fork if 'mock' not in bad_header else bad_header['mock']['fork'] + b = forkfun(bad_header) # type: Blockchain + self.blockchain = b + assert b.forkpoint == bad + return 'fork', height async def _search_headers_backwards(self, height, header): async def iterate(): diff --git a/electrum/keystore.py b/electrum/keystore.py index a942d0751..e0e21fa60 100644 --- a/electrum/keystore.py +++ b/electrum/keystore.py @@ -35,7 +35,7 @@ from .bip32 import (bip32_public_derivation, deserialize_xpub, CKD_pub, bip32_private_key, bip32_derivation, BIP32_PRIME, is_xpub, is_xprv) from .ecc import string_to_number, number_to_string -from .crypto import pw_decode, pw_encode, sha256d +from .crypto import (pw_decode, pw_encode, sha256d, PW_HASH_VERSION_LATEST) from .util import (PrintError, InvalidPassword, hfu, WalletFileException, BitcoinException, bh2u, bfh, print_error, inv_dict) from .mnemonic import Mnemonic, load_wordlist @@ -92,8 +92,9 @@ class KeyStore(PrintError): class Software_KeyStore(KeyStore): - def __init__(self): + def __init__(self, d): KeyStore.__init__(self) + self.pw_hash_version = d.get('pw_hash_version', 1) def may_have_password(self): return not self.is_watching_only() @@ -122,6 +123,12 @@ class Software_KeyStore(KeyStore): if keypairs: tx.sign(keypairs) + def update_password(self, old_password, new_password): + raise NotImplementedError() # implemented by subclasses + + def check_password(self, password): + raise NotImplementedError() # implemented by subclasses + class Imported_KeyStore(Software_KeyStore): # keystore for imported private keys @@ -129,7 +136,7 @@ class Imported_KeyStore(Software_KeyStore): type = 'imported' def __init__(self, d): - Software_KeyStore.__init__(self) + Software_KeyStore.__init__(self, d) self.keypairs = d.get('keypairs', {}) def is_deterministic(self): @@ -142,6 +149,7 @@ class Imported_KeyStore(Software_KeyStore): return { 'type': self.type, 'keypairs': self.keypairs, + 'pw_hash_version': self.pw_hash_version, } def can_import(self): @@ -161,14 +169,14 @@ class Imported_KeyStore(Software_KeyStore): # there will only be one pubkey-privkey pair for it in self.keypairs, # and the privkey will encode a txin_type but that txin_type cannot be trusted. # Removing keys complicates this further. - self.keypairs[pubkey] = pw_encode(serialized_privkey, password) + self.keypairs[pubkey] = pw_encode(serialized_privkey, password, version=self.pw_hash_version) return txin_type, pubkey def delete_imported_key(self, key): self.keypairs.pop(key) def get_private_key(self, pubkey, password): - sec = pw_decode(self.keypairs[pubkey], password) + sec = pw_decode(self.keypairs[pubkey], password, version=self.pw_hash_version) txin_type, privkey, compressed = deserialize_privkey(sec) # this checks the password if pubkey != ecc.ECPrivkey(privkey).get_public_key_hex(compressed=compressed): @@ -189,16 +197,17 @@ class Imported_KeyStore(Software_KeyStore): if new_password == '': new_password = None for k, v in self.keypairs.items(): - b = pw_decode(v, old_password) - c = pw_encode(b, new_password) + b = pw_decode(v, old_password, version=self.pw_hash_version) + c = pw_encode(b, new_password, version=PW_HASH_VERSION_LATEST) self.keypairs[k] = c + self.pw_hash_version = PW_HASH_VERSION_LATEST class Deterministic_KeyStore(Software_KeyStore): def __init__(self, d): - Software_KeyStore.__init__(self) + Software_KeyStore.__init__(self, d) self.seed = d.get('seed', '') self.passphrase = d.get('passphrase', '') @@ -206,12 +215,14 @@ class Deterministic_KeyStore(Software_KeyStore): return True def dump(self): - d = {} + d = { + 'type': self.type, + 'pw_hash_version': self.pw_hash_version, + } if self.seed: d['seed'] = self.seed if self.passphrase: d['passphrase'] = self.passphrase - d['type'] = self.type return d def has_seed(self): @@ -226,10 +237,13 @@ class Deterministic_KeyStore(Software_KeyStore): self.seed = self.format_seed(seed) def get_seed(self, password): - return pw_decode(self.seed, password) + return pw_decode(self.seed, password, version=self.pw_hash_version) def get_passphrase(self, password): - return pw_decode(self.passphrase, password) if self.passphrase else '' + if self.passphrase: + return pw_decode(self.passphrase, password, version=self.pw_hash_version) + else: + return '' class Xpub: @@ -312,10 +326,10 @@ class BIP32_KeyStore(Deterministic_KeyStore, Xpub): return d def get_master_private_key(self, password): - return pw_decode(self.xprv, password) + return pw_decode(self.xprv, password, version=self.pw_hash_version) def check_password(self, password): - xprv = pw_decode(self.xprv, password) + xprv = pw_decode(self.xprv, password, version=self.pw_hash_version) if deserialize_xprv(xprv)[4] != deserialize_xpub(self.xpub)[4]: raise InvalidPassword() @@ -325,13 +339,14 @@ class BIP32_KeyStore(Deterministic_KeyStore, Xpub): new_password = None if self.has_seed(): decoded = self.get_seed(old_password) - self.seed = pw_encode(decoded, new_password) + self.seed = pw_encode(decoded, new_password, version=PW_HASH_VERSION_LATEST) if self.passphrase: decoded = self.get_passphrase(old_password) - self.passphrase = pw_encode(decoded, new_password) + self.passphrase = pw_encode(decoded, new_password, version=PW_HASH_VERSION_LATEST) if self.xprv is not None: - b = pw_decode(self.xprv, old_password) - self.xprv = pw_encode(b, new_password) + b = pw_decode(self.xprv, old_password, version=self.pw_hash_version) + self.xprv = pw_encode(b, new_password, version=PW_HASH_VERSION_LATEST) + self.pw_hash_version = PW_HASH_VERSION_LATEST def is_watching_only(self): return self.xprv is None @@ -362,7 +377,7 @@ class Old_KeyStore(Deterministic_KeyStore): self.mpk = d.get('mpk') def get_hex_seed(self, password): - return pw_decode(self.seed, password).encode('utf8') + return pw_decode(self.seed, password, version=self.pw_hash_version).encode('utf8') def dump(self): d = Deterministic_KeyStore.dump(self) @@ -484,8 +499,9 @@ class Old_KeyStore(Deterministic_KeyStore): if new_password == '': new_password = None if self.has_seed(): - decoded = pw_decode(self.seed, old_password) - self.seed = pw_encode(decoded, new_password) + decoded = pw_decode(self.seed, old_password, version=self.pw_hash_version) + self.seed = pw_encode(decoded, new_password, version=PW_HASH_VERSION_LATEST) + self.pw_hash_version = PW_HASH_VERSION_LATEST diff --git a/electrum/network.py b/electrum/network.py index c0218f9cc..9fe78a5e8 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -44,6 +44,7 @@ from .util import PrintError, print_error, log_exceptions, ignore_exceptions, bf from .bitcoin import COIN from . import constants from . import blockchain +from . import bitcoin from .blockchain import Blockchain, HEADER_SIZE from .interface import Interface, serialize_server, deserialize_server, RequestTimedOut from .version import PROTOCOL_VERSION @@ -177,10 +178,10 @@ class Network(PrintError): if config is None: config = {} # Do not use mutables as default values! self.config = SimpleConfig(config) if isinstance(config, dict) else config # type: SimpleConfig - blockchain.blockchains = blockchain.read_blockchains(self.config) - self.print_error("blockchains", list(blockchain.blockchains)) + blockchain.read_blockchains(self.config) + self.print_error("blockchains", list(map(lambda b: b.forkpoint, blockchain.blockchains.values()))) self._blockchain_preferred_block = self.config.get('blockchain_preferred_block', None) # type: Optional[Dict] - self._blockchain_index = 0 + self._blockchain = blockchain.get_best_chain() # Server for addresses and transactions self.default_server = self.config.get('server', None) # Sanitize default server @@ -321,7 +322,11 @@ class Network(PrintError): self.banner = await session.send_request('server.banner') self.notify('banner') async def get_donation_address(): - self.donation_address = await session.send_request('server.donation_address') + addr = await session.send_request('server.donation_address') + if not bitcoin.is_address(addr): + self.print_error(f"invalid donation address from server: {addr}") + addr = '' + self.donation_address = addr async def get_server_peers(): self.server_peers = parse_servers(await session.send_request('server.peers.subscribe')) self.notify('servers') @@ -559,17 +564,24 @@ class Network(PrintError): filtered = list(filter(lambda iface: iface.blockchain.check_hash(pref_height, pref_hash), interfaces)) if filtered: + self.print_error("switching to preferred fork") chosen_iface = random.choice(filtered) await self.switch_to_interface(chosen_iface.server) return - # try to switch to longest chain - if self.blockchain().parent_id is None: - return # already on longest chain - filtered = list(filter(lambda iface: iface.blockchain.parent_id is None, + else: + self.print_error("tried to switch to preferred fork but no interfaces are on it") + # try to switch to best chain + if self.blockchain().parent is None: + return # already on best chain + filtered = list(filter(lambda iface: iface.blockchain.parent is None, interfaces)) if filtered: + self.print_error("switching to best chain") chosen_iface = random.choice(filtered) await self.switch_to_interface(chosen_iface.server) + else: + # FIXME switch to best available? + self.print_error("tried to switch to best chain but no interfaces are on it") async def switch_to_interface(self, server: str): """Switch to server as our main interface. If no connection exists, @@ -637,7 +649,7 @@ class Network(PrintError): @ignore_exceptions # do not kill main_taskgroup @log_exceptions async def _run_new_interface(self, server): - interface = Interface(self, server, self.config.path, self.proxy) + interface = Interface(self, server, self.proxy) timeout = 10 if not self.proxy else 20 try: await asyncio.wait_for(interface.ready, timeout) @@ -661,7 +673,7 @@ class Network(PrintError): self.trigger_callback('network_updated') async def _init_headers_file(self): - b = blockchain.blockchains[0] + b = blockchain.get_best_chain() filename = b.path() length = HEADER_SIZE * len(constants.net.CHECKPOINTS) * 2016 if not os.path.exists(filename) or os.path.getsize(filename) < length: @@ -739,8 +751,8 @@ class Network(PrintError): def blockchain(self) -> Blockchain: interface = self.interface if interface and interface.blockchain is not None: - self._blockchain_index = interface.blockchain.forkpoint - return blockchain.blockchains[self._blockchain_index] + self._blockchain = interface.blockchain + return self._blockchain def get_blockchains(self): out = {} # blockchain_id -> list(interfaces) @@ -752,13 +764,6 @@ class Network(PrintError): out[chain_id] = r return out - async def disconnect_from_interfaces_on_given_blockchain(self, chain: Blockchain) -> Sequence[Interface]: - chain_id = chain.forkpoint - ifaces = self.get_blockchains().get(chain_id) or [] - for interface in ifaces: - await self.connection_down(interface.server) - return ifaces - def _set_preferred_chain(self, chain: Blockchain): height = chain.get_max_forkpoint() header_hash = chain.get_hash(height) @@ -768,7 +773,7 @@ class Network(PrintError): } self.config.set_key('blockchain_preferred_block', self._blockchain_preferred_block) - async def follow_chain_given_id(self, chain_id: int) -> None: + async def follow_chain_given_id(self, chain_id: str) -> None: bc = blockchain.blockchains.get(chain_id) if not bc: raise Exception('blockchain {} not found'.format(chain_id)) diff --git a/electrum/plugins/coldcard/coldcard.py b/electrum/plugins/coldcard/coldcard.py index afbb13843..96d80777f 100644 --- a/electrum/plugins/coldcard/coldcard.py +++ b/electrum/plugins/coldcard/coldcard.py @@ -118,6 +118,8 @@ class CKCCClient: or (self.dev.master_fingerprint != expected_xfp) or (self.dev.master_xpub != expected_xpub)): # probably indicating programing error, not hacking + print_error("[coldcard]", f"xpubs. reported by device: {self.dev.master_xpub}. " + f"stored in file: {expected_xpub}") raise RuntimeError("Expecting 0x%08x but that's not whats connected?!" % expected_xfp) @@ -454,9 +456,12 @@ class Coldcard_KeyStore(Hardware_KeyStore): # inputs section for txin in inputs: - utxo = txin['prev_tx'].outputs()[txin['prevout_n']] - spendable = txin['prev_tx'].serialize_output(utxo) - write_kv(PSBT_IN_WITNESS_UTXO, spendable) + if Transaction.is_segwit_input(txin): + utxo = txin['prev_tx'].outputs()[txin['prevout_n']] + spendable = txin['prev_tx'].serialize_output(utxo) + write_kv(PSBT_IN_WITNESS_UTXO, spendable) + else: + write_kv(PSBT_IN_NON_WITNESS_UTXO, str(txin['prev_tx'])) pubkeys, x_pubkeys = tx.get_sorted_pubkeys(txin) diff --git a/electrum/plugins/digitalbitbox/digitalbitbox.py b/electrum/plugins/digitalbitbox/digitalbitbox.py index 25e69e27d..dd93f77fd 100644 --- a/electrum/plugins/digitalbitbox/digitalbitbox.py +++ b/electrum/plugins/digitalbitbox/digitalbitbox.py @@ -4,7 +4,7 @@ # try: - from electrum.crypto import sha256d, EncodeAES, DecodeAES + from electrum.crypto import sha256d, EncodeAES_base64, DecodeAES_base64 from electrum.bitcoin import (TYPE_ADDRESS, push_script, var_int, public_key_to_p2pkh, is_address) from electrum.bip32 import serialize_xpub, deserialize_xpub @@ -396,10 +396,10 @@ class DigitalBitbox_Client(): reply = "" try: secret = sha256d(self.password) - msg = EncodeAES(secret, msg) + msg = EncodeAES_base64(secret, msg) reply = self.hid_send_plain(msg) if 'ciphertext' in reply: - reply = DecodeAES(secret, ''.join(reply["ciphertext"])) + reply = DecodeAES_base64(secret, ''.join(reply["ciphertext"])) reply = to_string(reply, 'utf8') reply = json.loads(reply) if 'error' in reply: @@ -716,7 +716,7 @@ class DigitalBitboxPlugin(HW_PluginBase): key_s = base64.b64decode(self.digitalbitbox_config['encryptionprivkey']) args = 'c=data&s=0&dt=0&uuid=%s&pl=%s' % ( self.digitalbitbox_config['comserverchannelid'], - EncodeAES(key_s, json.dumps(payload).encode('ascii')).decode('ascii'), + EncodeAES_base64(key_s, json.dumps(payload).encode('ascii')).decode('ascii'), ) try: requests.post(url, args) diff --git a/electrum/plugins/hw_wallet/qt.py b/electrum/plugins/hw_wallet/qt.py index 2b5215eb6..fc188ad00 100644 --- a/electrum/plugins/hw_wallet/qt.py +++ b/electrum/plugins/hw_wallet/qt.py @@ -27,7 +27,8 @@ import threading from PyQt5.Qt import QVBoxLayout, QLabel -from electrum.gui.qt.password_dialog import PasswordDialog, PW_PASSPHRASE + +from electrum.gui.qt.password_dialog import PasswordLayout, PW_PASSPHRASE from electrum.gui.qt.util import * from electrum.i18n import _ @@ -114,11 +115,16 @@ class QtHandlerBase(QObject, PrintError): def passphrase_dialog(self, msg, confirm): # If confirm is true, require the user to enter the passphrase twice parent = self.top_level_window() + d = WindowModalDialog(parent, _("Enter Passphrase")) if confirm: - d = PasswordDialog(parent, None, msg, PW_PASSPHRASE) - confirmed, p, passphrase = d.run() + OK_button = OkButton(d) + playout = PasswordLayout(msg=msg, kind=PW_PASSPHRASE, OK_button=OK_button) + vbox = QVBoxLayout() + vbox.addLayout(playout.layout()) + vbox.addLayout(Buttons(CancelButton(d), OK_button)) + d.setLayout(vbox) + passphrase = playout.new_password() if d.exec_() else None else: - d = WindowModalDialog(parent, _("Enter Passphrase")) pw = QLineEdit() pw.setEchoMode(2) pw.setMinimumWidth(200) diff --git a/electrum/plugins/labels/labels.py b/electrum/plugins/labels/labels.py index 9405e70ed..3c5ff2068 100644 --- a/electrum/plugins/labels/labels.py +++ b/electrum/plugins/labels/labels.py @@ -73,7 +73,10 @@ class LabelsPlugin(BasePlugin): url = 'https://' + self.target_host + url async with make_aiohttp_session(self.proxy) as session: async with session.post(url, json=data) as result: - return await result.json() + try: + return await result.json() + except Exception as e: + raise Exception('Could not decode: ' + await result.text()) from e async def push_thread(self, wallet): wallet_data = self.wallets.get(wallet, None) diff --git a/electrum/qrscanner.py b/electrum/qrscanner.py index 463d5ec6f..ab1f1341e 100644 --- a/electrum/qrscanner.py +++ b/electrum/qrscanner.py @@ -40,7 +40,7 @@ except BaseException: libzbar = None -def scan_barcode(device='', timeout=-1, display=True, threaded=False, try_again=True): +def scan_barcode_ctypes(device='', timeout=-1, display=True, threaded=False, try_again=True): if libzbar is None: raise RuntimeError("Cannot start QR scanner; zbar not available.") libzbar.zbar_symbol_get_data.restype = ctypes.c_char_p @@ -69,6 +69,29 @@ def scan_barcode(device='', timeout=-1, display=True, threaded=False, try_again= data = libzbar.zbar_symbol_get_data(symbol) return data.decode('utf8') +def scan_barcode_osx(*args_ignored, **kwargs_ignored): + import subprocess + # NOTE: This code needs to be modified if the positions of this file changes with respect to the helper app! + # This assumes the built macOS .app bundle which ends up putting the helper app in + # .app/contrib/osx/CalinsQRReader/build/Release/CalinsQRReader.app. + root_ec_dir = os.path.abspath(os.path.dirname(__file__) + "/../") + prog = root_ec_dir + "/" + "contrib/osx/CalinsQRReader/build/Release/CalinsQRReader.app/Contents/MacOS/CalinsQRReader" + if not os.path.exists(prog): + raise RuntimeError("Cannot start QR scanner; helper app not found.") + data = '' + try: + # This will run the "CalinsQRReader" helper app (which also gets bundled with the built .app) + # Just like the zbar implementation -- the main app will hang until the QR window returns a QR code + # (or is closed). Communication with the subprocess is done via stdout. + # See contrib/CalinsQRReader for the helper app source code. + with subprocess.Popen([prog], stdout=subprocess.PIPE) as p: + data = p.stdout.read().decode('utf-8').strip() + return data + except OSError as e: + raise RuntimeError("Cannot start camera helper app; {}".format(e.strerror)) + +scan_barcode = scan_barcode_osx if sys.platform == 'darwin' else scan_barcode_ctypes + def _find_system_cameras(): device_root = "/sys/class/video4linux" devices = {} # Name -> device diff --git a/electrum/storage.py b/electrum/storage.py index 16a4cc90d..d526000e2 100644 --- a/electrum/storage.py +++ b/electrum/storage.py @@ -122,12 +122,7 @@ class JsonDB(PrintError): os.fsync(f.fileno()) mode = os.stat(self.path).st_mode if os.path.exists(self.path) else stat.S_IREAD | stat.S_IWRITE - # perform atomic write on POSIX systems - try: - os.rename(temp_path, self.path) - except: - os.remove(self.path) - os.rename(temp_path, self.path) + os.replace(temp_path, self.path) os.chmod(self.path, mode) self.print_error("saved", self.path) self.modified = False diff --git a/electrum/tests/test_bitcoin.py b/electrum/tests/test_bitcoin.py index cdf1257f1..c19473f10 100644 --- a/electrum/tests/test_bitcoin.py +++ b/electrum/tests/test_bitcoin.py @@ -11,11 +11,11 @@ from electrum.bitcoin import (public_key_to_p2pkh, address_from_private_key, from electrum.bip32 import (bip32_root, bip32_public_derivation, bip32_private_derivation, xpub_from_xprv, xpub_type, is_xprv, is_bip32_derivation, is_xpub, convert_bip32_path_to_list_of_uint32) -from electrum.crypto import sha256d +from electrum.crypto import sha256d, KNOWN_PW_HASH_VERSIONS from electrum import ecc, crypto, constants from electrum.ecc import number_to_string, string_to_number from electrum.transaction import opcodes -from electrum.util import bfh, bh2u +from electrum.util import bfh, bh2u, InvalidPassword from electrum.storage import WalletStorage from electrum.keystore import xtype_from_derivation @@ -219,23 +219,26 @@ class Test_bitcoin(SequentialTestCase): """Make sure AES is homomorphic.""" payload = u'\u66f4\u7a33\u5b9a\u7684\u4ea4\u6613\u5e73\u53f0' password = u'secret' - enc = crypto.pw_encode(payload, password) - dec = crypto.pw_decode(enc, password) - self.assertEqual(dec, payload) + for version in KNOWN_PW_HASH_VERSIONS: + enc = crypto.pw_encode(payload, password, version=version) + dec = crypto.pw_decode(enc, password, version=version) + self.assertEqual(dec, payload) @needs_test_with_all_aes_implementations def test_aes_encode_without_password(self): """When not passed a password, pw_encode is noop on the payload.""" payload = u'\u66f4\u7a33\u5b9a\u7684\u4ea4\u6613\u5e73\u53f0' - enc = crypto.pw_encode(payload, None) - self.assertEqual(payload, enc) + for version in KNOWN_PW_HASH_VERSIONS: + enc = crypto.pw_encode(payload, None, version=version) + self.assertEqual(payload, enc) @needs_test_with_all_aes_implementations def test_aes_deencode_without_password(self): """When not passed a password, pw_decode is noop on the payload.""" payload = u'\u66f4\u7a33\u5b9a\u7684\u4ea4\u6613\u5e73\u53f0' - enc = crypto.pw_decode(payload, None) - self.assertEqual(payload, enc) + for version in KNOWN_PW_HASH_VERSIONS: + enc = crypto.pw_decode(payload, None, version=version) + self.assertEqual(payload, enc) @needs_test_with_all_aes_implementations def test_aes_decode_with_invalid_password(self): @@ -243,8 +246,10 @@ class Test_bitcoin(SequentialTestCase): payload = u"blah" password = u"uber secret" wrong_password = u"not the password" - enc = crypto.pw_encode(payload, password) - self.assertRaises(Exception, crypto.pw_decode, enc, wrong_password) + for version in KNOWN_PW_HASH_VERSIONS: + enc = crypto.pw_encode(payload, password, version=version) + with self.assertRaises(InvalidPassword): + crypto.pw_decode(enc, wrong_password, version=version) def test_sha256d(self): self.assertEqual(b'\x95MZI\xfdp\xd9\xb8\xbc\xdb5\xd2R&x)\x95\x7f~\xf7\xfalt\xf8\x84\x19\xbd\xc5\xe8"\t\xf4', diff --git a/electrum/tests/test_blockchain.py b/electrum/tests/test_blockchain.py new file mode 100644 index 000000000..be29c1b03 --- /dev/null +++ b/electrum/tests/test_blockchain.py @@ -0,0 +1,239 @@ +import shutil +import tempfile +import os + +from electrum import constants, blockchain +from electrum.simple_config import SimpleConfig +from electrum.blockchain import Blockchain, deserialize_header, hash_header +from electrum.util import bh2u, bfh, make_dir + +from . import SequentialTestCase + + +class TestBlockchain(SequentialTestCase): + + HEADERS = { + 'A': deserialize_header(bfh("0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4adae5494dffff7f2002000000"), 0), + 'B': deserialize_header(bfh("0000002006226e46111a0b59caaf126043eb5bbf28c34f3a5e332a1fc7b2b73cf188910f186c8dfd970a4545f79916bc1d75c9d00432f57c89209bf3bb115b7612848f509c25f45bffff7f2000000000"), 1), + 'C': deserialize_header(bfh("00000020686bdfc6a3db73d5d93e8c9663a720a26ecb1ef20eb05af11b36cdbc57c19f7ebf2cbf153013a1c54abaf70e95198fcef2f3059cc6b4d0f7e876808e7d24d11cc825f45bffff7f2000000000"), 2), + 'D': deserialize_header(bfh("00000020122baa14f3ef54985ae546d1611559e3f487bd2a0f46e8dbb52fbacc9e237972e71019d7feecd9b8596eca9a67032c5f4641b23b5d731dc393e37de7f9c2f299e725f45bffff7f2000000000"), 3), + 'E': deserialize_header(bfh("00000020f8016f7ef3a17d557afe05d4ea7ab6bde1b2247b7643896c1b63d43a1598b747a3586da94c71753f27c075f57f44faf913c31177a0957bbda42e7699e3a2141aed25f45bffff7f2001000000"), 4), + 'F': deserialize_header(bfh("000000201d589c6643c1d121d73b0573e5ee58ab575b8fdf16d507e7e915c5fbfbbfd05e7aee1d692d1615c3bdf52c291032144ce9e3b258a473c17c745047f3431ff8e2ee25f45bffff7f2000000000"), 5), + 'O': deserialize_header(bfh("00000020b833ed46eea01d4c980f59feee44a66aa1162748b6801029565d1466790c405c3a141ce635cbb1cd2b3a4fcdd0a3380517845ba41736c82a79cab535d31128066526f45bffff7f2001000000"), 6), + 'P': deserialize_header(bfh("00000020abe8e119d1877c9dc0dc502d1a253fb9a67967c57732d2f71ee0280e8381ff0a9690c2fe7c1a4450c74dc908fe94dd96c3b0637d51475e9e06a78e944a0c7fe28126f45bffff7f2000000000"), 7), + 'Q': deserialize_header(bfh("000000202ce41d94eb70e1518bc1f72523f84a903f9705d967481e324876e1f8cf4d3452148be228a4c3f2061bafe7efdfc4a8d5a94759464b9b5c619994d45dfcaf49e1a126f45bffff7f2000000000"), 8), + 'R': deserialize_header(bfh("00000020552755b6c59f3d51e361d16281842a4e166007799665b5daed86a063dd89857415681cb2d00ff889193f6a68a93f5096aeb2d84ca0af6185a462555822552221a626f45bffff7f2000000000"), 9), + 'S': deserialize_header(bfh("00000020a13a491cbefc93cd1bb1938f19957e22a134faf14c7dee951c45533e2c750f239dc087fc977b06c24a69c682d1afd1020e6dc1f087571ccec66310a786e1548fab26f45bffff7f2000000000"), 10), + 'T': deserialize_header(bfh("00000020dbf3a9b55dfefbaf8b6e43a89cf833fa2e208bbc0c1c5d76c0d71b9e4a65337803b243756c25053253aeda309604363460a3911015929e68705bd89dff6fe064b026f45bffff7f2002000000"), 11), + 'U': deserialize_header(bfh("000000203d0932b3b0c78eccb39a595a28ae4a7c966388648d7783fd1305ec8d40d4fe5fd67cb902a7d807cee7676cb543feec3e053aa824d5dfb528d5b94f9760313d9db726f45bffff7f2001000000"), 12), + 'G': deserialize_header(bfh("00000020b833ed46eea01d4c980f59feee44a66aa1162748b6801029565d1466790c405c3a141ce635cbb1cd2b3a4fcdd0a3380517845ba41736c82a79cab535d31128066928f45bffff7f2001000000"), 6), + 'H': deserialize_header(bfh("00000020e19e687f6e7f83ca394c114144dbbbc4f3f9c9450f66331a125413702a2e1a719690c2fe7c1a4450c74dc908fe94dd96c3b0637d51475e9e06a78e944a0c7fe26a28f45bffff7f2002000000"), 7), + 'I': deserialize_header(bfh("0000002009dcb3b158293c89d7cf7ceeb513add122ebc3880a850f47afbb2747f5e48c54148be228a4c3f2061bafe7efdfc4a8d5a94759464b9b5c619994d45dfcaf49e16a28f45bffff7f2000000000"), 8), + 'J': deserialize_header(bfh("000000206a65f3bdd3374a5a6c4538008ba0b0a560b8566291f9ef4280ab877627a1742815681cb2d00ff889193f6a68a93f5096aeb2d84ca0af6185a462555822552221c928f45bffff7f2000000000"), 9), + 'K': deserialize_header(bfh("00000020bb3b421653548991998f96f8ba486b652fdb07ca16e9cee30ece033547cd1a6e9dc087fc977b06c24a69c682d1afd1020e6dc1f087571ccec66310a786e1548fca28f45bffff7f2000000000"), 10), + 'L': deserialize_header(bfh("00000020c391d74d37c24a130f4bf4737932bdf9e206dd4fad22860ec5408978eb55d46303b243756c25053253aeda309604363460a3911015929e68705bd89dff6fe064ca28f45bffff7f2000000000"), 11), + 'M': deserialize_header(bfh("000000206a65f3bdd3374a5a6c4538008ba0b0a560b8566291f9ef4280ab877627a1742815681cb2d00ff889193f6a68a93f5096aeb2d84ca0af6185a4625558225522214229f45bffff7f2000000000"), 9), + 'N': deserialize_header(bfh("00000020383dab38b57f98aa9b4f0d5ff868bc674b4828d76766bf048296f4c45fff680a9dc087fc977b06c24a69c682d1afd1020e6dc1f087571ccec66310a786e1548f4329f45bffff7f2003000000"), 10), + 'X': deserialize_header(bfh("0000002067f1857f54b7fef732cb4940f7d1b339472b3514660711a820330fd09d8fba6b03b243756c25053253aeda309604363460a3911015929e68705bd89dff6fe0649b29f45bffff7f2002000000"), 11), + 'Y': deserialize_header(bfh("00000020db33c9768a9e5f7c37d0f09aad88d48165946c87d08f7d63793f07b5c08c527fd67cb902a7d807cee7676cb543feec3e053aa824d5dfb528d5b94f9760313d9d9b29f45bffff7f2000000000"), 12), + 'Z': deserialize_header(bfh("0000002047822b67940e337fda38be6f13390b3596e4dea2549250256879722073824e7f0f2596c29203f8a0f71ae94193092dc8f113be3dbee4579f1e649fa3d6dcc38c622ef45bffff7f2003000000"), 13), + } + # tree of headers: + # - M <- N <- X <- Y <- Z + # / + # - G <- H <- I <- J <- K <- L + # / + # A <- B <- C <- D <- E <- F <- O <- P <- Q <- R <- S <- T <- U + + @classmethod + def setUpClass(cls): + super().setUpClass() + constants.set_regtest() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + constants.set_mainnet() + + def setUp(self): + super().setUp() + self.data_dir = tempfile.mkdtemp() + make_dir(os.path.join(self.data_dir, 'forks')) + self.config = SimpleConfig({'electrum_path': self.data_dir}) + blockchain.blockchains = {} + + def tearDown(self): + super().tearDown() + shutil.rmtree(self.data_dir) + + def _append_header(self, chain: Blockchain, header: dict): + self.assertTrue(chain.can_connect(header)) + chain.save_header(header) + + def test_forking_and_swapping(self): + blockchain.blockchains[constants.net.GENESIS] = chain_u = Blockchain( + config=self.config, forkpoint=0, parent=None, + forkpoint_hash=constants.net.GENESIS, prev_hash=None) + open(chain_u.path(), 'w+').close() + + self._append_header(chain_u, self.HEADERS['A']) + self._append_header(chain_u, self.HEADERS['B']) + self._append_header(chain_u, self.HEADERS['C']) + self._append_header(chain_u, self.HEADERS['D']) + self._append_header(chain_u, self.HEADERS['E']) + self._append_header(chain_u, self.HEADERS['F']) + self._append_header(chain_u, self.HEADERS['O']) + self._append_header(chain_u, self.HEADERS['P']) + self._append_header(chain_u, self.HEADERS['Q']) + self._append_header(chain_u, self.HEADERS['R']) + + chain_l = chain_u.fork(self.HEADERS['G']) + self._append_header(chain_l, self.HEADERS['H']) + self._append_header(chain_l, self.HEADERS['I']) + self._append_header(chain_l, self.HEADERS['J']) + + # do checks + self.assertEqual(2, len(blockchain.blockchains)) + self.assertEqual(1, len(os.listdir(os.path.join(self.data_dir, "forks")))) + self.assertEqual(0, chain_u.forkpoint) + self.assertEqual(None, chain_u.parent) + self.assertEqual(constants.net.GENESIS, chain_u._forkpoint_hash) + self.assertEqual(None, chain_u._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "blockchain_headers"), chain_u.path()) + self.assertEqual(10 * 80, os.stat(chain_u.path()).st_size) + self.assertEqual(6, chain_l.forkpoint) + self.assertEqual(chain_u, chain_l.parent) + self.assertEqual(hash_header(self.HEADERS['G']), chain_l._forkpoint_hash) + self.assertEqual(hash_header(self.HEADERS['F']), chain_l._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "forks", "fork2_6_5c400c7966145d56291080b6482716a16aa644eefe590f984c1da0ee46ed33b8_711a2e2a701354121a33660f45c9f9f3c4bbdb4441114c39ca837f6e7f689ee1"), chain_l.path()) + self.assertEqual(4 * 80, os.stat(chain_l.path()).st_size) + + self._append_header(chain_l, self.HEADERS['K']) + + # chains were swapped, do checks + self.assertEqual(2, len(blockchain.blockchains)) + self.assertEqual(1, len(os.listdir(os.path.join(self.data_dir, "forks")))) + self.assertEqual(6, chain_u.forkpoint) + self.assertEqual(chain_l, chain_u.parent) + self.assertEqual(hash_header(self.HEADERS['O']), chain_u._forkpoint_hash) + self.assertEqual(hash_header(self.HEADERS['F']), chain_u._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "forks", "fork2_6_5c400c7966145d56291080b6482716a16aa644eefe590f984c1da0ee46ed33b8_aff81830e28e01ef7d23277c56779a6b93f251a2d50dcc09d7c87d119e1e8ab"), chain_u.path()) + self.assertEqual(4 * 80, os.stat(chain_u.path()).st_size) + self.assertEqual(0, chain_l.forkpoint) + self.assertEqual(None, chain_l.parent) + self.assertEqual(constants.net.GENESIS, chain_l._forkpoint_hash) + self.assertEqual(None, chain_l._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "blockchain_headers"), chain_l.path()) + self.assertEqual(11 * 80, os.stat(chain_l.path()).st_size) + for b in (chain_u, chain_l): + self.assertTrue(all([b.can_connect(b.read_header(i), False) for i in range(b.height())])) + + self._append_header(chain_u, self.HEADERS['S']) + self._append_header(chain_u, self.HEADERS['T']) + self._append_header(chain_u, self.HEADERS['U']) + self._append_header(chain_l, self.HEADERS['L']) + + chain_z = chain_l.fork(self.HEADERS['M']) + self._append_header(chain_z, self.HEADERS['N']) + self._append_header(chain_z, self.HEADERS['X']) + self._append_header(chain_z, self.HEADERS['Y']) + self._append_header(chain_z, self.HEADERS['Z']) + + # chain_z became best chain, do checks + self.assertEqual(3, len(blockchain.blockchains)) + self.assertEqual(2, len(os.listdir(os.path.join(self.data_dir, "forks")))) + self.assertEqual(0, chain_z.forkpoint) + self.assertEqual(None, chain_z.parent) + self.assertEqual(constants.net.GENESIS, chain_z._forkpoint_hash) + self.assertEqual(None, chain_z._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "blockchain_headers"), chain_z.path()) + self.assertEqual(14 * 80, os.stat(chain_z.path()).st_size) + self.assertEqual(9, chain_l.forkpoint) + self.assertEqual(chain_z, chain_l.parent) + self.assertEqual(hash_header(self.HEADERS['J']), chain_l._forkpoint_hash) + self.assertEqual(hash_header(self.HEADERS['I']), chain_l._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "forks", "fork2_9_2874a1277687ab8042eff9916256b860a5b0a08b0038456c5a4a37d3bdf3656a_6e1acd473503ce0ee3cee916ca07db2f656b48baf8968f999189545316423bbb"), chain_l.path()) + self.assertEqual(3 * 80, os.stat(chain_l.path()).st_size) + self.assertEqual(6, chain_u.forkpoint) + self.assertEqual(chain_z, chain_u.parent) + self.assertEqual(hash_header(self.HEADERS['O']), chain_u._forkpoint_hash) + self.assertEqual(hash_header(self.HEADERS['F']), chain_u._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "forks", "fork2_6_5c400c7966145d56291080b6482716a16aa644eefe590f984c1da0ee46ed33b8_aff81830e28e01ef7d23277c56779a6b93f251a2d50dcc09d7c87d119e1e8ab"), chain_u.path()) + self.assertEqual(7 * 80, os.stat(chain_u.path()).st_size) + for b in (chain_u, chain_l, chain_z): + self.assertTrue(all([b.can_connect(b.read_header(i), False) for i in range(b.height())])) + + self.assertEqual(constants.net.GENESIS, chain_z.get_hash(0)) + self.assertEqual(hash_header(self.HEADERS['F']), chain_z.get_hash(5)) + self.assertEqual(hash_header(self.HEADERS['G']), chain_z.get_hash(6)) + self.assertEqual(hash_header(self.HEADERS['I']), chain_z.get_hash(8)) + self.assertEqual(hash_header(self.HEADERS['M']), chain_z.get_hash(9)) + self.assertEqual(hash_header(self.HEADERS['Z']), chain_z.get_hash(13)) + + def test_doing_multiple_swaps_after_single_new_header(self): + blockchain.blockchains[constants.net.GENESIS] = chain_u = Blockchain( + config=self.config, forkpoint=0, parent=None, + forkpoint_hash=constants.net.GENESIS, prev_hash=None) + open(chain_u.path(), 'w+').close() + + self._append_header(chain_u, self.HEADERS['A']) + self._append_header(chain_u, self.HEADERS['B']) + self._append_header(chain_u, self.HEADERS['C']) + self._append_header(chain_u, self.HEADERS['D']) + self._append_header(chain_u, self.HEADERS['E']) + self._append_header(chain_u, self.HEADERS['F']) + self._append_header(chain_u, self.HEADERS['O']) + self._append_header(chain_u, self.HEADERS['P']) + self._append_header(chain_u, self.HEADERS['Q']) + self._append_header(chain_u, self.HEADERS['R']) + self._append_header(chain_u, self.HEADERS['S']) + + self.assertEqual(1, len(blockchain.blockchains)) + self.assertEqual(0, len(os.listdir(os.path.join(self.data_dir, "forks")))) + + chain_l = chain_u.fork(self.HEADERS['G']) + self._append_header(chain_l, self.HEADERS['H']) + self._append_header(chain_l, self.HEADERS['I']) + self._append_header(chain_l, self.HEADERS['J']) + self._append_header(chain_l, self.HEADERS['K']) + # now chain_u is best chain, but it's tied with chain_l + + self.assertEqual(2, len(blockchain.blockchains)) + self.assertEqual(1, len(os.listdir(os.path.join(self.data_dir, "forks")))) + + chain_z = chain_l.fork(self.HEADERS['M']) + self._append_header(chain_z, self.HEADERS['N']) + self._append_header(chain_z, self.HEADERS['X']) + + self.assertEqual(3, len(blockchain.blockchains)) + self.assertEqual(2, len(os.listdir(os.path.join(self.data_dir, "forks")))) + + # chain_z became best chain, do checks + self.assertEqual(0, chain_z.forkpoint) + self.assertEqual(None, chain_z.parent) + self.assertEqual(constants.net.GENESIS, chain_z._forkpoint_hash) + self.assertEqual(None, chain_z._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "blockchain_headers"), chain_z.path()) + self.assertEqual(12 * 80, os.stat(chain_z.path()).st_size) + self.assertEqual(9, chain_l.forkpoint) + self.assertEqual(chain_z, chain_l.parent) + self.assertEqual(hash_header(self.HEADERS['J']), chain_l._forkpoint_hash) + self.assertEqual(hash_header(self.HEADERS['I']), chain_l._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "forks", "fork2_9_2874a1277687ab8042eff9916256b860a5b0a08b0038456c5a4a37d3bdf3656a_6e1acd473503ce0ee3cee916ca07db2f656b48baf8968f999189545316423bbb"), chain_l.path()) + self.assertEqual(2 * 80, os.stat(chain_l.path()).st_size) + self.assertEqual(6, chain_u.forkpoint) + self.assertEqual(chain_z, chain_u.parent) + self.assertEqual(hash_header(self.HEADERS['O']), chain_u._forkpoint_hash) + self.assertEqual(hash_header(self.HEADERS['F']), chain_u._prev_hash) + self.assertEqual(os.path.join(self.data_dir, "forks", "fork2_6_5c400c7966145d56291080b6482716a16aa644eefe590f984c1da0ee46ed33b8_aff81830e28e01ef7d23277c56779a6b93f251a2d50dcc09d7c87d119e1e8ab"), chain_u.path()) + self.assertEqual(5 * 80, os.stat(chain_u.path()).st_size) + + self.assertEqual(constants.net.GENESIS, chain_z.get_hash(0)) + self.assertEqual(hash_header(self.HEADERS['F']), chain_z.get_hash(5)) + self.assertEqual(hash_header(self.HEADERS['G']), chain_z.get_hash(6)) + self.assertEqual(hash_header(self.HEADERS['I']), chain_z.get_hash(8)) + self.assertEqual(hash_header(self.HEADERS['M']), chain_z.get_hash(9)) + self.assertEqual(hash_header(self.HEADERS['X']), chain_z.get_hash(11)) + + for b in (chain_u, chain_l, chain_z): + self.assertTrue(all([b.can_connect(b.read_header(i), False) for i in range(b.height())])) diff --git a/electrum/tests/test_network.py b/electrum/tests/test_network.py index c69375bd6..ece54056f 100644 --- a/electrum/tests/test_network.py +++ b/electrum/tests/test_network.py @@ -6,6 +6,9 @@ from electrum import constants from electrum.simple_config import SimpleConfig from electrum import blockchain from electrum.interface import Interface +from electrum.crypto import sha256 +from electrum.util import bh2u + class MockTaskGroup: async def spawn(self, x): return @@ -17,10 +20,14 @@ class MockNetwork: class MockInterface(Interface): def __init__(self, config): self.config = config - super().__init__(MockNetwork(), 'mock-server:50000:t', self.config.electrum_path(), None) + network = MockNetwork() + network.config = config + super().__init__(network, 'mock-server:50000:t', None) self.q = asyncio.Queue() - self.blockchain = blockchain.Blockchain(self.config, 2002, None) + self.blockchain = blockchain.Blockchain(config=self.config, forkpoint=0, + parent=None, forkpoint_hash=constants.net.GENESIS, prev_hash=None) self.tip = 12 + self.blockchain._size = self.tip + 1 async def get_block_header(self, height, assert_mode): assert self.q.qsize() > 0, (height, assert_mode) item = await self.q.get() @@ -56,7 +63,7 @@ class TestNetwork(unittest.TestCase): self.interface.q.put_nowait({'block_height': 5, 'mock': {'binary':1,'check':lambda x: True, 'connect': lambda x: True}}) self.interface.q.put_nowait({'block_height': 6, 'mock': {'binary':1,'check':lambda x: True, 'connect': lambda x: True}}) ifa = self.interface - self.assertEqual(('fork_noconflict', 8), asyncio.get_event_loop().run_until_complete(ifa.sync_until(8, next_height=7))) + self.assertEqual(('fork', 8), asyncio.get_event_loop().run_until_complete(ifa.sync_until(8, next_height=7))) self.assertEqual(self.interface.q.qsize(), 0) def test_fork_conflict(self): @@ -70,7 +77,7 @@ class TestNetwork(unittest.TestCase): self.interface.q.put_nowait({'block_height': 5, 'mock': {'binary':1,'check':lambda x: True, 'connect': lambda x: True}}) self.interface.q.put_nowait({'block_height': 6, 'mock': {'binary':1,'check':lambda x: True, 'connect': lambda x: True}}) ifa = self.interface - self.assertEqual(('fork_conflict', 8), asyncio.get_event_loop().run_until_complete(ifa.sync_until(8, next_height=7))) + self.assertEqual(('fork', 8), asyncio.get_event_loop().run_until_complete(ifa.sync_until(8, next_height=7))) self.assertEqual(self.interface.q.qsize(), 0) def test_can_connect_during_backward(self): @@ -87,7 +94,10 @@ class TestNetwork(unittest.TestCase): self.assertEqual(self.interface.q.qsize(), 0) def mock_fork(self, bad_header): - return blockchain.Blockchain(self.config, bad_header['block_height'], None) + forkpoint = bad_header['block_height'] + b = blockchain.Blockchain(config=self.config, forkpoint=forkpoint, parent=None, + forkpoint_hash=bh2u(sha256(str(forkpoint))), prev_hash=bh2u(sha256(str(forkpoint-1)))) + return b def test_chain_false_during_binary(self): blockchain.blockchains = {} diff --git a/electrum/tests/test_wallet.py b/electrum/tests/test_wallet.py index 9117392ea..275f20bba 100644 --- a/electrum/tests/test_wallet.py +++ b/electrum/tests/test_wallet.py @@ -3,9 +3,16 @@ import tempfile import sys import os import json +from decimal import Decimal +from unittest import TestCase +import time from io import StringIO from electrum.storage import WalletStorage, FINAL_SEED_VERSION +from electrum.wallet import Abstract_Wallet +from electrum.exchange_rate import ExchangeBase, FxThread +from electrum.util import TxMinedStatus +from electrum.bitcoin import COIN from . import SequentialTestCase @@ -64,7 +71,70 @@ class TestWalletStorage(WalletTestCase): storage.put(key, value) storage.write() - contents = "" with open(self.wallet_path, "r") as f: contents = f.read() self.assertEqual(some_dict, json.loads(contents)) + +class FakeExchange(ExchangeBase): + def __init__(self, rate): + super().__init__(lambda self: None, lambda self: None) + self.quotes = {'TEST': rate} + +class FakeFxThread: + def __init__(self, exchange): + self.exchange = exchange + self.ccy = 'TEST' + + remove_thousands_separator = staticmethod(FxThread.remove_thousands_separator) + timestamp_rate = FxThread.timestamp_rate + ccy_amount_str = FxThread.ccy_amount_str + history_rate = FxThread.history_rate + +class FakeWallet: + def __init__(self, fiat_value): + super().__init__() + self.fiat_value = fiat_value + self.transactions = self.verified_tx = {'abc': 'Tx'} + + def get_tx_height(self, txid): + # because we use a current timestamp, and history is empty, + # FxThread.history_rate will use spot prices + return TxMinedStatus(height=10, conf=10, timestamp=time.time(), header_hash='def') + + default_fiat_value = Abstract_Wallet.default_fiat_value + price_at_timestamp = Abstract_Wallet.price_at_timestamp + class storage: + put = lambda self, x: None + +txid = 'abc' +ccy = 'TEST' + +class TestFiat(TestCase): + def setUp(self): + self.value_sat = COIN + self.fiat_value = {} + self.wallet = FakeWallet(fiat_value=self.fiat_value) + self.fx = FakeFxThread(FakeExchange(Decimal('1000.001'))) + default_fiat = Abstract_Wallet.default_fiat_value(self.wallet, txid, self.fx, self.value_sat) + self.assertEqual(Decimal('1000.001'), default_fiat) + self.assertEqual('1,000.00', self.fx.ccy_amount_str(default_fiat, commas=True)) + + def test_save_fiat_and_reset(self): + self.assertEqual(False, Abstract_Wallet.set_fiat_value(self.wallet, txid, ccy, '1000.01', self.fx, self.value_sat)) + saved = self.fiat_value[ccy][txid] + self.assertEqual('1,000.01', self.fx.ccy_amount_str(Decimal(saved), commas=True)) + self.assertEqual(True, Abstract_Wallet.set_fiat_value(self.wallet, txid, ccy, '', self.fx, self.value_sat)) + self.assertNotIn(txid, self.fiat_value[ccy]) + # even though we are not setting it to the exact fiat value according to the exchange rate, precision is truncated away + self.assertEqual(True, Abstract_Wallet.set_fiat_value(self.wallet, txid, ccy, '1,000.002', self.fx, self.value_sat)) + + def test_too_high_precision_value_resets_with_no_saved_value(self): + self.assertEqual(True, Abstract_Wallet.set_fiat_value(self.wallet, txid, ccy, '1,000.001', self.fx, self.value_sat)) + + def test_empty_resets(self): + self.assertEqual(True, Abstract_Wallet.set_fiat_value(self.wallet, txid, ccy, '', self.fx, self.value_sat)) + self.assertNotIn(ccy, self.fiat_value) + + def test_save_garbage(self): + self.assertEqual(False, Abstract_Wallet.set_fiat_value(self.wallet, txid, ccy, 'garbage', self.fx, self.value_sat)) + self.assertNotIn(ccy, self.fiat_value) diff --git a/electrum/util.py b/electrum/util.py index 9a0e81d2c..ec28136e1 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -39,6 +39,7 @@ import urllib.request, urllib.parse, urllib.error import builtins import json import time +from typing import NamedTuple, Optional import aiohttp from aiohttp_socks import SocksConnector, SocksVer @@ -129,31 +130,15 @@ class UserCancelled(Exception): '''An exception that is suppressed from the user''' pass -class Satoshis(object): - __slots__ = ('value',) - - def __new__(cls, value): - self = super(Satoshis, cls).__new__(cls) - self.value = value - return self - - def __repr__(self): - return 'Satoshis(%d)'%self.value +class Satoshis(NamedTuple): + value: int def __str__(self): return format_satoshis(self.value) + " BTC" -class Fiat(object): - __slots__ = ('value', 'ccy') - - def __new__(cls, value, ccy): - self = super(Fiat, cls).__new__(cls) - self.ccy = ccy - self.value = value - return self - - def __repr__(self): - return 'Fiat(%s)'% self.__str__() +class Fiat(NamedTuple): + value: Optional[Decimal] + ccy: str def __str__(self): if self.value is None or self.value.is_nan(): diff --git a/electrum/version.py b/electrum/version.py index 53387af39..5866941f3 100644 --- a/electrum/version.py +++ b/electrum/version.py @@ -1,5 +1,5 @@ -ELECTRUM_VERSION = '3.2.3' # version of the client package -APK_VERSION = '3.2.3.1' # read by buildozer.spec +ELECTRUM_VERSION = '3.3.0' # version of the client package +APK_VERSION = '3.3.0.0' # read by buildozer.spec PROTOCOL_VERSION = '1.4' # protocol version requested diff --git a/electrum/wallet.py b/electrum/wallet.py index c104d7bee..b38f441da 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -182,7 +182,7 @@ class Abstract_Wallet(AddressSynchronizer): self.invoices = InvoiceStore(self.storage) self.contacts = Contacts(self.storage) - self.coin_price_cache = {} + self._coin_price_cache = {} def load_and_cleanup(self): self.load_keystore() @@ -247,24 +247,37 @@ class Abstract_Wallet(AddressSynchronizer): self.storage.put('labels', self.labels) return changed - def set_fiat_value(self, txid, ccy, text): + def set_fiat_value(self, txid, ccy, text, fx, value_sat): if txid not in self.transactions: return - if not text: + # since fx is inserting the thousands separator, + # and not util, also have fx remove it + text = fx.remove_thousands_separator(text) + def_fiat = self.default_fiat_value(txid, fx, value_sat) + formatted = fx.ccy_amount_str(def_fiat, commas=False) + def_fiat_rounded = Decimal(formatted) + reset = not text + if not reset: + try: + text_dec = Decimal(text) + text_dec_rounded = Decimal(fx.ccy_amount_str(text_dec, commas=False)) + reset = text_dec_rounded == def_fiat_rounded + except: + # garbage. not resetting, but not saving either + return False + if reset: d = self.fiat_value.get(ccy, {}) if d and txid in d: d.pop(txid) else: - return + # avoid saving empty dict + return True else: - try: - Decimal(text) - except: - return - if ccy not in self.fiat_value: - self.fiat_value[ccy] = {} - self.fiat_value[ccy][txid] = text + if ccy not in self.fiat_value: + self.fiat_value[ccy] = {} + self.fiat_value[ccy][txid] = text self.storage.put('fiat_value', self.fiat_value) + return reset def get_fiat_value(self, txid, ccy): fiat_value = self.fiat_value.get(ccy, {}).get(txid) @@ -423,21 +436,11 @@ class Abstract_Wallet(AddressSynchronizer): income += value # fiat computations if fx and fx.is_enabled() and fx.get_history_config(): - fiat_value = self.get_fiat_value(tx_hash, fx.ccy) - fiat_default = fiat_value is None - fiat_rate = self.price_at_timestamp(tx_hash, fx.timestamp_rate) - fiat_value = fiat_value if fiat_value is not None else value / Decimal(COIN) * fiat_rate - fiat_fee = tx_fee / Decimal(COIN) * fiat_rate if tx_fee is not None else None - item['fiat_value'] = Fiat(fiat_value, fx.ccy) - item['fiat_fee'] = Fiat(fiat_fee, fx.ccy) if fiat_fee else None - item['fiat_default'] = fiat_default + fiat_fields = self.get_tx_item_fiat(tx_hash, value, fx, tx_fee) + fiat_value = fiat_fields['fiat_value'].value + item.update(fiat_fields) if value < 0: - acquisition_price = - value / Decimal(COIN) * self.average_price(tx_hash, fx.timestamp_rate, fx.ccy) - liquidation_price = - fiat_value - item['acquisition_price'] = Fiat(acquisition_price, fx.ccy) - cg = liquidation_price - acquisition_price - item['capital_gain'] = Fiat(cg, fx.ccy) - capital_gains += cg + capital_gains += fiat_fields['capital_gain'].value fiat_expenditures += -fiat_value else: fiat_income += fiat_value @@ -478,6 +481,27 @@ class Abstract_Wallet(AddressSynchronizer): 'summary': summary } + def default_fiat_value(self, tx_hash, fx, value_sat): + return value_sat / Decimal(COIN) * self.price_at_timestamp(tx_hash, fx.timestamp_rate) + + def get_tx_item_fiat(self, tx_hash, value, fx, tx_fee): + item = {} + fiat_value = self.get_fiat_value(tx_hash, fx.ccy) + fiat_default = fiat_value is None + fiat_rate = self.price_at_timestamp(tx_hash, fx.timestamp_rate) + fiat_value = fiat_value if fiat_value is not None else self.default_fiat_value(tx_hash, fx, value) + fiat_fee = tx_fee / Decimal(COIN) * fiat_rate if tx_fee is not None else None + item['fiat_value'] = Fiat(fiat_value, fx.ccy) + item['fiat_fee'] = Fiat(fiat_fee, fx.ccy) if fiat_fee else None + item['fiat_default'] = fiat_default + if value < 0: + acquisition_price = - value / Decimal(COIN) * self.average_price(tx_hash, fx.timestamp_rate, fx.ccy) + liquidation_price = - fiat_value + item['acquisition_price'] = Fiat(acquisition_price, fx.ccy) + cg = liquidation_price - acquisition_price + item['capital_gain'] = Fiat(cg, fx.ccy) + return item + def get_label(self, tx_hash): label = self.labels.get(tx_hash, '') if label is '': @@ -1154,6 +1178,9 @@ class Abstract_Wallet(AddressSynchronizer): total_price += self.coin_price(ser.split(':')[0], price_func, ccy, v) return total_price / (input_value/Decimal(COIN)) + def clear_coin_price_cache(self): + self._coin_price_cache = {} + def coin_price(self, txid, price_func, ccy, txin_value): """ Acquisition price of a coin. @@ -1162,13 +1189,12 @@ class Abstract_Wallet(AddressSynchronizer): if txin_value is None: return Decimal('NaN') cache_key = "{}:{}:{}".format(str(txid), str(ccy), str(txin_value)) - result = self.coin_price_cache.get(cache_key, None) + result = self._coin_price_cache.get(cache_key, None) if result is not None: return result if self.txi.get(txid, {}) != {}: result = self.average_price(txid, price_func, ccy) * txin_value/Decimal(COIN) - if not result.is_nan(): - self.coin_price_cache[cache_key] = result + self._coin_price_cache[cache_key] = result return result else: fiat_value = self.get_fiat_value(txid, ccy) @@ -1353,8 +1379,8 @@ class Imported_Wallet(Simple_Wallet): def get_public_key(self, address): return self.addresses[address].get('pubkey') - def import_private_keys(self, keys: List[str], password: Optional[str]) -> Tuple[List[str], - List[Tuple[str, str]]]: + def import_private_keys(self, keys: List[str], password: Optional[str], + write_to_disk=True) -> Tuple[List[str], List[Tuple[str, str]]]: good_addr = [] # type: List[str] bad_keys = [] # type: List[Tuple[str, str]] for key in keys: @@ -1372,7 +1398,7 @@ class Imported_Wallet(Simple_Wallet): self.add_address(addr) self.save_keystore() self.save_addresses() - self.save_transactions(write=True) + self.save_transactions(write=write_to_disk) return good_addr, bad_keys def import_private_key(self, key: str, password: Optional[str]) -> str: