plugins/coldcard/coldcard.py: include derivation path for each part of multisig (for v3.2.1 of firmware)

and poetry has a gazillion dependencies...

Collecting dnspython==2.1.0
  Downloading dnspython-2.1.0.zip (389 kB)
     |████████████████████████████████| 389 kB 2.1 MB/s
  Installing build dependencies ... |
error
  ERROR: Command errored out with exit status 1:
   command: /opt/electrum/contrib/build-linux/appimage/../../../contrib/build-linux/appimage/build/appimage/electrum.AppDir/usr/bin/python3.7 /opt/electrum/contrib/build-linux/appimage/build/appimage/electrum.AppDir/usr/lib/python3.7/site-packages/pip install --ignore-installed --no-user --prefix /tmp/pip-build-env-5z1gx14i/overlay --no-warn-script-location --no-binary :all: --only-binary :none: -i https://pypi.org/simple -- 'poetry>=0.12'
       cwd: None
  Complete output (195 lines):
  Collecting poetry>=0.12
    Downloading poetry-1.1.4.tar.gz (132 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting pkginfo<2.0,>=1.4
    Downloading pkginfo-1.6.1.tar.gz (37 kB)
  Collecting tomlkit<1.0.0,>=0.7.0
    Downloading tomlkit-0.7.0.tar.gz (163 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting keyring<22.0.0,>=21.2.0; python_version >= "3.6" and python_version < "4.0"
    Downloading keyring-21.8.0.tar.gz (58 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting cleo<0.9.0,>=0.8.1
    Downloading cleo-0.8.1.tar.gz (19 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting pexpect<5.0.0,>=4.7.0
    Downloading pexpect-4.8.0.tar.gz (157 kB)
  Collecting poetry-core<2.0.0,>=1.0.0
    Using cached poetry-core-1.0.0.tar.gz (333 kB)
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting importlib-metadata<2.0.0,>=1.6.0; python_version < "3.8"
    Using cached importlib_metadata-1.7.0.tar.gz (29 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting crashtest<0.4.0,>=0.3.0; python_version >= "3.6" and python_version < "4.0"
    Downloading crashtest-0.3.1.tar.gz (4.3 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting shellingham<2.0,>=1.1
    Downloading shellingham-1.3.2.tar.gz (9.7 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting virtualenv<21.0.0,>=20.0.26
    Downloading virtualenv-20.2.2.tar.gz (9.1 MB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting requests-toolbelt<0.10.0,>=0.9.1
    Downloading requests-toolbelt-0.9.1.tar.gz (207 kB)
  Collecting html5lib<2.0,>=1.0
    Downloading html5lib-1.1.tar.gz (272 kB)
  Collecting cachecontrol[filecache]<0.13.0,>=0.12.4
    Downloading CacheControl-0.12.6.tar.gz (14 kB)
  Collecting clikit<0.7.0,>=0.6.2
    Downloading clikit-0.6.2.tar.gz (56 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting requests<3.0,>=2.18
    Downloading requests-2.25.1.tar.gz (102 kB)
  Collecting cachy<0.4.0,>=0.3.0
    Downloading cachy-0.3.0.tar.gz (15 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting packaging<21.0,>=20.4
    Downloading packaging-20.8.tar.gz (79 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting jeepney>=0.4.2; sys_platform == "linux"
    Downloading jeepney-0.6.0.tar.gz (49 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting SecretStorage>=3.2; sys_platform == "linux"
    Downloading SecretStorage-3.3.0.tar.gz (19 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting ptyprocess>=0.5
    Downloading ptyprocess-0.7.0.tar.gz (70 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting zipp>=0.5
    Using cached zipp-3.4.0.tar.gz (15 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Collecting appdirs<2,>=1.4.3
    Downloading appdirs-1.4.4.tar.gz (13 kB)
  Collecting distlib<1,>=0.3.1
    Downloading distlib-0.3.1.zip (578 kB)
  Collecting six<2,>=1.9.0
    Downloading six-1.15.0.tar.gz (33 kB)
  Collecting filelock<4,>=3.0.0
    Downloading filelock-3.0.12.tar.gz (8.5 kB)
  Collecting webencodings
    Downloading webencodings-0.5.1.tar.gz (9.7 kB)
  Collecting msgpack>=0.5.2
    Downloading msgpack-1.0.2.tar.gz (123 kB)
  Collecting lockfile>=0.9
    Downloading lockfile-0.12.2.tar.gz (20 kB)
      ERROR: Command errored out with exit status 1:

.gitignore

@@ -16,6 +16,7 @@ bin/
 .idea
 .mypy_cache
 .vscode
+electrum_data
 
 # icons
 electrum/gui/kivy/theming/light-0.png
@@ -31,9 +32,13 @@ electrum/gui/kivy/theming/light.atlas
 # build workspaces
 contrib/build-wine/tmp/
 contrib/build-wine/fresh_clone/
+contrib/build-linux/sdist/fresh_clone/
 contrib/build-linux/appimage/build/
 contrib/build-linux/appimage/.cache/
 contrib/android_debug.keystore
+contrib/android/android_debug.keystore
+contrib/secp256k1/
+contrib/zbar/
 
 # shared objects
 electrum/*.so

.travis.yml

@@ -26,11 +26,11 @@ jobs:
       language: python
       python: 3.7
       install:
-        - sudo add-apt-repository -y ppa:bitcoin/bitcoin
+        - sudo add-apt-repository -y ppa:luke-jr/bitcoincore
         - sudo apt-get -qq update
         - sudo apt-get install -yq bitcoind
         - sudo apt-get -y install libsecp256k1-0
-        - pip install -r contrib/requirements/requirements.txt
+        - pip install .[tests]
         - pip install electrumx
       before_script:
           - electrum/tests/regtest/start_bitcoind.sh
@@ -43,7 +43,7 @@ jobs:
       install: pip install flake8
       script: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
     - stage: binary builds
-      if: branch = master
+      if: (branch = master) OR (tag IS present)
       name: "Windows build"
       language: c
       python: false
@@ -56,7 +56,7 @@ jobs:
       script:
         - sudo docker run --name electrum-wine-builder-cont -v $PWD:/opt/wine64/drive_c/electrum --rm --workdir /opt/wine64/drive_c/electrum/contrib/build-wine electrum-wine-builder-img ./build.sh
       after_success: true
-    - if: branch = master
+    - if: (branch = master) OR (tag IS present)
       name: "Android build"
       language: python
       python: 3.7
@@ -65,18 +65,19 @@ jobs:
       install:
         - pip install requests && ./contrib/pull_locale
         - ./contrib/make_packages
-        - sudo docker build --no-cache -t electrum-android-builder-img electrum/gui/kivy/tools
+        - sudo docker build --no-cache -t electrum-android-builder-img contrib/android
       script:
         - sudo chown -R 1000:1000 .
         # Output something every minute or Travis kills the job
         - while sleep 60; do echo "=====[ $SECONDS seconds still running ]====="; done &
-        - sudo docker run -it -u 1000:1000 --rm --name electrum-android-builder-cont --env CI=true -v $PWD:/home/user/wspace/electrum --workdir /home/user/wspace/electrum electrum-android-builder-img ./contrib/make_apk
+        - sudo docker run -it -u 1000:1000 --rm --name electrum-android-builder-cont --env CI=true -v $PWD:/home/user/wspace/electrum --workdir /home/user/wspace/electrum electrum-android-builder-img ./contrib/android/make_apk
         # kill background sleep loop
         - kill %1
         - ls -la bin
         - if [ $(ls bin | grep -c Electrum-*) -eq 0 ]; then exit 1; fi
       after_success: true
-    - if: branch = master
+      # disabled for now as travis started to always time out:
+    - if: false AND ((branch = master) OR (tag IS present))
       name: "MacOS build"
       os: osx
       language: c
@@ -88,7 +89,7 @@ jobs:
       script: ./contrib/osx/make_osx
       after_script: ls -lah dist && md5 dist/*
       after_success: true
-    - if: branch = master
+    - if: (branch = master) OR (tag IS present)
       name: "AppImage build"
       language: c
       python: false
@@ -99,6 +100,26 @@ jobs:
       script:
         - sudo docker run --name electrum-appimage-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/appimage electrum-appimage-builder-img ./build.sh
       after_success: true
+    - if: (branch = master) OR (tag IS present)
+      name: "tarball build"
+      language: c
+      python: false
+      services:
+        - docker
+      before_install:
+        # hack: travis already cloned the repo, but we re-clone now, as we need to have umask set BEFORE cloning
+        - umask 0022
+        - mkdir fresh_clone && cd fresh_clone
+        - git clone https://github.com/$TRAVIS_REPO_SLUG.git && cd electrum
+        - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then git fetch origin pull/$TRAVIS_PULL_REQUEST/merge; fi
+        - git checkout $TRAVIS_COMMIT
+        - echo "Second git clone ready at $PWD"
+      install:
+        - sudo docker build --no-cache -t electrum-sdist-builder-img ./contrib/build-linux/sdist/
+      script:
+        - echo "Building sdist at $PWD"
+        - sudo docker run --name electrum-sdist-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/sdist electrum-sdist-builder-img ./build.sh
+      after_success: true
     - stage: release check
       install:
           - git fetch --all --tags

MANIFEST.in

@@ -3,11 +3,14 @@ include README.rst
 include electrum.desktop
 include *.py
 include run_electrum
-include contrib/requirements/requirements.txt
-include contrib/requirements/requirements-hw.txt
 recursive-include packages *.py
 recursive-include packages cacert.pem
 
+include contrib/requirements/requirements*.txt
+include contrib/deterministic-build/requirements*.txt
+include contrib/make_libsecp256k1.sh
+include contrib/build_tools_util.sh
+
 graft electrum
 prune electrum/tests
 graft contrib/udev

README.rst

@@ -26,31 +26,54 @@ Electrum - Lightweight Bitcoin client
 Getting started
 ===============
 
-Electrum itself is pure Python, and so are most of the required dependencies.
+(*If you've come here looking to simply run Electrum,* `you may download it here`_.)
 
-Non-python dependencies
------------------------
+.. _you may download it here: https://electrum.org/#download
+
+Electrum itself is pure Python, and so are most of the required dependencies,
+but not everything. The following sections describe how to run from source, but here
+is a TL;DR::
+
+    sudo apt-get install libsecp256k1-0
+    python3 -m pip install --user .[gui,crypto]
+
+
+Not pure-python dependencies
+----------------------------
 
 If you want to use the Qt interface, install the Qt dependencies::
 
     sudo apt-get install python3-pyqt5
 
-For elliptic curve operations, libsecp256k1 is a required dependency::
+For elliptic curve operations, `libsecp256k1`_ is a required dependency::
 
     sudo apt-get install libsecp256k1-0
 
 Alternatively, when running from a cloned repository, a script is provided to build
 libsecp256k1 yourself::
 
+    sudo apt-get install automake libtool
     ./contrib/make_libsecp256k1.sh
 
+Due to the need for fast symmetric ciphers, `cryptography`_ is required.
+Install from your package manager (or from pip)::
+
+    sudo apt-get install python3-cryptography
+
+
+If you would like hardware wallet support, see `this`_.
+
+.. _libsecp256k1: https://github.com/bitcoin-core/secp256k1
+.. _pycryptodomex: https://github.com/Legrandin/pycryptodome
+.. _cryptography: https://github.com/pyca/cryptography
+.. _this: https://github.com/spesmilo/electrum-docs/blob/master/hardware-linux.rst
 
 Running from tar.gz
 -------------------
 
 If you downloaded the official package (tar.gz), you can run
 Electrum from its root directory without installing it on your
-system; all the python dependencies are included in the 'packages'
+system; all the pure python dependencies are included in the 'packages'
 directory. To run Electrum from its root directory, just do::
 
     ./run_electrum
@@ -62,14 +85,12 @@ You can also install Electrum on your system, by running this command::
 
 This will download and install the Python dependencies used by
 Electrum instead of using the 'packages' directory.
-
-If you cloned the git repository, you need to compile extra files
-before you can run Electrum. Read the next section, "Development
-version".
+It will also place an executable named :code:`electrum` in :code:`~/.local/bin`,
+so make sure that is on your :code:`PATH` variable.
 
 
-Development version
--------------------
+Development version (git clone)
+-------------------------------
 
 Check out the code from GitHub::
 
@@ -79,7 +100,7 @@ Check out the code from GitHub::
 
 Run install (this should install dependencies)::
 
-    python3 -m pip install --user .
+    python3 -m pip install --user -e .
 
 
 Create translations (optional)::
@@ -87,6 +108,9 @@ Create translations (optional)::
     sudo apt-get install python-requests gettext
     ./contrib/pull_locale
 
+Finally, to start Electrum::
+
+    ./run_electrum
 
 
 
@@ -96,7 +120,7 @@ Creating Binaries
 Linux (tarball)
 ---------------
 
-See :code:`contrib/build-linux/README.md`.
+See :code:`contrib/build-linux/sdist/README.md`.
 
 
 Linux (AppImage)
@@ -120,4 +144,4 @@ See :code:`contrib/build-wine/README.md`.
 Android
 -------
 
-See :code:`electrum/gui/kivy/Readme.md`.
+See :code:`contrib/android/Readme.md`.

RELEASE-NOTES

@@ -1,8 +1,167 @@
-# Release 4.0 - (Not released yet; release notes are incomplete)
+# Release 4.0.9 - (Dec 18, 2020)
+ * fixes a regression introduced in 4.0.8, that prevents from
+   paying BIP70 invoices (#6859)
+ * reflect frozen channels and disconnected peers in the displayed
+   'can send/can receive' amounts.
+
+# Release 4.0.8 - (Dec 17, 2020)
+ * fix decoding BIP21 URIs with uppercase schema (d40bedb2)
+ * psbt: put full derivation paths into PSBT by default (c8155129)
+ * invoices: allow address-reuse (#6609, #6852)
+ * A few other minor bugfixes.
+
+# Release 4.0.7 - (Dec 9, 2020)
+ * kivy: fix open channel with 'max' amount
+ * kivy: fix regression introduced in last release (a9fc440)
+ * other minor GUI fixes
+ * Dependencies: as part of adapting to new dnspython (#6828),
+   - python-ecdsa is no longer needed at all,
+   - cryptography is now required (min 2.6), the user can no
+     longer choose between cryptography and pycryptodomex
+
+# Release 4.0.6 - (Dec 4, 2020)
+ * Fix 'Max' button issue for submarine swaps button (#6770)
+ * Fix 'Max' button in kivy (#6169)
+ * Various fixes for Kivy/Android install wizard
+ * More robust account keypath for BitBox02 (#6766)
+
+# Release 4.0.5 - (Nov 18, 2020)
+ * Fix .dmg binary hanging on recently released macOS 11 Big Sur (#6461)
+ * Lightning:
+   - bugfix: during LN channel opening, if the client crashed at the
+     wrong moment, the channel might not get fully persisted to disk,
+     and would need manual console-tinkering to recover (#6656)
+   - Lightning is enabled by default. Electrum will not connect to
+     the Lightning Network until the user opens a channel. (#6639)
+   - smarter node recommendation (to open channels with) (#6705)
+ * user interface: some minor changes that aim to improve usability
+ * Ledger:
+   - fix enumerating devices with new bitcoin app (1.5.1) (b78cbcff)
+   - fix compat with HW.1 (200f547a)
+ * A few other minor bugfixes.
+
+# Release 4.0.4 - (Oct 15, 2020)
+ * PSBT: fix regression in 4.0.3 where UTXO data was not included in
+   QR codes (#6600)
+ * new feature: "Cancel tx" (#6641). The Qt/kivy GUI allows cancelling
+   an unconfirmed RBF tx by double-spending its inputs to self.
+ * Windows binary:
+   - fix some issues with QR scanning by building zbar ourselves (#6593)
+   - when using setup exe, also install a debug binary (#6603)
+ * Ledger: fix "The derivation path is unusual" warnings (#6512)
+   (needs Bitcoin app 1.4.8+ installed on device)
+ * A few other minor bugfixes and usability improvements.
+
+# Release 4.0.3 - (Sep 11, 2020)
+ * PSBT: restore compatibility with Bitcoin Core following CVE-2020-14199:
+   we now allow a PSBT input to have both UTXO and WITNESS_UTXO (#6429).
+   (PSBTs created since 4.0.1 already contained UTXO for segwit inputs)
+ * Hardware wallets:
+   - bitbox02: better multisig UX: implement get_soft_device_id (#6386)
+   - coldcard: fix "show address" for multisig (#6517)
+   - all: run all device communication on a dedicated thread (#6561).
+     This should resolve some threading issues.
+ * new feature: "Automated BIP39 recovery" (#6219, #6155)
+   When restoring from a BIP39 seed, add option to scan many known
+   derivation paths for history, and show them to user to choose from.
+ * show derivation path of keystores in Qt GUI Wallet>Information (#4700)
+ * fix "signtransaction" RPC command (#6502)
+ * Dependencies: pyaes is no longer needed (#6563)
+ * The tar.gz source dist now bundles make_libsecp256k1.sh, to help
+   users getting libsecp256k1 (#6323).
+ * A few other minor bugfixes and usability improvements.
+
+# Release 4.0.2 - (July 8, 2020)
+ - rm old corrupted non-bip70 invoices (#6345)
+ - other minor fixes
+
+# Release 4.0.1 - (July 3, 2020)
+ * Lightning Network support (experimental)
+   - Our implementation of Lightning relies on Electrum servers to
+     query channel states. Since servers can lie about the state of a
+     channel, users should either use a server that they trust, or
+     setup a private watchtower (see below). A watchtower is also
+     recommended for lightning wallets that remain offline for
+     extended periods of time (the default CSV 'to_self_delay' is 1
+     week). Please note that Electrum Personal Server (EPS) cannot be
+     used with lightning wallets, because channels funding addresses
+     are arbitrary.
+   - Lightning funds cannot be restored from seed. Instead, users need
+     to create static backups of their channels. Static backups cannot
+     be used to perform lightning transactions, they can only be used
+     to trigger a remote-force-close of a channel.
+   - Lightning-enabled wallet files must not be copied. Instead, a
+     backup of the wallet can be created from the Qt menu, and it will
+     contain static backups of all its channels. Backups can also be
+     exported for each channel (e.g. via QR code), and imported in
+     another wallet. Since backups are encrypted with a key derived
+     from the wallet's xpub, they can only be imported into another
+     instance of the same wallet, or a watch-only version of it. The
+     force-close is not triggered automatically when the backup is
+     imported; imported backups can live inside a wallet file.
+   - Lightning can be enabled in the GUI (Wallet>Information) or from
+     the CLI (init_lightning). Lightning is currently restricted to HD
+     p2wpkh wallets (including watch-only and hardware wallets). The
+     Qt GUI, CLI/RPC, and the kivy GUI (Android) all have LN support,
+     with feature-richness in that order.
+   - LN protocol details: dataloss_protect and static_remotekey are
+     required; varonion and payment_secret are implemented, MPP not yet.
+     Channels are not announced ('private'), forwarding is disabled.
+     We do not serve gossip queries, only consume them.
+   - Submarine swaps: the GUI integrates a service that offers
+     atomically exchanging on-chain and lightning bitcoins for a fee.
+     Electrum Technologies runs a central server for this, powered by
+     the Boltz backend.
+   - Watchtowers: Electrum can run a local watchtower (GUI setting),
+     or it can connect to a remote watchtower. A watchtower contains
+     pre-signed transactions and does not need your private keys. A
+     local watchtower will watch your channels whenever an Electrum
+     instance is running, without needing access to your wallet file.
+     An Electrum daemon can be configured to be used as a remote
+     watchtower by setting 'watchtower_address', 'watchtower_user' and
+     'watchtower_password'.
+ * Partially Signed Bitcoin Transactions (PSBT, BIP-174) are supported
+   (#5721). The previous Electrum partial transaction format is no
+   longer supported, i.e. this is an incompatible change. Users should
+   make sure that all instances of Electrum they use to co-sign or
+   offline sign, are updated together.
+ * Hardware wallets: several fixes in general; notable changes:
+   - The BitBox02 is now supported (#5993)
+   - Multisig support for Coldcard (#5440)
+   - Compatibility with latest Trezor fw (#6064, #6198, #5692)
+ * Dependencies (see README for install instructions):
+   - libsecp256k1 is now required (previously optional). python-ecdsa
+     remains a dependency but it is now only used for DNSSEC.
+   - Added: either one of pycryptodomex or cryptography is now required,
+     mainly due to LN (previously pycryptodomex was optional, for fast AES)
+   - Removed: jsonrpclib-pelix, the JSON-RPC library used for CLI/daemon
+ * Qt GUI: several changes, notably:
+   - Separation between output selection and transaction finalization.
+   - Coin selection moved to the Coins tab, and it affects all txns,
+     e.g. RBF fee-bumping, LN channel opens, submarine swaps.
+   - Editable tx preview dialog that allows e.g. changing the locktime,
+     toggling RBF, and manual coinjoins.
+ * HTTP PayServer: The configuration of a bitcoin-accepting website
+   using Electrum has been simplified and requires fewer steps (see
+   documentation). The Payserver supports BIP70 and Lightning payments.
+ * Android:
+   - We now build two APKs, one for ARMv7 and one for ARMv8
+   - The kivy GUI now supports importing BIP39 seeds
+   - Each wallet on kivy now can have a separate generic password,
+     using which the wallet files are encrypted. An optional PIN,
+     shared among all wallets, can be added to get prompted for spends.
+ * The API of several CLI/RPC commands have changed, and several new
+   commands have been introduced (mainly for LN).
+ * Distributables:
+   - The .tar.gz source dist is now built reproducibly.
+     Relatedly, we no longer distribute a .zip sdist.
+   - The MacOS binary now conforms to macOS 10.15; it is notarized
+     by Apple. This required bumping the min macOS version to 10.13.
+     Startup times should now be faster on 10.15. (#6128, #6225)
+ * Transactions:
+   - we now grind low R for ECDSA signatures to match bitcoind (#5820)
+ * Lots and lots of other minor bugfixes and improvements.
 
- * Lightning Network
- * Qt GUI: Separation between output selection and transaction finalization.
- * Http PayServer can be configured from GUI
 
 # Release 3.3.8 - (July 11, 2019)
 

contrib/android/Dockerfile

@@ -1,6 +1,8 @@
 # based on https://github.com/kivy/python-for-android/blob/master/Dockerfile
 
-FROM ubuntu:18.04
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
 
 ENV ANDROID_HOME="/opt/android"
 
@@ -18,7 +20,7 @@ RUN apt -y update -qq \
 
 
 ENV ANDROID_NDK_HOME="${ANDROID_HOME}/android-ndk"
-ENV ANDROID_NDK_VERSION="19b"
+ENV ANDROID_NDK_VERSION="19c"
 ENV ANDROID_NDK_HOME_V="${ANDROID_NDK_HOME}-r${ANDROID_NDK_VERSION}"
 
 # get the latest version from https://developer.android.com/ndk/downloads/index.html
@@ -38,10 +40,11 @@ RUN curl --location --progress-bar \
 ENV ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk"
 
 # get the latest version from https://developer.android.com/studio/index.html
-ENV ANDROID_SDK_TOOLS_VERSION="4333796"
-ENV ANDROID_SDK_BUILD_TOOLS_VERSION="28.0.3"
-ENV ANDROID_SDK_TOOLS_ARCHIVE="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip"
+ENV ANDROID_SDK_TOOLS_VERSION="6514223"
+ENV ANDROID_SDK_BUILD_TOOLS_VERSION="29.0.3"
+ENV ANDROID_SDK_TOOLS_ARCHIVE="commandlinetools-linux-${ANDROID_SDK_TOOLS_VERSION}_latest.zip"
 ENV ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}"
+ENV ANDROID_SDK_MANAGER="${ANDROID_SDK_HOME}/tools/bin/sdkmanager --sdk_root=${ANDROID_SDK_HOME}"
 
 # download and install Android SDK
 RUN curl --location --progress-bar \
@@ -58,15 +61,15 @@ RUN mkdir --parents "${ANDROID_SDK_HOME}/.android/" \
 
 # accept Android licenses (JDK necessary!)
 RUN apt -y update -qq \
-    && apt -y install -qq --no-install-recommends openjdk-8-jdk \
+    && apt -y install -qq --no-install-recommends openjdk-13-jdk \
     && apt -y autoremove
-RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" --licenses > /dev/null
+RUN yes | ${ANDROID_SDK_MANAGER} --licenses > /dev/null
 
 # download platforms, API, build tools
-RUN "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-24" > /dev/null && \
-    "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-28" > /dev/null && \
-    "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" > /dev/null && \
-    "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "extras;android;m2repository" > /dev/null && \
+RUN ${ANDROID_SDK_MANAGER} "platforms;android-24" > /dev/null && \
+    ${ANDROID_SDK_MANAGER} "platforms;android-29" > /dev/null && \
+    ${ANDROID_SDK_MANAGER} "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" > /dev/null && \
+    ${ANDROID_SDK_MANAGER} "extras;android;m2repository" > /dev/null && \
     chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager"
 
 # download ANT
@@ -89,37 +92,38 @@ ENV HOME_DIR="/home/${USER}"
 ENV WORK_DIR="${HOME_DIR}/wspace" \
     PATH="${HOME_DIR}/.local/bin:${PATH}"
 
-# install system dependencies
+# install system/build dependencies
+# https://github.com/kivy/buildozer/blob/master/docs/source/installation.rst#android-on-ubuntu-2004-64bit
 RUN apt -y update -qq \
     && apt -y install -qq --no-install-recommends \
-        python3 virtualenv python3-pip python3-setuptools git wget lbzip2 patch sudo \
+        python3 \
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        wget \
+        lbzip2 \
+        patch \
+        sudo \
         software-properties-common \
-    && apt -y autoremove
-
-# install kivy
-RUN add-apt-repository ppa:kivy-team/kivy \
-    && apt -y update -qq  \
-    && apt -y install -qq --no-install-recommends python3-kivy \
-    && apt -y autoremove \
-    && apt -y clean
-RUN python3 -m pip install image
-
-# build dependencies
-# https://buildozer.readthedocs.io/en/latest/installation.html#android-on-ubuntu-16-04-64bit
-RUN dpkg --add-architecture i386 \
-    && apt -y update -qq \
-    && apt -y install -qq --no-install-recommends \
-        build-essential ccache git python3 python3-dev \
-        libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \
-        libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 \
-        zip zlib1g-dev zlib1g:i386 \
-    && apt -y autoremove \
-    && apt -y clean
-
-# specific recipes dependencies (e.g. libffi requires autoreconf binary)
-RUN apt -y update -qq \
-    && apt -y install -qq --no-install-recommends \
-        libffi-dev autoconf automake cmake gettext libltdl-dev libtool pkg-config \
+        git \
+        zip \
+        unzip \
+        build-essential \
+        ccache \
+        openjdk-13-jdk \
+        autoconf \
+        libtool \
+        pkg-config \
+        zlib1g-dev \
+        libncurses5-dev \
+        libncursesw5-dev \
+        libtinfo5 \
+        cmake \
+        libffi-dev \
+        libssl-dev \
+        automake \
+        gettext \
+        libltdl-dev \
     && apt -y autoremove \
     && apt -y clean
 
@@ -140,9 +144,11 @@ RUN chown ${USER} /opt
 USER ${USER}
 
 
-RUN python3 -m pip install --upgrade cython==0.28.6
-RUN python3 -m pip install --upgrade pip
-RUN python3 -m pip install --user wheel
+RUN python3 -m pip install --user --upgrade pip
+RUN python3 -m pip install --user --upgrade wheel
+RUN python3 -m pip install --user --upgrade cython==0.29.19
+RUN python3 -m pip install --user --pre kivy
+RUN python3 -m pip install --user image
 
 # prepare git
 RUN git config --global user.name "John Doe" \
@@ -154,7 +160,8 @@ RUN cd /opt \
     && cd buildozer \
     && git remote add sombernight https://github.com/SomberNight/buildozer \
     && git fetch --all \
-    && git checkout 7578fea609d4445b3fed1f441813ab4c86ef0086 \
+    # commit: kivy/buildozer "1.2.0" tag
+    && git checkout "94cfcb8d591c11d6ad0e11f129b08c1e27a161c5^{commit}" \
     && python3 -m pip install --user -e .
 
 # install python-for-android
@@ -163,7 +170,8 @@ RUN cd /opt \
     && cd python-for-android \
     && git remote add sombernight https://github.com/SomberNight/python-for-android \
     && git fetch --all \
-    && git checkout 9162ec6b4af464672960f6f9bb7c481af2d01802 \
+    # commit: from branch sombernight/electrum_20200703
+    && git checkout "0dd2ce87a8f380d20505ca5dc1e2d2357b4a08fc^{commit}" \
     && python3 -m pip install --user -e .
 
 # build env vars

contrib/android/Makefile

@@ -0,0 +1,33 @@
+PYTHON = python3
+
+# needs kivy installed or in PYTHONPATH
+
+.PHONY: theming apk clean
+
+theming:
+	#bash -c 'for i in network lightning; do convert -background none theming/light/$i.{svg,png}; done'
+	$(PYTHON) -m kivy.atlas ../../electrum/gui/kivy/theming/light 1024 ../../electrum/gui/kivy/theming/light/*.png
+prepare:
+	# running pre build setup
+	@cp buildozer.spec ../../buildozer.spec
+	# copy electrum to main.py
+	@cp ../../run_electrum ../../main.py
+	@-if [ ! -d "../../.buildozer" ];then \
+		cd ../..; buildozer android debug;\
+		cp -f blacklist.txt .buildozer/android/platform/python-for-android/src/blacklist.txt;\
+		rm -rf ./.buildozer/android/platform/python-for-android/dist;\
+	fi
+apk:
+	@make prepare
+	@-cd ../..; buildozer android debug deploy run
+	@make clean
+release:
+	@make prepare
+	@-cd ../..; buildozer android release
+	@make clean
+clean:
+	# Cleaning up
+	# rename main.py to electrum
+	@-rm ../../main.py
+	# remove buildozer.spec
+	@-rm ../../buildozer.spec

contrib/android/Readme.md

@@ -24,7 +24,7 @@ folder.
 2. Build image
 
     ```
-    $ sudo docker build -t electrum-android-builder-img electrum/gui/kivy/tools
+    $ sudo docker build -t electrum-android-builder-img contrib/android
     ```
 
 3. Build locale files
@@ -50,7 +50,7 @@ folder.
         -v ~/.keystore:/home/user/.keystore \
         --workdir /home/user/wspace/electrum \
         electrum-android-builder-img \
-        ./contrib/make_apk
+        ./contrib/android/make_apk
     ```
     This mounts the project dir inside the container,
     and so the modifications will affect it, e.g. `.buildozer` folder
@@ -63,7 +63,7 @@ folder.
 ## FAQ
 
 ### I changed something but I don't see any differences on the phone. What did I do wrong?
-You probably need to clear the cache: `rm -rf .buildozer/android/platform/build/{build,dists}`
+You probably need to clear the cache: `rm -rf .buildozer/android/platform/build-*/{build,dists}`
 
 
 ### How do I deploy on connected phone for quick testing?
@@ -102,7 +102,7 @@ adb logcat | grep -F "`adb shell ps | grep org.electrum.electrum | cut -c14-19`"
 ### Kivy can be run directly on Linux Desktop. How?
 Install Kivy.
 
-Build atlas: `(cd electrum/gui/kivy/; make theming)`
+Build atlas: `(cd contrib/android/; make theming)`
 
 Run electrum with the `-g` switch: `electrum -g kivy`
 
@@ -115,3 +115,13 @@ keystore, back it up safely, and run `./contrib/make_apk release`.
 
 See e.g. [kivy wiki](https://github.com/kivy/kivy/wiki/Creating-a-Release-APK)
 and [android dev docs](https://developer.android.com/studio/build/building-cmdline#sign_cmdline).
+
+### Access datadir on Android from desktop (e.g. to copy wallet file)
+Note that this only works for debug builds! Otherwise the security model
+of Android does not let you access the internal storage of an app without root.
+(See [this](https://stackoverflow.com/q/9017073))
+```
+$ adb shell
+$ run-as org.electrum.electrum ls /data/data/org.electrum.electrum/files/data
+$ run-as org.electrum.electrum cp /data/data/org.electrum.electrum/files/data/wallets/my_wallet /sdcard/some_path/my_wallet
+```

contrib/android/buildozer.spec

@@ -13,7 +13,7 @@ package.domain = org.electrum
 source.dir = .
 
 # (list) Source files to include (let empty to include all the files)
-source.include_exts = py,png,jpg,kv,atlas,ttf,txt,gif,pem,mo,vs,fs,json
+source.include_exts = py,png,jpg,kv,atlas,ttf,txt,gif,pem,mo,vs,fs,json,csv
 
 # (list) Source files to exclude (let empty to not exclude anything)
 source.exclude_exts = spec
@@ -22,10 +22,9 @@ source.exclude_exts = spec
 source.exclude_dirs = bin, build, dist, contrib,
     electrum/tests,
     electrum/gui/qt,
-    electrum/gui/kivy/tools,
     electrum/gui/kivy/theming/light,
-    # exclude pycryptodomex built by make_packages; android needs custom version
-    packages/cryptodome
+    packages/qdarkstyle,
+    packages/qtpy
 # (list) List of exclusions using pattern matching
 source.exclude_patterns = Makefile,setup*
 
@@ -38,7 +37,9 @@ version.filename = %(source.dir)s/electrum/version.py
 
 # (list) Application requirements
 requirements =
-    python3,
+    # note: re python3.8, see #6147
+    hostpython3==3.7.9,
+    python3==3.7.9,
     android,
     openssl,
     plyer,
@@ -46,7 +47,7 @@ requirements =
     kivy==39c17457bae91baf8fe710dc989791e45879f136,
     libffi,
     libsecp256k1,
-    pycryptodomex==bfc1cca093a7344c9ed2b7c34bc560db6dca662a
+    cryptography
 
 # (str) Presplash of the application
 #presplash.filename = %(source.dir)s/gui/kivy/theming/splash.png
@@ -69,14 +70,15 @@ fullscreen = False
 # (list) Permissions
 android.permissions = INTERNET, CAMERA, WRITE_EXTERNAL_STORAGE
 
-# (int) Android API to use
-android.api = 28
+# (int) Android API to use  (targetSdkVersion AND compileSdkVersion)
+# note: when changing, Dockerfile also needs to be changed to install corresponding build tools
+android.api = 29
 
 # (int) Minimum API required. You will need to set the android.ndk_api to be as low as this value.
 android.minapi = 21
 
 # (str) Android NDK version to use
-android.ndk = 19b
+android.ndk = 19c
 
 # (int) Android NDK API to use (optional). This is the minimum API your app will support.
 android.ndk_api = 21
@@ -123,7 +125,7 @@ android.add_activities = org.electrum.qr.SimpleScannerActivity
 #android.ouya.icon.filename = %(source.dir)s/data/ouya_icon.png
 
 # (str) XML file to include as an intent filters in <activity> tag
-android.manifest.intent_filters = electrum/gui/kivy/tools/bitcoin_intent.xml
+android.manifest.intent_filters = contrib/android/bitcoin_intent.xml
 
 # (str) launchMode to set for the main activity
 android.manifest.launch_mode = singleTask

contrib/android/make_apk

@@ -2,8 +2,8 @@
 
 set -e
 
-CONTRIB="$(dirname "$(readlink -e "$0")")"
-ROOT_FOLDER="$CONTRIB"/..
+CONTRIB_ANDROID="$(dirname "$(readlink -e "$0")")"
+ROOT_FOLDER="$CONTRIB_ANDROID"/../..
 PACKAGES="$ROOT_FOLDER"/packages/
 LOCALE="$ROOT_FOLDER"/electrum/locale/
 
@@ -17,7 +17,7 @@ if [ ! -d "$PACKAGES" ]; then
   exit 1
 fi
 
-pushd ./electrum/gui/kivy/
+pushd ./contrib/android
 
 make theming
 
@@ -34,13 +34,13 @@ if [[ -n "$1"  && "$1" == "release" ]] ; then
     export APP_ANDROID_ARCH=arm64-v8a
     make release
 else
-    export P4A_DEBUG_KEYSTORE="$CONTRIB"/android_debug.keystore
+    export P4A_DEBUG_KEYSTORE="$CONTRIB_ANDROID"/android_debug.keystore
     export P4A_DEBUG_KEYSTORE_PASSWD=unsafepassword
     export P4A_DEBUG_KEYALIAS_PASSWD=unsafepassword
     export P4A_DEBUG_KEYALIAS=electrum
     # create keystore if needed
     if [ ! -f "$P4A_DEBUG_KEYSTORE" ]; then
-        keytool -genkey -v -keystore "$CONTRIB"/android_debug.keystore \
+        keytool -genkey -v -keystore "$CONTRIB_ANDROID"/android_debug.keystore \
             -alias "$P4A_DEBUG_KEYALIAS" -keyalg RSA -keysize 2048 -validity 10000 \
             -dname "CN=mqttserver.ibm.com, OU=ID, O=IBM, L=Hursley, S=Hants, C=GB" \
             -storepass "$P4A_DEBUG_KEYSTORE_PASSWD" \

contrib/build-linux/README.md

@@ -1,16 +0,0 @@
-Source tarballs
-===============
-
-✗ _This script does not produce reproducible output (yet!)._
-
-1. Prepare python dependencies used by Electrum.
-
-    ```
-    contrib/make_packages
-    ```
-
-2. Create source tarball.
-
-    ```
-    contrib/make_tgz
-    ```

contrib/build-linux/appimage/Dockerfile

@@ -1,29 +1,30 @@
-FROM ubuntu:16.04@sha256:97b54e5692c27072234ff958a7442dde4266af21e7b688e7fca5dc5acc8ed7d9
+FROM ubuntu:16.04@sha256:a4fc0c40360ff2224db3a483e5d80e9164fe3fdce2a8439d2686270643974632
 
 ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
 
 RUN apt-get update -q && \
     apt-get install -qy \
-        git=1:2.7.4-0ubuntu1.7 \
+        git=1:2.7.4-0ubuntu1.9 \
         wget=1.17.1-1ubuntu1.5 \
         make=4.1-6 \
         autotools-dev=20150820.1 \
         autoconf=2.69-9 \
         libtool=2.4.6-0.1 \
         xz-utils=5.1.1alpha+20120614-2ubuntu2 \
-        libssl-dev=1.0.2g-1ubuntu4.15 \
-        libssl1.0.0=1.0.2g-1ubuntu4.15 \
-        openssl=1.0.2g-1ubuntu4.15 \
+        libssl-dev=1.0.2g-1ubuntu4.18 \
+        libssl1.0.0=1.0.2g-1ubuntu4.18 \
+        openssl=1.0.2g-1ubuntu4.18 \
         zlib1g-dev=1:1.2.8.dfsg-2ubuntu4.3 \
         libffi-dev=3.2.1-4 \
         libncurses5-dev=6.0+20160213-1ubuntu1 \
-        libsqlite3-dev=3.11.0-1ubuntu1.3 \
+        libsqlite3-dev=3.11.0-1ubuntu1.5 \
         libusb-1.0-0-dev=2:1.0.20-1 \
-        libudev-dev=229-4ubuntu21.27 \
+        libudev-dev=229-4ubuntu21.29 \
         gettext=0.19.7-2ubuntu3.1 \
         libzbar0=0.10+doc-10ubuntu1  \
-        libdbus-1-3=1.10.6-1ubuntu3.4 \
+        libdbus-1-3=1.10.6-1ubuntu3.6 \
         libxkbcommon-x11-0=0.5.0-1ubuntu2.1 \
+        libc6-dev=2.23-0ubuntu11.2 \
         && \
     rm -rf /var/lib/apt/lists/* && \
     apt-get autoremove -y && \

contrib/build-linux/appimage/README.md

@@ -61,6 +61,11 @@ diff sha256sum1 sha256sum2 > d
 cat d
 ```
 
+For file metadata, e.g. timestamps:
+```
+rsync -n -a -i --delete squashfs-root1/ squashfs-root2/
+```
+
 Useful binary comparison tools:
 - vbindiff
 - diffoscope

contrib/build-linux/appimage/build.sh

@@ -13,7 +13,7 @@ CACHEDIR="$CONTRIB_APPIMAGE/.cache/appimage"
 export GCC_STRIP_BINARIES="1"
 
 # pinned versions
-PYTHON_VERSION=3.7.6
+PYTHON_VERSION=3.7.9
 PKG2APPIMAGE_COMMIT="eb8f3acdd9f11ab19b78f5cb15daa772367daf15"
 SQUASHFSKIT_COMMIT="ae0d656efa2d0df2fcac795b6823b44462f19386"
 
@@ -38,7 +38,7 @@ download_if_not_exist "$CACHEDIR/appimagetool" "https://github.com/AppImage/AppI
 verify_hash "$CACHEDIR/appimagetool" "d918b4df547b388ef253f3c9e7f6529ca81a885395c31f619d9aaf7030499a13"
 
 download_if_not_exist "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tar.xz"
-verify_hash "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "55a2cce72049f0794e9a11a84862e9039af9183603b78bc60d89539f82cf533f"
+verify_hash "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "91923007b05005b5f9bd46f3b9172248aea5abc1543e8a636d59e629c3331b01"
 
 
 
@@ -71,7 +71,7 @@ info "Building squashfskit"
 git clone "https://github.com/squashfskit/squashfskit.git" "$BUILDDIR/squashfskit"
 (
     cd "$BUILDDIR/squashfskit"
-    git checkout "$SQUASHFSKIT_COMMIT"
+    git checkout "${SQUASHFSKIT_COMMIT}^{commit}"
     make -C squashfs-tools mksquashfs || fail "Could not build squashfskit"
 )
 MKSQUASHFS="$BUILDDIR/squashfskit/squashfs-tools/mksquashfs"
@@ -94,6 +94,8 @@ python='appdir_python'
 info "installing pip."
 "$python" -m ensurepip
 
+break_legacy_easy_install
+
 
 info "preparing electrum-locale."
 (
@@ -113,12 +115,25 @@ info "preparing electrum-locale."
 )
 
 
-info "installing electrum and its dependencies."
+info "Installing build dependencies."
 mkdir -p "$CACHEDIR/pip_cache"
-"$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements.txt"
-"$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-binaries.txt"
-"$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-hw.txt"
-"$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" "$PROJECT_ROOT"
+"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \
+    --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-build-appimage.txt"
+
+info "installing electrum and its dependencies."
+# note: we prefer compiling C extensions ourselves, instead of using binary wheels,
+#       hence "--no-binary :all:" flags. However, we specifically allow
+#       - PyQt5, as it's harder to build from source
+#       - cryptography, as building it would need openssl 1.1, not available on ubuntu 16.04
+"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \
+    --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements.txt"
+"$python" -m pip install --no-dependencies --no-binary :all: --only-binary pyqt5,cryptography --no-warn-script-location \
+    --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-binaries.txt"
+"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \
+    --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-hw.txt"
+
+"$python" -m pip install --no-dependencies --no-warn-script-location \
+    --cache-dir "$CACHEDIR/pip_cache" "$PROJECT_ROOT"
 
 # was only needed during build time, not runtime
 "$python" -m pip uninstall -y Cython
@@ -206,13 +221,11 @@ rm -rf "$PYDIR"/site-packages/PyQt5/Qt.so
 
 # these are deleted as they were not deterministic; and are not needed anyway
 find "$APPDIR" -path '*/__pycache__*' -delete
-# note that jsonschema-*.dist-info is needed by that package as it uses 'pkg_resources.get_distribution'
-# also, see https://gitlab.com/python-devs/importlib_metadata/issues/71
-for f in "$PYDIR"/site-packages/jsonschema-*.dist-info; do mv "$f" "$(echo "$f" | sed s/\.dist-info/\.dist-info2/)"; done
+# note that *.dist-info is needed by certain packages.
+# e.g. see https://gitlab.com/python-devs/importlib_metadata/issues/71
 for f in "$PYDIR"/site-packages/importlib_metadata-*.dist-info; do mv "$f" "$(echo "$f" | sed s/\.dist-info/\.dist-info2/)"; done
 rm -rf "$PYDIR"/site-packages/*.dist-info/
 rm -rf "$PYDIR"/site-packages/*.egg-info/
-for f in "$PYDIR"/site-packages/jsonschema-*.dist-info2; do mv "$f" "$(echo "$f" | sed s/\.dist-info2/\.dist-info/)"; done
 for f in "$PYDIR"/site-packages/importlib_metadata-*.dist-info2; do mv "$f" "$(echo "$f" | sed s/\.dist-info2/\.dist-info/)"; done
 
 

contrib/build-linux/sdist/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:20.04@sha256:5747316366b8cc9e3021cd7286f42b2d6d81e3d743e2ab571f55bcd5df788cc8
+
+ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -q && \
+    apt-get install -qy \
+        git \
+        gettext \
+        python3 \
+        python3-pip \
+        python3-setuptools \
+        faketime \
+        && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get autoremove -y && \
+    apt-get clean

contrib/build-linux/sdist/README.md

@@ -0,0 +1,52 @@
+Source tarballs
+===============
+
+✓ _This file should be reproducible, meaning you should be able to generate
+   distributables that match the official releases._
+
+This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another
+similar system. The docker commands should be executed in the project's root
+folder.
+
+1. Install Docker
+
+    ```
+    $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    $ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    $ sudo apt-get update
+    $ sudo apt-get install -y docker-ce
+    ```
+
+2. Build image
+
+    ```
+    $ sudo docker build -t electrum-sdist-builder-img contrib/build-linux/sdist
+    ```
+
+3. Build source tarballs
+
+    It's recommended to build from a fresh clone
+    (but you can skip this if reproducibility is not necessary).
+
+    ```
+    $ FRESH_CLONE=contrib/build-linux/sdist/fresh_clone && \
+        sudo rm -rf $FRESH_CLONE && \
+        umask 0022 && \
+        mkdir -p $FRESH_CLONE && \
+        cd $FRESH_CLONE  && \
+        git clone https://github.com/spesmilo/electrum.git && \
+        cd electrum
+    ```
+
+    And then build from this directory:
+    ```
+    $ git checkout $REV
+    $ sudo docker run -it \
+        --name electrum-sdist-builder-cont \
+        -v $PWD:/opt/electrum \
+        --rm \
+        --workdir /opt/electrum/contrib/build-linux/sdist \
+        electrum-sdist-builder-img \
+        ./build.sh
+    ```
+4. The generated distributables are in `./dist`.

contrib/build-linux/sdist/build.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.."
+CONTRIB="$PROJECT_ROOT/contrib"
+CONTRIB_SDIST="$CONTRIB/build-linux/sdist"
+DISTDIR="$PROJECT_ROOT/dist"
+
+. "$CONTRIB"/build_tools_util.sh
+
+# note that at least py3.7 is needed, to have https://bugs.python.org/issue30693
+python3 --version || fail "python interpreter not found"
+
+break_legacy_easy_install
+
+# upgrade to modern pip so that it knows the flags we need.
+# we will then install a pinned version of pip as part of requirements-build-sdist
+python3 -m pip install --upgrade pip
+
+info "Installing pinned requirements."
+python3 -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-build-sdist.txt
+
+
+"$CONTRIB"/make_packages || fail "make_packages failed"
+
+"$CONTRIB_SDIST"/make_tgz || fail "make_tgz failed"
+
+
+info "done."
+ls -la "$DISTDIR"
+sha256sum "$DISTDIR"/*

contrib/build-linux/sdist/make_tgz

@@ -2,7 +2,8 @@
 
 set -e
 
-CONTRIB="$(dirname "$(readlink -e "$0")")"
+CONTRIB_SDIST="$(dirname "$(readlink -e "$0")")"
+CONTRIB="$CONTRIB_SDIST"/../..
 ROOT_FOLDER="$CONTRIB"/..
 PACKAGES="$ROOT_FOLDER"/packages/
 LOCALE="$ROOT_FOLDER"/electrum/locale/
@@ -39,5 +40,8 @@ git submodule update --init
     # we could build the kivy atlas potentially?
     #(cd electrum/gui/kivy/; make theming) || echo "building kivy atlas failed! skipping."
 
-    python3 setup.py --quiet sdist --format=zip,gztar
+    find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} +
+
+    # note: .zip sdists would not be reproducible due to https://bugs.python.org/issue40963
+    TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar
 )

contrib/build-wine/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04@sha256:5f4bdc3467537cbbe563e80db2c3ec95d548a9145d64453b06939c4592d67b6d
+FROM ubuntu:18.04@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f
 
 ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
 
@@ -6,19 +6,22 @@ RUN dpkg --add-architecture i386 && \
     apt-get update -q && \
     apt-get install -qy \
         wget=1.19.4-1ubuntu2.2 \
-        gnupg2=2.2.4-1ubuntu1.2 \
-        dirmngr=2.2.4-1ubuntu1.2 \
+        gnupg2=2.2.4-1ubuntu1.3 \
+        dirmngr=2.2.4-1ubuntu1.3 \
         python3-software-properties=0.96.24.32.1 \
         software-properties-common=0.96.24.32.1
 
 RUN apt-get update -q && \
         apt-get install -qy \
-        git=1:2.17.1-1ubuntu0.5 \
+        git=1:2.17.1-1ubuntu0.7 \
         p7zip-full=16.02+dfsg-6 \
         make=4.1-9.1ubuntu1 \
         mingw-w64=5.0.3-1 \
+        mingw-w64-tools=5.0.3-1 \
+        win-iconv-mingw-w64-dev=0.0.8-2 \
         autotools-dev=20180224.1 \
         autoconf=2.69-11 \
+        autopoint=0.19.8.1-6ubuntu0.3 \
         libtool=2.4.6-2 \
         gettext=0.19.8.1-6
 

contrib/build-wine/build-electrum-git.sh

@@ -53,10 +53,6 @@ $PYTHON -m pip install --no-dependencies --no-warn-script-location .
 popd
 
 
-# these are deleted as they were not deterministic; and are not needed anyway
-rm "$WINEPREFIX"/drive_c/python3/Lib/site-packages/jsonschema-*.dist-info/RECORD
-
-
 rm -rf dist/
 
 # build standalone and portable versions

contrib/build-wine/build.sh

@@ -11,7 +11,7 @@ export CACHEDIR="$here/.cache"
 export PIP_CACHE_DIR="$CACHEDIR/pip_cache"
 
 export BUILD_TYPE="wine"
-export GCC_TRIPLET_HOST="i686-w64-mingw32"
+export GCC_TRIPLET_HOST="i686-w64-mingw32"  # make sure to clear caches if changing this
 export GCC_TRIPLET_BUILD="x86_64-pc-linux-gnu"
 export GCC_STRIP_BINARIES="1"
 
@@ -29,6 +29,12 @@ else
     "$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp"
 fi
 
+if [ -f "$PROJECT_ROOT/electrum/libzbar-0.dll" ]; then
+    info "libzbar already built, skipping"
+else
+    "$CONTRIB"/make_zbar.sh || fail "Could not build zbar"
+fi
+
 $here/prepare-wine.sh || fail "prepare-wine failed"
 
 info "Resetting modification time in C:\Python..."

contrib/build-wine/deterministic.spec

@@ -16,12 +16,14 @@ home = 'C:\\electrum\\'
 
 # see https://github.com/pyinstaller/pyinstaller/issues/2005
 hiddenimports = []
+hiddenimports += collect_submodules('pkg_resources')  # workaround for https://github.com/pypa/setuptools/issues/1963
 hiddenimports += collect_submodules('trezorlib')
 hiddenimports += collect_submodules('safetlib')
 hiddenimports += collect_submodules('btchip')
 hiddenimports += collect_submodules('keepkeylib')
 hiddenimports += collect_submodules('websocket')
 hiddenimports += collect_submodules('ckcc')
+hiddenimports += collect_submodules('bitbox02')
 hiddenimports += ['PyQt5.QtPrintSupport']  # needed by Revealer
 
 
@@ -32,13 +34,14 @@ binaries += [b for b in collect_dynamic_libs('PyQt5') if 'qwindowsvista' in b[0]
 
 binaries += [('C:/tmp/libsecp256k1-0.dll', '.')]
 binaries += [('C:/tmp/libusb-1.0.dll', '.')]
+binaries += [('C:/tmp/libzbar-0.dll', '.')]
 
 datas = [
     (home+'electrum/*.json', 'electrum'),
+    (home+'electrum/lnwire/*.csv', 'electrum/lnwire'),
     (home+'electrum/wordlist/english.txt', 'electrum/wordlist'),
     (home+'electrum/locale', 'electrum/locale'),
     (home+'electrum/plugins', 'electrum/plugins'),
-    ('C:\\Program Files (x86)\\ZBar\\bin\\', '.'),
     (home+'electrum/gui/icons', 'electrum/gui/icons'),
 ]
 datas += collect_data_files('trezorlib')
@@ -46,8 +49,7 @@ datas += collect_data_files('safetlib')
 datas += collect_data_files('btchip')
 datas += collect_data_files('keepkeylib')
 datas += collect_data_files('ckcc')
-datas += collect_data_files('jsonrpcserver')
-datas += collect_data_files('jsonrpcclient')
+datas += collect_data_files('bitbox02')
 
 # We don't put these files in to actually include them in the script but to make the Analysis method scan them for imports
 a = Analysis([home+'run_electrum',
@@ -137,7 +139,7 @@ exe_portable = EXE(
 #####
 # exe and separate files that NSIS uses to build installer "setup" exe
 
-exe_dependent = EXE(
+exe_inside_setup_noconsole = EXE(
     pyz,
     a.scripts,
     exclude_binaries=True,
@@ -148,8 +150,20 @@ exe_dependent = EXE(
     icon=home+'electrum/gui/icons/electrum.ico',
     console=False)
 
+exe_inside_setup_console = EXE(
+    pyz,
+    a.scripts,
+    exclude_binaries=True,
+    name=os.path.join('build\\pyi.win32\\electrum', cmdline_name+"-debug"),
+    debug=False,
+    strip=None,
+    upx=False,
+    icon=home+'electrum/gui/icons/electrum.ico',
+    console=True)
+
 coll = COLLECT(
-    exe_dependent,
+    exe_inside_setup_noconsole,
+    exe_inside_setup_console,
     a.binaries,
     a.zipfiles,
     a.datas,

contrib/build-wine/prepare-wine.sh

@@ -2,22 +2,18 @@
 
 # Please update these carefully, some versions won't work under Wine
 NSIS_FILENAME=nsis-3.05-setup.exe
-NSIS_URL=https://prdownloads.sourceforge.net/nsis/$NSIS_FILENAME?download
+NSIS_URL=https://downloads.sourceforge.net/project/nsis/NSIS%203/3.05/$NSIS_FILENAME
 NSIS_SHA256=1a3cc9401667547b9b9327a177b13485f7c59c2303d4b6183e7bc9e6c8d6bfdb
 
-ZBAR_FILENAME=zbarw-20121031-setup.exe
-ZBAR_URL=https://sourceforge.net/projects/zbarw/files/$ZBAR_FILENAME/download
-ZBAR_SHA256=177e32b272fa76528a3af486b74e9cb356707be1c5ace4ed3fcee9723e2c2c02
-
 LIBUSB_REPO="https://github.com/libusb/libusb.git"
-LIBUSB_COMMIT=e782eeb2514266f6738e242cdcb18e3ae1ed06fa
-# ^ tag v1.0.23
+LIBUSB_COMMIT="c6a35c56016ea2ab2f19115d2ea1e85e0edae155"
+# ^ tag v1.0.24
 
 PYINSTALLER_REPO="https://github.com/SomberNight/pyinstaller.git"
-PYINSTALLER_COMMIT=e934539374e30d1500fcdbe8e4eb0860413935b2
-# ^ tag 3.6, plus a custom commit that fixes cross-compilation with MinGW
+PYINSTALLER_COMMIT="31fda9dc83feb1b3f2ff08c89ff7ae61506fc1ca"
+# ^ tag 4.1, plus a custom commit that fixes cross-compilation with MinGW
 
-PYTHON_VERSION=3.7.6
+PYTHON_VERSION=3.7.9
 
 ## These settings probably don't need change
 export WINEPREFIX=/opt/wine64
@@ -47,27 +43,31 @@ info "Installing Python."
 # keys from https://www.python.org/downloads/#pubkeys
 KEYRING_PYTHON_DEV="keyring-electrum-build-python-dev.gpg"
 gpg --no-default-keyring --keyring $KEYRING_PYTHON_DEV --import "$here"/gpg_keys/7ED10B6531D7C8E1BC296021FC624643487034E5.asc
-PYTHON_DOWNLOADS="$CACHEDIR/python$PYTHON_VERSION"
+if [ "$GCC_TRIPLET_HOST" = "i686-w64-mingw32" ] ; then
+    ARCH="win32"
+elif [ "$GCC_TRIPLET_HOST" = "x86_64-w64-mingw32" ] ; then
+    ARCH="amd64"
+else
+    fail "unexpected GCC_TRIPLET_HOST: $GCC_TRIPLET_HOST"
+fi
+PYTHON_DOWNLOADS="$CACHEDIR/python$PYTHON_VERSION-$ARCH"
 mkdir -p "$PYTHON_DOWNLOADS"
 for msifile in core dev exe lib pip tools; do
     echo "Installing $msifile..."
-    download_if_not_exist "$PYTHON_DOWNLOADS/${msifile}.msi" "https://www.python.org/ftp/python/$PYTHON_VERSION/win32/${msifile}.msi"
-    download_if_not_exist "$PYTHON_DOWNLOADS/${msifile}.msi.asc" "https://www.python.org/ftp/python/$PYTHON_VERSION/win32/${msifile}.msi.asc"
+    download_if_not_exist "$PYTHON_DOWNLOADS/${msifile}.msi" "https://www.python.org/ftp/python/$PYTHON_VERSION/$ARCH/${msifile}.msi"
+    download_if_not_exist "$PYTHON_DOWNLOADS/${msifile}.msi.asc" "https://www.python.org/ftp/python/$PYTHON_VERSION/$ARCH/${msifile}.msi.asc"
     verify_signature "$PYTHON_DOWNLOADS/${msifile}.msi.asc" $KEYRING_PYTHON_DEV
     wine msiexec /i "$PYTHON_DOWNLOADS/${msifile}.msi" /qb TARGETDIR=$PYHOME
 done
 
+break_legacy_easy_install
+
 info "Installing build dependencies."
-$PYTHON -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-wine-build.txt
+$PYTHON -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-build-wine.txt
 
 info "Installing dependencies specific to binaries."
 $PYTHON -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-binaries.txt
 
-info "Installing ZBar."
-download_if_not_exist "$CACHEDIR/$ZBAR_FILENAME" "$ZBAR_URL"
-verify_hash "$CACHEDIR/$ZBAR_FILENAME" "$ZBAR_SHA256"
-wine "$CACHEDIR/$ZBAR_FILENAME" /S
-
 info "Installing NSIS."
 download_if_not_exist "$CACHEDIR/$NSIS_FILENAME" "$NSIS_URL"
 verify_hash "$CACHEDIR/$NSIS_FILENAME" "$NSIS_SHA256"
@@ -88,21 +88,22 @@ info "Compiling libusb..."
     git init
     git remote add origin $LIBUSB_REPO
     git fetch --depth 1 origin $LIBUSB_COMMIT
-    git checkout -b pinned FETCH_HEAD
+    git checkout -b pinned "${LIBUSB_COMMIT}^{commit}"
     echo "libusb_1_0_la_LDFLAGS += -Wc,-static" >> libusb/Makefile.am
     ./bootstrap.sh || fail "Could not bootstrap libusb"
-    host="i686-w64-mingw32"
+    host="$GCC_TRIPLET_HOST"
     LDFLAGS="-Wl,--no-insert-timestamp" ./configure \
         --host=$host \
-        --build=x86_64-pc-linux-gnu || fail "Could not run ./configure for libusb"
+        --build=$GCC_TRIPLET_BUILD || fail "Could not run ./configure for libusb"
     make -j4 || fail "Could not build libusb"
     ${host}-strip libusb/.libs/libusb-1.0.dll
 ) || fail "libusb build failed"
 cp "$CACHEDIR/libusb/libusb/.libs/libusb-1.0.dll" $WINEPREFIX/drive_c/tmp/  || fail "Could not copy libusb to its destination"
 
 
-# copy libsecp dll (already built)
+# copy already built DLLs
 cp "$PROJECT_ROOT/electrum/libsecp256k1-0.dll" $WINEPREFIX/drive_c/tmp/ || fail "Could not copy libsecp to its destination"
+cp "$PROJECT_ROOT/electrum/libzbar-0.dll" $WINEPREFIX/drive_c/tmp/ || fail "Could not copy libzbar to its destination"
 
 
 info "Building PyInstaller."
@@ -119,17 +120,28 @@ info "Building PyInstaller."
     git init
     git remote add origin $PYINSTALLER_REPO
     git fetch --depth 1 origin $PYINSTALLER_COMMIT
-    git checkout -b pinned FETCH_HEAD
+    git checkout -b pinned "${PYINSTALLER_COMMIT}^{commit}"
     rm -fv PyInstaller/bootloader/Windows-*/run*.exe || true
     # add reproducible randomness. this ensures we build a different bootloader for each commit.
     # if we built the same one for all releases, that might also get anti-virus false positives
     echo "const char *electrum_tag = \"tagged by Electrum@$ELECTRUM_COMMIT_HASH\";" >> ./bootloader/src/pyi_main.c
     pushd bootloader
     # cross-compile to Windows using host python
-    python3 ./waf all CC=i686-w64-mingw32-gcc CFLAGS="-static -Wno-dangling-else -Wno-error=unused-value"
+    python3 ./waf all CC="${GCC_TRIPLET_HOST}-gcc" \
+                      CFLAGS="-static \
+                              -Wno-dangling-else \
+                              -Wno-error=unused-value \
+                              -Wno-error=implicit-function-declaration \
+                              -Wno-error=int-to-pointer-cast"
     popd
     # sanity check bootloader is there:
-    [[ -e PyInstaller/bootloader/Windows-32bit/runw.exe ]] || fail "Could not find runw.exe in target dir!"
+    if [ "$GCC_TRIPLET_HOST" = "i686-w64-mingw32" ] ; then
+        [[ -e PyInstaller/bootloader/Windows-32bit/runw.exe ]] || fail "Could not find runw.exe in target dir! (32bit)"
+    elif [ "$GCC_TRIPLET_HOST" = "x86_64-w64-mingw32" ] ; then
+        [[ -e PyInstaller/bootloader/Windows-64bit/runw.exe ]] || fail "Could not find runw.exe in target dir! (64bit)"
+    else
+        fail "unexpected GCC_TRIPLET_HOST: $GCC_TRIPLET_HOST"
+    fi
 ) || fail "PyInstaller build failed"
 info "Installing PyInstaller."
 $PYTHON -m pip install --no-dependencies --no-warn-script-location ./pyinstaller

contrib/build-wine/sign.sh

@@ -23,6 +23,7 @@ echo "Found $(ls *.exe | wc -w) files to sign."
 for f in $(ls *.exe); do
     echo "Signing $f..."
     osslsigncode sign \
+      -h sha256 \
       -certs "$CERT_FILE" \
       -key "$KEY_FILE" \
       -n "Electrum" \

contrib/build_tools_util.sh

@@ -119,8 +119,6 @@ export SOURCE_DATE_EPOCH=1530212462
 export PYTHONHASHSEED=22
 # Set the build type, overridden by wine build
 export BUILD_TYPE="${BUILD_TYPE:-$(uname | tr '[:upper:]' '[:lower:]')}"
-# No additional autoconf flags by default
-export AUTOCONF_FLAGS=""
 # Add host / build flags if the triplets are set
 if [ -n "$GCC_TRIPLET_HOST" ] ; then
     export AUTOCONF_FLAGS="$AUTOCONF_FLAGS --host=$GCC_TRIPLET_HOST"
@@ -131,3 +129,25 @@ fi
 
 export GCC_STRIP_BINARIES="${GCC_STRIP_BINARIES:-0}"
 
+
+function break_legacy_easy_install() {
+    # We don't want setuptools sneakily installing dependencies, invisible to pip.
+    # This ensures that if setuptools calls distutils which then calls easy_install,
+    # easy_install will not download packages over the network.
+    # see https://pip.pypa.io/en/stable/reference/pip_install/#controlling-setup-requires
+    # see https://github.com/pypa/setuptools/issues/1916#issuecomment-743350566
+    info "Intentionally breaking legacy easy_install."
+    DISTUTILS_CFG="${HOME}/.pydistutils.cfg"
+    DISTUTILS_CFG_BAK="${HOME}/.pydistutils.cfg.orig"
+    # If we are not inside docker, we might be overwriting a config file on the user's system...
+    if [ -e "$DISTUTILS_CFG" ] && [ ! -e "$DISTUTILS_CFG_BAK" ]; then
+        warn "Overwriting python distutils config file at '$DISTUTILS_CFG'. A copy will be saved at '$DISTUTILS_CFG_BAK'."
+        mv "$DISTUTILS_CFG" "$DISTUTILS_CFG_BAK"
+    fi
+    cat <<EOF > "$DISTUTILS_CFG"
+[easy_install]
+index_url = ''
+find_links = ''
+EOF
+}
+

contrib/deterministic-build/electrum-locale

@@ -1 +1 @@
-Subproject commit aafd932d37f35a1f276909b6ec27d2f7a60e606a
+Subproject commit aa3b991e2b43a038284adb832b07fe7c1fa0ff96

contrib/deterministic-build/requirements-binaries-mac.txt

@@ -0,0 +1,95 @@
+cffi==1.14.4 \
+    --hash=sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e \
+    --hash=sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d \
+    --hash=sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a \
+    --hash=sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec \
+    --hash=sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362 \
+    --hash=sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668 \
+    --hash=sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c \
+    --hash=sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b \
+    --hash=sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06 \
+    --hash=sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698 \
+    --hash=sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2 \
+    --hash=sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c \
+    --hash=sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7 \
+    --hash=sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009 \
+    --hash=sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03 \
+    --hash=sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b \
+    --hash=sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909 \
+    --hash=sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53 \
+    --hash=sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35 \
+    --hash=sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26 \
+    --hash=sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b \
+    --hash=sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01 \
+    --hash=sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb \
+    --hash=sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293 \
+    --hash=sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd \
+    --hash=sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d \
+    --hash=sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3 \
+    --hash=sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d \
+    --hash=sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e \
+    --hash=sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca \
+    --hash=sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d \
+    --hash=sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775 \
+    --hash=sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375 \
+    --hash=sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b \
+    --hash=sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b \
+    --hash=sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f
+cryptography==3.3.1 \
+    --hash=sha256:0003a52a123602e1acee177dc90dd201f9bb1e73f24a070db7d36c588e8f5c7d \
+    --hash=sha256:0e85aaae861d0485eb5a79d33226dd6248d2a9f133b81532c8f5aae37de10ff7 \
+    --hash=sha256:594a1db4511bc4d960571536abe21b4e5c3003e8750ab8365fafce71c5d86901 \
+    --hash=sha256:69e836c9e5ff4373ce6d3ab311c1a2eed274793083858d3cd4c7d12ce20d5f9c \
+    --hash=sha256:788a3c9942df5e4371c199d10383f44a105d67d401fb4304178020142f020244 \
+    --hash=sha256:7e177e4bea2de937a584b13645cab32f25e3d96fc0bc4a4cf99c27dc77682be6 \
+    --hash=sha256:83d9d2dfec70364a74f4e7c70ad04d3ca2e6a08b703606993407bf46b97868c5 \
+    --hash=sha256:84ef7a0c10c24a7773163f917f1cb6b4444597efd505a8aed0a22e8c4780f27e \
+    --hash=sha256:9e21301f7a1e7c03dbea73e8602905a4ebba641547a462b26dd03451e5769e7c \
+    --hash=sha256:9f6b0492d111b43de5f70052e24c1f0951cb9e6022188ebcb1cc3a3d301469b0 \
+    --hash=sha256:a69bd3c68b98298f490e84519b954335154917eaab52cf582fa2c5c7efc6e812 \
+    --hash=sha256:b4890d5fb9b7a23e3bf8abf5a8a7da8e228f1e97dc96b30b95685df840b6914a \
+    --hash=sha256:c366df0401d1ec4e548bebe8f91d55ebcc0ec3137900d214dd7aac8427ef3030 \
+    --hash=sha256:dc42f645f8f3a489c3dd416730a514e7a91a59510ddaadc09d04224c098d3302
+pip==20.1.1 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
+pycparser==2.20 \
+    --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \
+    --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705
+PyQt5==5.15.2 \
+    --hash=sha256:29889845688a54d62820585ad5b2e0200a36b304ff3d7a555e95599f110ba4ce \
+    --hash=sha256:372b08dc9321d1201e4690182697c5e7ffb2e0770e6b4a45519025134b12e4fc \
+    --hash=sha256:894ca4ae767a8d6cf5903784b71f755073c78cb8c167eecf6e4ed6b3b055ac6a \
+    --hash=sha256:ea24f24b7679bf393dd2e4f53fe0ce65021be18304c1ff7a226c2fc5c356d0da \
+    --hash=sha256:faaecb76ec65e12673a968e7f5bc02495957e6996f0a3fa0d98895f9e4113746
+PyQt5-sip==12.8.1 \
+    --hash=sha256:0304ca9114b9817a270f67f421355075b78ff9fc25ac58ffd72c2601109d2194 \
+    --hash=sha256:0cd969be528c27bbd4755bd323dff4a79a8fdda28215364e6ce3e069cb56c2a9 \
+    --hash=sha256:2f35e82fd7ec1e1f6716e9154721c7594956a4f5bd4f826d8c6a6453833cc2f0 \
+    --hash=sha256:30e944db9abee9cc757aea16906d4198129558533eb7fadbe48c5da2bd18e0bd \
+    --hash=sha256:34dcd29be47553d5f016ff86e89e24cbc5eebae92eb2f96fb32d2d7ba028c43c \
+    --hash=sha256:5a011aeff89660622a6d5c3388d55a9d76932f3b82c95e82fc31abd8b1d2990d \
+    --hash=sha256:6c1ebee60f1d2b3c70aff866b7933d8d8d7646011f7c32f9321ee88c290aa4f9 \
+    --hash=sha256:7b81382ce188d63890a0e35abe0f9bb946cabc873a31873b73583b0fc84ac115 \
+    --hash=sha256:832fd60a264de4134c2824d393320838f3ab648180c9c357ec58a74524d24507 \
+    --hash=sha256:84ba7746762bd223bed22428e8561aa267a229c28344c2d28c5d5d3f8970cffb \
+    --hash=sha256:9312ec47cac4e33c11503bc1cbeeb0bdae619620472f38e2078c5a51020a930f \
+    --hash=sha256:a1b8ef013086e224b8e86c93f880f776d01b59195bdfa2a8e0b23f0480678fec \
+    --hash=sha256:a29e2ac399429d3b7738f73e9081e50783e61ac5d29344e0802d0dcd6056c5a2 \
+    --hash=sha256:b6d42250baec52a5f77de64e2951d001c5501c3a2df2179f625b241cbaec3369 \
+    --hash=sha256:bb5a87b66fc1445915104ee97f7a20a69decb42f52803e3b0795fa17ff88226c \
+    --hash=sha256:c317ab1263e6417c498b81f5c970a9b1af7acefab1f80b4cc0f2f8e661f29fc5 \
+    --hash=sha256:c9800729badcb247765e4ffe2241549d02da1fa435b9db224845bc37c3e99cb0 \
+    --hash=sha256:c9d6d448c29dc6606bb7974696608f81f4316c8234f7c7216396ed110075e777 \
+    --hash=sha256:da9c9f1e65b9d09e73bd75befc82961b6b61b5a3b9d0a7c832168e1415f163c6 \
+    --hash=sha256:ed897c58acf4a3cdca61469daa31fe6e44c33c6c06a37c3f21fab31780b3b86a \
+    --hash=sha256:f168f0a7f32b81bfeffdf003c36f25d81c97dee5eb67072a5183e761fe250f13
+setuptools==46.4.0 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+six==1.15.0 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+wheel==0.34.2 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e

contrib/deterministic-build/requirements-binaries.txt

@@ -1,27 +1,95 @@
-pip==19.3.1 \
-    --hash=sha256:21207d76c1031e517668898a6b46a9fb1501c7a4710ef5dfd6a40ad9e6757ea7 \
-    --hash=sha256:6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7
-PyQt5==5.11.3 \
-    --hash=sha256:517e4339135c4874b799af0d484bc2e8c27b54850113a68eec40a0b56534f450 \
-    --hash=sha256:ac1eb5a114b6e7788e8be378be41c5e54b17d5158994504e85e43b5fca006a39 \
-    --hash=sha256:d2309296a5a79d0a1c0e6c387c30f0398b65523a6dcc8a19cc172e46b949e00d \
-    --hash=sha256:e85936bae1581bcb908847d2038e5b34237a5e6acc03130099a78930770e7ead
-PyQt5-sip==4.19.13 \
-    --hash=sha256:125f77c087572c9272219cda030a63c2f996b8507592b2a54d7ef9b75f9f054d \
-    --hash=sha256:14c37b06e3fb7c2234cb208fa461ec4e62b4ba6d8b32ca3753c0b2cfd61b00e3 \
-    --hash=sha256:1cb2cf52979f9085fc0eab7e0b2438eb4430d4aea8edec89762527e17317175b \
-    --hash=sha256:4babef08bccbf223ec34464e1ed0a23caeaeea390ca9a3529227d9a57f0d6ee4 \
-    --hash=sha256:53cb9c1208511cda0b9ed11cffee992a5a2f5d96eb88722569b2ce65ecf6b960 \
-    --hash=sha256:549449d9461d6c665cbe8af4a3808805c5e6e037cd2ce4fd93308d44a049bfac \
-    --hash=sha256:5f5b3089b200ff33de3f636b398e7199b57a6b5c1bb724bdb884580a072a14b5 \
-    --hash=sha256:a4d9bf6e1fa2dd6e73f1873f1a47cee11a6ba0cf9ba8cf7002b28c76823600d0 \
-    --hash=sha256:a4ee6026216f1fbe25c8847f9e0fbce907df5b908f84816e21af16ec7666e6fe \
-    --hash=sha256:a91a308a5e0cc99de1e97afd8f09f46dd7ca20cfaa5890ef254113eebaa1adff \
-    --hash=sha256:b0342540da479d2713edc68fb21f307473f68da896ad5c04215dae97630e0069 \
-    --hash=sha256:f997e21b4e26a3397cb7b255b8d1db5b9772c8e0c94b6d870a5a0ab5c27eacaa
-setuptools==42.0.2 \
-    --hash=sha256:c5b372090d7c8709ce79a6a66872a91e518f7d65af97fca78135e1cb10d4b940 \
-    --hash=sha256:c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6
-wheel==0.33.6 \
-    --hash=sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646 \
-    --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28
+cffi==1.14.4 \
+    --hash=sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e \
+    --hash=sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d \
+    --hash=sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a \
+    --hash=sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec \
+    --hash=sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362 \
+    --hash=sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668 \
+    --hash=sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c \
+    --hash=sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b \
+    --hash=sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06 \
+    --hash=sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698 \
+    --hash=sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2 \
+    --hash=sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c \
+    --hash=sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7 \
+    --hash=sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009 \
+    --hash=sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03 \
+    --hash=sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b \
+    --hash=sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909 \
+    --hash=sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53 \
+    --hash=sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35 \
+    --hash=sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26 \
+    --hash=sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b \
+    --hash=sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01 \
+    --hash=sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb \
+    --hash=sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293 \
+    --hash=sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd \
+    --hash=sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d \
+    --hash=sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3 \
+    --hash=sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d \
+    --hash=sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e \
+    --hash=sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca \
+    --hash=sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d \
+    --hash=sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775 \
+    --hash=sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375 \
+    --hash=sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b \
+    --hash=sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b \
+    --hash=sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f
+cryptography==3.3.1 \
+    --hash=sha256:0003a52a123602e1acee177dc90dd201f9bb1e73f24a070db7d36c588e8f5c7d \
+    --hash=sha256:0e85aaae861d0485eb5a79d33226dd6248d2a9f133b81532c8f5aae37de10ff7 \
+    --hash=sha256:594a1db4511bc4d960571536abe21b4e5c3003e8750ab8365fafce71c5d86901 \
+    --hash=sha256:69e836c9e5ff4373ce6d3ab311c1a2eed274793083858d3cd4c7d12ce20d5f9c \
+    --hash=sha256:788a3c9942df5e4371c199d10383f44a105d67d401fb4304178020142f020244 \
+    --hash=sha256:7e177e4bea2de937a584b13645cab32f25e3d96fc0bc4a4cf99c27dc77682be6 \
+    --hash=sha256:83d9d2dfec70364a74f4e7c70ad04d3ca2e6a08b703606993407bf46b97868c5 \
+    --hash=sha256:84ef7a0c10c24a7773163f917f1cb6b4444597efd505a8aed0a22e8c4780f27e \
+    --hash=sha256:9e21301f7a1e7c03dbea73e8602905a4ebba641547a462b26dd03451e5769e7c \
+    --hash=sha256:9f6b0492d111b43de5f70052e24c1f0951cb9e6022188ebcb1cc3a3d301469b0 \
+    --hash=sha256:a69bd3c68b98298f490e84519b954335154917eaab52cf582fa2c5c7efc6e812 \
+    --hash=sha256:b4890d5fb9b7a23e3bf8abf5a8a7da8e228f1e97dc96b30b95685df840b6914a \
+    --hash=sha256:c366df0401d1ec4e548bebe8f91d55ebcc0ec3137900d214dd7aac8427ef3030 \
+    --hash=sha256:dc42f645f8f3a489c3dd416730a514e7a91a59510ddaadc09d04224c098d3302
+pip==20.1.1 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
+pycparser==2.20 \
+    --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \
+    --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705
+PyQt5==5.15.2 \
+    --hash=sha256:29889845688a54d62820585ad5b2e0200a36b304ff3d7a555e95599f110ba4ce \
+    --hash=sha256:372b08dc9321d1201e4690182697c5e7ffb2e0770e6b4a45519025134b12e4fc \
+    --hash=sha256:894ca4ae767a8d6cf5903784b71f755073c78cb8c167eecf6e4ed6b3b055ac6a \
+    --hash=sha256:ea24f24b7679bf393dd2e4f53fe0ce65021be18304c1ff7a226c2fc5c356d0da \
+    --hash=sha256:faaecb76ec65e12673a968e7f5bc02495957e6996f0a3fa0d98895f9e4113746
+PyQt5-sip==12.8.1 \
+    --hash=sha256:0304ca9114b9817a270f67f421355075b78ff9fc25ac58ffd72c2601109d2194 \
+    --hash=sha256:0cd969be528c27bbd4755bd323dff4a79a8fdda28215364e6ce3e069cb56c2a9 \
+    --hash=sha256:2f35e82fd7ec1e1f6716e9154721c7594956a4f5bd4f826d8c6a6453833cc2f0 \
+    --hash=sha256:30e944db9abee9cc757aea16906d4198129558533eb7fadbe48c5da2bd18e0bd \
+    --hash=sha256:34dcd29be47553d5f016ff86e89e24cbc5eebae92eb2f96fb32d2d7ba028c43c \
+    --hash=sha256:5a011aeff89660622a6d5c3388d55a9d76932f3b82c95e82fc31abd8b1d2990d \
+    --hash=sha256:6c1ebee60f1d2b3c70aff866b7933d8d8d7646011f7c32f9321ee88c290aa4f9 \
+    --hash=sha256:7b81382ce188d63890a0e35abe0f9bb946cabc873a31873b73583b0fc84ac115 \
+    --hash=sha256:832fd60a264de4134c2824d393320838f3ab648180c9c357ec58a74524d24507 \
+    --hash=sha256:84ba7746762bd223bed22428e8561aa267a229c28344c2d28c5d5d3f8970cffb \
+    --hash=sha256:9312ec47cac4e33c11503bc1cbeeb0bdae619620472f38e2078c5a51020a930f \
+    --hash=sha256:a1b8ef013086e224b8e86c93f880f776d01b59195bdfa2a8e0b23f0480678fec \
+    --hash=sha256:a29e2ac399429d3b7738f73e9081e50783e61ac5d29344e0802d0dcd6056c5a2 \
+    --hash=sha256:b6d42250baec52a5f77de64e2951d001c5501c3a2df2179f625b241cbaec3369 \
+    --hash=sha256:bb5a87b66fc1445915104ee97f7a20a69decb42f52803e3b0795fa17ff88226c \
+    --hash=sha256:c317ab1263e6417c498b81f5c970a9b1af7acefab1f80b4cc0f2f8e661f29fc5 \
+    --hash=sha256:c9800729badcb247765e4ffe2241549d02da1fa435b9db224845bc37c3e99cb0 \
+    --hash=sha256:c9d6d448c29dc6606bb7974696608f81f4316c8234f7c7216396ed110075e777 \
+    --hash=sha256:da9c9f1e65b9d09e73bd75befc82961b6b61b5a3b9d0a7c832168e1415f163c6 \
+    --hash=sha256:ed897c58acf4a3cdca61469daa31fe6e44c33c6c06a37c3f21fab31780b3b86a \
+    --hash=sha256:f168f0a7f32b81bfeffdf003c36f25d81c97dee5eb67072a5183e761fe250f13
+setuptools==46.4.0 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+six==1.15.0 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+wheel==0.34.2 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e

contrib/deterministic-build/requirements-build-appimage.txt

@@ -0,0 +1,45 @@
+Cython==0.29.21 \
+    --hash=sha256:0ac10bf476476a9f7ef61ec6e44c280ef434473124ad31d3132b720f7b0e8d2a \
+    --hash=sha256:0e25c209c75df8785480dcef85db3d36c165dbc0f4c503168e8763eb735704f2 \
+    --hash=sha256:171b9f70ceafcec5852089d0f9c1e75b0d554f46c882cd4e2e4acaba9bd7d148 \
+    --hash=sha256:23f3a00b843a19de8bb4468b087db5b413a903213f67188729782488d67040e0 \
+    --hash=sha256:2922e3031ba9ebbe7cb9200b585cc33b71d66023d78450dcb883f824f4969371 \
+    --hash=sha256:31c71a615f38401b0dc1f2a5a9a6c421ffd8908c4cd5bbedc4014c1b876488e8 \
+    --hash=sha256:473df5d5e400444a36ed81c6596f56a5b52a3481312d0a48d68b777790f730ae \
+    --hash=sha256:497841897942f734b0abc2dead2d4009795ee992267a70a23485fd0e937edc0b \
+    --hash=sha256:539e59949aab4955c143a468810123bf22d3e8556421e1ce2531ed4893914ca0 \
+    --hash=sha256:540b3bee0711aac2e99bda4fa0a46dbcd8c74941666bfc1ef9236b1a64eeffd9 \
+    --hash=sha256:57ead89128dee9609119c93d3926c7a2add451453063147900408a50144598c6 \
+    --hash=sha256:5c4276fdcbccdf1e3c1756c7aeb8395e9a36874fa4d30860e7694f43d325ae13 \
+    --hash=sha256:5da187bebe38030325e1c0b5b8a804d489410be2d384c0ef3ba39493c67eb51e \
+    --hash=sha256:5e545a48f919e40079b0efe7b0e081c74b96f9ef25b9c1ff4cdbd95764426b58 \
+    --hash=sha256:603b9f1b8e93e8b494d3e89320c410679e21018e48b6cbc77280f5db71f17dc0 \
+    --hash=sha256:695a6bcaf9e12b1e471dfce96bbecf22a1487adc2ac6106b15960a2b51b97f5d \
+    --hash=sha256:715294cd2246b39a8edca464a8366eb635f17213e4a6b9e74e52d8b877a8cb63 \
+    --hash=sha256:7ebaa8800c376bcdae596fb1372cb4232a5ef957619d35839520d2786f2debb9 \
+    --hash=sha256:856c7fb31d247ce713d60116375e1f8153d0291ab5e92cca7d8833a524ba9991 \
+    --hash=sha256:8c6e25e9cc4961bb2abb1777c6fa9d0fa2d9b014beb3276cebe69996ff162b78 \
+    --hash=sha256:9207fdedc7e789a3dcaca628176b80c82fbed9ae0997210738cbb12536a56699 \
+    --hash=sha256:93f5fed1c9445fb7afe20450cdaf94b0e0356d47cc75008105be89c6a2e417b1 \
+    --hash=sha256:9ce5e5209f8406ffc2b058b1293cce7a954911bb7991e623564d489197c9ba30 \
+    --hash=sha256:a0674f246ad5e1571ef29d4c5ec1d6ecabe9e6c424ad0d6fee46b914d5d24d69 \
+    --hash=sha256:b2f9172e4d6358f33ecce6a4339b5960f9f83eab67ea244baa812737793826b7 \
+    --hash=sha256:b8a8a31b9e8860634adbca30fea1d0c7f08e208b3d7611f3e580e5f20992e5d7 \
+    --hash=sha256:b8d8497091c1dc8705d1575c71e908a93b1f127a174b2d472020f3d84263ac28 \
+    --hash=sha256:c111ac9abdf715762e4fb87395e59d61c0fbb6ce79eb2e24167700b6cfa8ba79 \
+    --hash=sha256:c4b78356074fcaac04ecb4de289f11d506e438859877670992ece11f9c90f37b \
+    --hash=sha256:c541b2b49c6638f2b5beb9316726db84a8d1c132bf31b942dae1f9c7f6ad3b92 \
+    --hash=sha256:c8435959321cf8aec867bbad54b83b7fb8343204b530d85d9ea7a1f5329d5ac2 \
+    --hash=sha256:ccb77faeaad99e99c6c444d04862c6cf604204fe0a07d4c8f9cbf2c9012d7d5a \
+    --hash=sha256:e272ed97d20b026f4f25a012b25d7d7672a60e4f72b9ca385239d693cd91b2d5 \
+    --hash=sha256:e57acb89bd55943c8d8bf813763d20b9099cc7165c0f16b707631a7654be9cad \
+    --hash=sha256:e93acd1f603a0c1786e0841f066ae7cef014cf4750e3cd06fd03cfdf46361419
+pip==20.1.1 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
+setuptools==46.4.0 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+wheel==0.34.2 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e

contrib/deterministic-build/requirements-build-mac.txt

@@ -0,0 +1,56 @@
+altgraph==0.17 \
+    --hash=sha256:1f05a47122542f97028caf78775a095fbe6a2699b5089de8477eb583167d69aa \
+    --hash=sha256:c623e5f3408ca61d4016f23a681b9adb100802ca3e3da5e718915a9e4052cebe
+Cython==0.29.21 \
+    --hash=sha256:0ac10bf476476a9f7ef61ec6e44c280ef434473124ad31d3132b720f7b0e8d2a \
+    --hash=sha256:0e25c209c75df8785480dcef85db3d36c165dbc0f4c503168e8763eb735704f2 \
+    --hash=sha256:171b9f70ceafcec5852089d0f9c1e75b0d554f46c882cd4e2e4acaba9bd7d148 \
+    --hash=sha256:23f3a00b843a19de8bb4468b087db5b413a903213f67188729782488d67040e0 \
+    --hash=sha256:2922e3031ba9ebbe7cb9200b585cc33b71d66023d78450dcb883f824f4969371 \
+    --hash=sha256:31c71a615f38401b0dc1f2a5a9a6c421ffd8908c4cd5bbedc4014c1b876488e8 \
+    --hash=sha256:473df5d5e400444a36ed81c6596f56a5b52a3481312d0a48d68b777790f730ae \
+    --hash=sha256:497841897942f734b0abc2dead2d4009795ee992267a70a23485fd0e937edc0b \
+    --hash=sha256:539e59949aab4955c143a468810123bf22d3e8556421e1ce2531ed4893914ca0 \
+    --hash=sha256:540b3bee0711aac2e99bda4fa0a46dbcd8c74941666bfc1ef9236b1a64eeffd9 \
+    --hash=sha256:57ead89128dee9609119c93d3926c7a2add451453063147900408a50144598c6 \
+    --hash=sha256:5c4276fdcbccdf1e3c1756c7aeb8395e9a36874fa4d30860e7694f43d325ae13 \
+    --hash=sha256:5da187bebe38030325e1c0b5b8a804d489410be2d384c0ef3ba39493c67eb51e \
+    --hash=sha256:5e545a48f919e40079b0efe7b0e081c74b96f9ef25b9c1ff4cdbd95764426b58 \
+    --hash=sha256:603b9f1b8e93e8b494d3e89320c410679e21018e48b6cbc77280f5db71f17dc0 \
+    --hash=sha256:695a6bcaf9e12b1e471dfce96bbecf22a1487adc2ac6106b15960a2b51b97f5d \
+    --hash=sha256:715294cd2246b39a8edca464a8366eb635f17213e4a6b9e74e52d8b877a8cb63 \
+    --hash=sha256:7ebaa8800c376bcdae596fb1372cb4232a5ef957619d35839520d2786f2debb9 \
+    --hash=sha256:856c7fb31d247ce713d60116375e1f8153d0291ab5e92cca7d8833a524ba9991 \
+    --hash=sha256:8c6e25e9cc4961bb2abb1777c6fa9d0fa2d9b014beb3276cebe69996ff162b78 \
+    --hash=sha256:9207fdedc7e789a3dcaca628176b80c82fbed9ae0997210738cbb12536a56699 \
+    --hash=sha256:93f5fed1c9445fb7afe20450cdaf94b0e0356d47cc75008105be89c6a2e417b1 \
+    --hash=sha256:9ce5e5209f8406ffc2b058b1293cce7a954911bb7991e623564d489197c9ba30 \
+    --hash=sha256:a0674f246ad5e1571ef29d4c5ec1d6ecabe9e6c424ad0d6fee46b914d5d24d69 \
+    --hash=sha256:b2f9172e4d6358f33ecce6a4339b5960f9f83eab67ea244baa812737793826b7 \
+    --hash=sha256:b8a8a31b9e8860634adbca30fea1d0c7f08e208b3d7611f3e580e5f20992e5d7 \
+    --hash=sha256:b8d8497091c1dc8705d1575c71e908a93b1f127a174b2d472020f3d84263ac28 \
+    --hash=sha256:c111ac9abdf715762e4fb87395e59d61c0fbb6ce79eb2e24167700b6cfa8ba79 \
+    --hash=sha256:c4b78356074fcaac04ecb4de289f11d506e438859877670992ece11f9c90f37b \
+    --hash=sha256:c541b2b49c6638f2b5beb9316726db84a8d1c132bf31b942dae1f9c7f6ad3b92 \
+    --hash=sha256:c8435959321cf8aec867bbad54b83b7fb8343204b530d85d9ea7a1f5329d5ac2 \
+    --hash=sha256:ccb77faeaad99e99c6c444d04862c6cf604204fe0a07d4c8f9cbf2c9012d7d5a \
+    --hash=sha256:e272ed97d20b026f4f25a012b25d7d7672a60e4f72b9ca385239d693cd91b2d5 \
+    --hash=sha256:e57acb89bd55943c8d8bf813763d20b9099cc7165c0f16b707631a7654be9cad \
+    --hash=sha256:e93acd1f603a0c1786e0841f066ae7cef014cf4750e3cd06fd03cfdf46361419
+macholib==1.14 \
+    --hash=sha256:0c436bc847e7b1d9bda0560351bf76d7caf930fb585a828d13608839ef42c432 \
+    --hash=sha256:c500f02867515e6c60a27875b408920d18332ddf96b4035ef03beddd782d4281
+pip==20.1.1 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
+pyinstaller==4.1 \
+    --hash=sha256:954ae81de9a4bc096ff02433b3e245b9272fe53f27cac319e71fe7540952bd3d
+pyinstaller-hooks-contrib==2020.11 \
+    --hash=sha256:fa8280b79d8a2b267a2e43ff44f73b3e4a68fc8d205b8d34e8e06c960f7c2fcf \
+    --hash=sha256:fc3290a2ca337d1d58c579c223201360bfe74caed6454eaf5a2550b77dbda45c
+setuptools==46.4.0 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+wheel==0.34.2 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e

contrib/deterministic-build/requirements-build-sdist.txt

@@ -0,0 +1,9 @@
+pip==20.1.1 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
+setuptools==46.4.0 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+wheel==0.34.2 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e

contrib/deterministic-build/requirements-build-wine.txt

@@ -0,0 +1,22 @@
+altgraph==0.17 \
+    --hash=sha256:1f05a47122542f97028caf78775a095fbe6a2699b5089de8477eb583167d69aa \
+    --hash=sha256:c623e5f3408ca61d4016f23a681b9adb100802ca3e3da5e718915a9e4052cebe
+future==0.18.2 \
+    --hash=sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d
+pefile==2019.4.18 \
+    --hash=sha256:a5d6e8305c6b210849b47a6174ddf9c452b2888340b8177874b862ba6c207645
+pip==20.1.1 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
+pyinstaller-hooks-contrib==2020.11 \
+    --hash=sha256:fa8280b79d8a2b267a2e43ff44f73b3e4a68fc8d205b8d34e8e06c960f7c2fcf \
+    --hash=sha256:fc3290a2ca337d1d58c579c223201360bfe74caed6454eaf5a2550b77dbda45c
+pywin32-ctypes==0.2.0 \
+    --hash=sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942 \
+    --hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98
+setuptools==46.4.0 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+wheel==0.34.2 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e

contrib/deterministic-build/requirements-hw.txt

@@ -1,120 +1,163 @@
-btchip-python==0.1.28 \
-    --hash=sha256:da09d0d7a6180d428833795ea9a233c3b317ddfcccea8cc6f0eba59435e5dd83
-certifi==2019.11.28 \
-    --hash=sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3 \
-    --hash=sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f
-chardet==3.0.4 \
-    --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
-    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691
-ckcc-protocol==0.8.0 \
-    --hash=sha256:bad1d1448423472df95ba67621fdd0ad919e625fbe0a4d3ba93648f34ea286e0 \
-    --hash=sha256:f0851c98b91825d19567d0d3bac1b28044d40a3d5f194c8b04c5338f114d7ad5
-click==7.0 \
-    --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
-    --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7
-construct==2.9.45 \
-    --hash=sha256:2271a0efd0798679dea825ff47e22a4c550456a5db0ba8baa82f7eae0af0118c
-Cython==0.29.10 \
-    --hash=sha256:0afa0b121b89de619e71587e25702e2b7068d7da2164c47e6eee80c17823a62f \
-    --hash=sha256:1c608ba76f7a20cc9f0c021b7fe5cb04bc1a70327ae93a9298b1bc3e0edddebe \
-    --hash=sha256:26229570d6787ff3caa932fe9d802960f51a89239b990d275ae845405ce43857 \
-    --hash=sha256:2a9deafa437b6154cac2f25bb88e0bfd075a897c8dc847669d6f478d7e3ee6b1 \
-    --hash=sha256:2f28396fbce6d9d68a40edbf49a6729cf9d92a4d39ff0f501947a89188e9099f \
-    --hash=sha256:3983dd7b67297db299b403b29b328d9e03e14c4c590ea90aa1ad1d7b35fb178b \
-    --hash=sha256:4100a3f8e8bbe47d499cdac00e56d5fe750f739701ea52dc049b6c56f5421d97 \
-    --hash=sha256:51abfaa7b6c66f3f18028876713c8804e73d4c2b6ceddbcbcfa8ec62429377f0 \
-    --hash=sha256:61c24f4554efdb8fb1ac6c8e75dab301bcdf2b7b739ed0c2b267493bb43163c5 \
-    --hash=sha256:700ccf921b2fdc9b23910e95b5caae4b35767685e0812343fa7172409f1b5830 \
-    --hash=sha256:7b41eb2e792822a790cb2a171df49d1a9e0baaa8e81f58077b7380a273b93d5f \
-    --hash=sha256:803987d3b16d55faa997bfc12e8b97f1091f145930dee229b020487aed8a1f44 \
-    --hash=sha256:99af5cfcd208c81998dcf44b3ca466dee7e17453cfb50e98b87947c3a86f8753 \
-    --hash=sha256:9faea1cca34501c7e139bc7ef8e504d532b77865c58592493e2c154a003b450f \
-    --hash=sha256:a7ba4c9a174db841cfee9a0b92563862a0301d7ca543334666c7266b541f141a \
-    --hash=sha256:b26071c2313d1880599c69fd831a07b32a8c961ba69d7ccbe5db1cd8d319a4ca \
-    --hash=sha256:b49dc8e1116abde13a3e6a9eb8da6ab292c5a3325155fb872e39011b110b37e6 \
-    --hash=sha256:bd40def0fd013569887008baa6da9ca428e3d7247adeeaeada153006227bb2e7 \
-    --hash=sha256:bfd0db770e8bd4e044e20298dcae6dfc42561f85d17ee546dcd978c8b23066ae \
-    --hash=sha256:c2fad1efae5889925c8fd7867fdd61f59480e4e0b510f9db096c912e884704f1 \
-    --hash=sha256:c81aea93d526ccf6bc0b842c91216ee9867cd8792f6725a00f19c8b5837e1715 \
-    --hash=sha256:da786e039b4ad2bce3d53d4799438cf1f5e01a0108f1b8d78ac08e6627281b1a \
-    --hash=sha256:deab85a069397540987082d251e9c89e0e5b2e3e044014344ff81f60e211fc4b \
-    --hash=sha256:e3f1e6224c3407beb1849bdc5ae3150929e593e4cffff6ca41c6ec2b10942c80 \
-    --hash=sha256:e74eb224e53aae3943d66e2d29fe42322d5753fd4c0641329bccb7efb3a46552 \
-    --hash=sha256:ee697c7ea65cb14915a64f36874da8ffc2123df43cf8bc952172e04a26656cd6 \
-    --hash=sha256:f37792b16d11606c28e428460bd6a3d14b8917b109e77cdbe4ca78b0b9a52c87 \
-    --hash=sha256:fd2906b54cbf879c09d875ad4e4687c58d87f5ed03496063fec1c9065569fd5d
-ecdsa==0.14.1 \
-    --hash=sha256:64c613005f13efec6541bb0a33290d0d03c27abab5f15fbab20fb0ee162bdd8e \
-    --hash=sha256:e108a5fe92c67639abae3260e43561af914e7fd0d27bae6d2ec1312ae7934dfe
-hidapi==0.7.99.post21 \
-    --hash=sha256:1ac170f4d601c340f2cd52fd06e85c5e77bad7ceac811a7bb54b529f7dc28c24 \
-    --hash=sha256:6424ad75da0021ce8c1bcd78056a04adada303eff3c561f8d132b85d0a914cb3 \
-    --hash=sha256:8d3be666f464347022e2b47caf9132287885d9eacc7895314fc8fefcb4e42946 \
-    --hash=sha256:92878bad7324dee619b7832fbfc60b5360d378aa7c5addbfef0a410d8fd342c7 \
-    --hash=sha256:b4b1f6aff0192e9be153fe07c1b7576cb7a1ff52e78e3f76d867be95301a8e87 \
-    --hash=sha256:bf03f06f586ce7d8aeb697a94b7dba12dc9271aae92d7a8d4486360ff711a660 \
-    --hash=sha256:c76de162937326fcd57aa399f94939ce726242323e65c15c67e183da1f6c26f7 \
-    --hash=sha256:d4ad1e46aef98783a9e6274d523b8b1e766acfc3d72828cd44a337564d984cfa \
-    --hash=sha256:d4b5787a04613503357606bb10e59c3e2c1114fa00ee328b838dd257f41cbd7b \
-    --hash=sha256:e0be1aa6566979266a8fc845ab0e18613f4918cf2c977fe67050f5dc7e2a9a97 \
-    --hash=sha256:edfb16b16a298717cf05b8c8a9ad1828b6ff3de5e93048ceccd74e6ae4ff0922
-idna==2.8 \
-    --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
-    --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c
+base58==2.0.1 \
+    --hash=sha256:365c9561d9babac1b5f18ee797508cd54937a724b6e419a130abad69cec5ca79 \
+    --hash=sha256:447adc750d6b642987ffc6d397ecd15a799852d5f6a1d308d384500243825058
+bitbox02==5.1.0 \
+    --hash=sha256:0562bc93d87afd89879e130c60c8dbfaffa8a1c3deff01201702939c9594d242 \
+    --hash=sha256:7d0efad2516604c0275452506f415730ac9e790569dedc79668b67db2ed13cdf
+btchip-python==0.1.31 \
+    --hash=sha256:4167f3c6ea832dd189d447d0d7a8c2a968027671ae6f43c680192f2b72c39b2c
+certifi==2020.12.5 \
+    --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \
+    --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830
+cffi==1.14.4 \
+    --hash=sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e \
+    --hash=sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d \
+    --hash=sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a \
+    --hash=sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec \
+    --hash=sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362 \
+    --hash=sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668 \
+    --hash=sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c \
+    --hash=sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b \
+    --hash=sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06 \
+    --hash=sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698 \
+    --hash=sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2 \
+    --hash=sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c \
+    --hash=sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7 \
+    --hash=sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009 \
+    --hash=sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03 \
+    --hash=sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b \
+    --hash=sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909 \
+    --hash=sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53 \
+    --hash=sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35 \
+    --hash=sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26 \
+    --hash=sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b \
+    --hash=sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01 \
+    --hash=sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb \
+    --hash=sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293 \
+    --hash=sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd \
+    --hash=sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d \
+    --hash=sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3 \
+    --hash=sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d \
+    --hash=sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e \
+    --hash=sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca \
+    --hash=sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d \
+    --hash=sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775 \
+    --hash=sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375 \
+    --hash=sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b \
+    --hash=sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b \
+    --hash=sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f
+chardet==4.0.0 \
+    --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \
+    --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5
+ckcc-protocol==1.0.2 \
+    --hash=sha256:2a34e1b2db2dc4f3e5503fac598e010370250dbb07224090eb475b3361f87ab3 \
+    --hash=sha256:31c01e4e460b949d6a570501996c54ee17f5ea25c1ec70b4e1535fe5631df67e
+click==7.1.2 \
+    --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \
+    --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc
+construct==2.10.56 \
+    --hash=sha256:97ba13edcd98546f10f7555af41c8ce7ae9d8221525ec4062c03f9adbf940661
+cryptography==3.3.1 \
+    --hash=sha256:0003a52a123602e1acee177dc90dd201f9bb1e73f24a070db7d36c588e8f5c7d \
+    --hash=sha256:0e85aaae861d0485eb5a79d33226dd6248d2a9f133b81532c8f5aae37de10ff7 \
+    --hash=sha256:594a1db4511bc4d960571536abe21b4e5c3003e8750ab8365fafce71c5d86901 \
+    --hash=sha256:69e836c9e5ff4373ce6d3ab311c1a2eed274793083858d3cd4c7d12ce20d5f9c \
+    --hash=sha256:788a3c9942df5e4371c199d10383f44a105d67d401fb4304178020142f020244 \
+    --hash=sha256:7e177e4bea2de937a584b13645cab32f25e3d96fc0bc4a4cf99c27dc77682be6 \
+    --hash=sha256:83d9d2dfec70364a74f4e7c70ad04d3ca2e6a08b703606993407bf46b97868c5 \
+    --hash=sha256:84ef7a0c10c24a7773163f917f1cb6b4444597efd505a8aed0a22e8c4780f27e \
+    --hash=sha256:9e21301f7a1e7c03dbea73e8602905a4ebba641547a462b26dd03451e5769e7c \
+    --hash=sha256:9f6b0492d111b43de5f70052e24c1f0951cb9e6022188ebcb1cc3a3d301469b0 \
+    --hash=sha256:a69bd3c68b98298f490e84519b954335154917eaab52cf582fa2c5c7efc6e812 \
+    --hash=sha256:b4890d5fb9b7a23e3bf8abf5a8a7da8e228f1e97dc96b30b95685df840b6914a \
+    --hash=sha256:c366df0401d1ec4e548bebe8f91d55ebcc0ec3137900d214dd7aac8427ef3030 \
+    --hash=sha256:dc42f645f8f3a489c3dd416730a514e7a91a59510ddaadc09d04224c098d3302
+ecdsa==0.16.1 \
+    --hash=sha256:881fa5e12bb992972d3d1b3d4dfbe149ab76a89f13da02daa5ea1ec7dea6e747 \
+    --hash=sha256:cfc046a2ddd425adbd1a78b3c46f0d1325c657811c0f45ecc3a0a6236c1e50ff
+hidapi==0.10.1 \
+    --hash=sha256:0c92b398f6907654b07f7dbd7e06661abe9ad6119b403eb5fd3c2af4ce66a3b7 \
+    --hash=sha256:310c53aa81697bf16b5f0c127afda36e5e9ea37794147afe1461422623263ef7 \
+    --hash=sha256:3b93d3f9bae38a3459491194ba1abf5c292b59dbd8738c3ac66f01b593cf3724 \
+    --hash=sha256:4bab0e8ab066527e09856a6a345e2e0c10061f2640e9281323da9a04b94bdec1 \
+    --hash=sha256:59f5205928dbe92513038c50dfb4f939395f8f781e176259a40f37d7a291313f \
+    --hash=sha256:a1170b18050bc57fae3840a51084e8252fd319c0fc6043d68c8501deb0e25846 \
+    --hash=sha256:b1becc9f09c85c473e91cf869b592d5d87fb8b89672988de33776b20b4c53ce1 \
+    --hash=sha256:b686b2b547890c8ed17ebeabded0050ce377180a56daefa20822b4d66d3a5dea \
+    --hash=sha256:f49a0de45217366b85597c2edb4be8bd61c9f26f533b854b058dded4352dd89d
+idna==2.10 \
+    --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \
+    --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0
 keepkey==6.3.1 \
     --hash=sha256:88e2b5291c85c8e8567732f675697b88241082884aa1aba32257f35ee722fc09 \
     --hash=sha256:cef1e862e195ece3e42640a0f57d15a63086fd1dedc8b5ddfcbc9c2657f0bb1e \
     --hash=sha256:f369d640c65fec7fd8e72546304cdc768c04224a6b9b00a19dc2cd06fa9d2a6b
-libusb1==1.7.1 \
-    --hash=sha256:adf64a4f3f5c94643a1286f8153bcf4bc787c348b38934aacd7fe17fbeebc571
+libusb1==1.9.1 \
+    --hash=sha256:16203d77a1f623b6f8f4e6c9d6bac79c1293b8d3e11de5f2f3c30d91380ae478 \
+    --hash=sha256:3905e907156f0a3fade75ddf82a777a6a901b245aa14500429275d221a1606c2 \
+    --hash=sha256:3a53d94add2799eaa1b412e7a5e384486c9109745217b9ac7f94101ad0f41b96 \
+    --hash=sha256:46708965226154681f8e0b14c48325c6d02e253c218e5d3aeff846ec274ceda8 \
+    --hash=sha256:4a024fffe195c49f3e7eadd2266087b4be065982f0cb41ef4b7e2c5053e7e65c \
+    --hash=sha256:b12666e8ad4df78e8f1bae36298c7d6f8f45d70ceea058b88631ef8478fd1eb0 \
+    --hash=sha256:d03ef15248c8b8ce440f6be4248eaadc074fc2dc5edd36c48e6e78eef3999292
 mnemonic==0.19 \
     --hash=sha256:4e37eb02b2cbd56a0079cabe58a6da93e60e3e4d6e757a586d9f23d96abea931 \
     --hash=sha256:a8d78c5100acfa7df9bab6b9db7390831b0e54490934b718ff9efd68f0d731a6
-pip==19.3.1 \
-    --hash=sha256:21207d76c1031e517668898a6b46a9fb1501c7a4710ef5dfd6a40ad9e6757ea7 \
-    --hash=sha256:6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7
-protobuf==3.11.1 \
-    --hash=sha256:0265379852b9e1f76af6d3d3fe4b3c383a595cc937594bda8565cf69a96baabd \
-    --hash=sha256:200b77e51f17fbc1d3049045f5835f60405dec3a00fe876b9b986592e46d908c \
-    --hash=sha256:29bd1ed46b2536ad8959401a2f02d2d7b5a309f8e97518e4f92ca6c5ba74dbed \
-    --hash=sha256:3175d45698edb9a07c1a78a1a4850e674ce8988f20596580158b1d0921d0f057 \
-    --hash=sha256:34a7270940f86da7a28be466ac541c89b6dbf144a6348b9cf7ac6f56b71006ce \
-    --hash=sha256:38cbc830a4a5ba9956763b0f37090bfd14dd74e72762be6225de2ceac55f4d03 \
-    --hash=sha256:665194f5ad386511ac8d8a0bd57b9ab37b8dd2cd71969458777318e774b9cd46 \
-    --hash=sha256:839bad7d115c77cdff29b488fae6a3ab503ce9a4192bd4c42302a6ea8e5d0f33 \
-    --hash=sha256:934a9869a7f3b0d84eca460e386fba1f7ba2a0c1a120a2648bc41fadf50efd1c \
-    --hash=sha256:aecdf12ef6dc7fd91713a6da93a86c2f2a8fe54840a3b1670853a2b7402e77c9 \
-    --hash=sha256:c4e90bc27c0691c76e09b5dc506133451e52caee1472b8b3c741b7c912ce43ef \
-    --hash=sha256:c65d135ea2d85d40309e268106dab02d3bea723db2db21c23ecad4163ced210b \
-    --hash=sha256:c98dea04a1ff41a70aff2489610f280004831798cb36a068013eed04c698903d \
-    --hash=sha256:d9049aa194378a426f0b2c784e2054565bf6f754d20fcafdee7102a6250556e8 \
-    --hash=sha256:e028fee51c96de4e81924484c77111dfdea14010ecfc906ea5b252209b0c4de6 \
-    --hash=sha256:e84ad26fb50091b1ea676403c0dd2bd47663099454aa6d88000b1dafecab0941 \
-    --hash=sha256:e88a924b591b06d0191620e9c8aa75297b3111066bb09d49a24bae1054a10c13
+noiseprotocol==0.3.1 \
+    --hash=sha256:2e1a603a38439636cf0ffd8b3e8b12cee27d368a28b41be7dbe568b2abb23111 \
+    --hash=sha256:b092a871b60f6a8f07f17950dc9f7098c8fe7d715b049bd4c24ee3752b90d645
+pip==20.1.1 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
+protobuf==3.14.0 \
+    --hash=sha256:0e247612fadda953047f53301a7b0407cb0c3cb4ae25a6fde661597a04039b3c \
+    --hash=sha256:0fc96785262042e4863b3f3b5c429d4636f10d90061e1840fce1baaf59b1a836 \
+    --hash=sha256:1c51fda1bbc9634246e7be6016d860be01747354ed7015ebe38acf4452f470d2 \
+    --hash=sha256:1d63eb389347293d8915fb47bee0951c7b5dab522a4a60118b9a18f33e21f8ce \
+    --hash=sha256:22bcd2e284b3b1d969c12e84dc9b9a71701ec82d8ce975fdda19712e1cfd4e00 \
+    --hash=sha256:2a7e2fe101a7ace75e9327b9c946d247749e564a267b0515cf41dfe450b69bac \
+    --hash=sha256:43b554b9e73a07ba84ed6cf25db0ff88b1e06be610b37656e292e3cbb5437472 \
+    --hash=sha256:4b74301b30513b1a7494d3055d95c714b560fbb630d8fb9956b6f27992c9f980 \
+    --hash=sha256:4e75105c9dfe13719b7293f75bd53033108f4ba03d44e71db0ec2a0e8401eafd \
+    --hash=sha256:5b7a637212cc9b2bcf85dd828b1178d19efdf74dbfe1ddf8cd1b8e01fdaaa7f5 \
+    --hash=sha256:5e9806a43232a1fa0c9cf5da8dc06f6910d53e4390be1fa06f06454d888a9142 \
+    --hash=sha256:629b03fd3caae7f815b0c66b41273f6b1900a579e2ccb41ef4493a4f5fb84f3a \
+    --hash=sha256:72230ed56f026dd664c21d73c5db73ebba50d924d7ba6b7c0d81a121e390406e \
+    --hash=sha256:86a75477addde4918e9a1904e5c6af8d7b691f2a3f65587d73b16100fbe4c3b2 \
+    --hash=sha256:8971c421dbd7aad930c9bd2694122f332350b6ccb5202a8b7b06f3f1a5c41ed5 \
+    --hash=sha256:9616f0b65a30851e62f1713336c931fcd32c057202b7ff2cfbfca0fc7d5e3043 \
+    --hash=sha256:b0d5d35faeb07e22a1ddf8dce620860c8fe145426c02d1a0ae2688c6e8ede36d \
+    --hash=sha256:ecc33531a213eee22ad60e0e2aaea6c8ba0021f0cce35dbf0ab03dee6e2a23a1
 pyaes==1.6.1 \
     --hash=sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f
-requests==2.22.0 \
-    --hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \
-    --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31
+pycparser==2.20 \
+    --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \
+    --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705
+requests==2.25.1 \
+    --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \
+    --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e
 safet==0.1.5 \
     --hash=sha256:a7fd4b68bb1bc6185298af665c8e8e00e2bb2bcbddbb22844ead929b845c635e \
     --hash=sha256:f966a23243312f64d14c7dfe02e8f13f6eeba4c3f51341f2c11ae57831f07de3
-setuptools==42.0.2 \
-    --hash=sha256:c5b372090d7c8709ce79a6a66872a91e518f7d65af97fca78135e1cb10d4b940 \
-    --hash=sha256:c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6
-six==1.13.0 \
-    --hash=sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd \
-    --hash=sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66
-trezor==0.11.5 \
-    --hash=sha256:711137bb83e7e0aef4009745e0da1b7d258146f246b43e3f7f5b849405088ef1 \
-    --hash=sha256:cd8aafd70a281daa644c4a3fb021ffac20b7a88e86226ecc8bb3e78e1734a184
-typing-extensions==3.7.4.1 \
-    --hash=sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2 \
-    --hash=sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d \
-    --hash=sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575
-urllib3==1.25.7 \
-    --hash=sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293 \
-    --hash=sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745
-wheel==0.33.6 \
-    --hash=sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646 \
-    --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28
+semver==2.13.0 \
+    --hash=sha256:ced8b23dceb22134307c1b8abfa523da14198793d9787ac838e70e29e77458d4 \
+    --hash=sha256:fa0fe2722ee1c3f57eac478820c3a5ae2f624af8264cbdf9000c980ff7f75e3f
+setuptools==46.4.0 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+six==1.15.0 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+trezor==0.12.2 \
+    --hash=sha256:5bd226b829e5f6ad6c7263f5303f58b54e07b0f21263c4b8ba57981881071264 \
+    --hash=sha256:b05d3042aaf12b77a86d603fa0e2b48120055c08ce6e9c85df3c2384d51194f1
+typing-extensions==3.7.4.3 \
+    --hash=sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918 \
+    --hash=sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c \
+    --hash=sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f
+urllib3==1.26.2 \
+    --hash=sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08 \
+    --hash=sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473
+wheel==0.34.2 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e

contrib/deterministic-build/requirements-wine-build.txt

@@ -1,19 +0,0 @@
-altgraph==0.16.1 \
-    --hash=sha256:d6814989f242b2b43025cba7161fc1b8fb487a62cd49c49245d6fd01c18ac997 \
-    --hash=sha256:ddf5320017147ba7b810198e0b6619bd7b5563aa034da388cea8546b877f9b0c
-future==0.18.2 \
-    --hash=sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d
-pefile==2019.4.18 \
-    --hash=sha256:a5d6e8305c6b210849b47a6174ddf9c452b2888340b8177874b862ba6c207645
-pip==19.3.1 \
-    --hash=sha256:21207d76c1031e517668898a6b46a9fb1501c7a4710ef5dfd6a40ad9e6757ea7 \
-    --hash=sha256:6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7
-pywin32-ctypes==0.2.0 \
-    --hash=sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942 \
-    --hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98
-setuptools==42.0.2 \
-    --hash=sha256:c5b372090d7c8709ce79a6a66872a91e518f7d65af97fca78135e1cb10d4b940 \
-    --hash=sha256:c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6
-wheel==0.33.6 \
-    --hash=sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646 \
-    --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28

contrib/deterministic-build/requirements.txt

@@ -1,183 +1,201 @@
-aiohttp==3.6.2 \
-    --hash=sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e \
-    --hash=sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326 \
-    --hash=sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a \
-    --hash=sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654 \
-    --hash=sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a \
-    --hash=sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4 \
-    --hash=sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17 \
-    --hash=sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec \
-    --hash=sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd \
-    --hash=sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48 \
-    --hash=sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59 \
-    --hash=sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965
-aiohttp-socks==0.2.2 \
-    --hash=sha256:e473ee222b001fe33798957b9ce3352b32c187cf41684f8e2259427925914993 \
-    --hash=sha256:eebd8939a7c3c1e3e7e1b2552c60039b4c65ef6b8b2351efcbdd98290538e310
-aiorpcX==0.18.4 \
-    --hash=sha256:bec9c0feb328d62ba80b79931b07f7372c98f2891ad51300be0b7163d5ccfb4a \
-    --hash=sha256:d424a55bcf52ebf1b3610a7809c0748fac91ce926854ad33ce952463bc6017e8
-apply-defaults==0.1.4 \
-    --hash=sha256:1ce26326a61d8773d38a9726a345c6525a91a6120d7333af79ad792dacb6246c
+aiohttp==3.7.3 \
+    --hash=sha256:0b795072bb1bf87b8620120a6373a3c61bfcb8da7e5c2377f4bb23ff4f0b62c9 \
+    --hash=sha256:0d438c8ca703b1b714e82ed5b7a4412c82577040dadff479c08405e2a715564f \
+    --hash=sha256:16a3cb5df5c56f696234ea9e65e227d1ebe9c18aa774d36ff42f532139066a5f \
+    --hash=sha256:1edfd82a98c5161497bbb111b2b70c0813102ad7e0aa81cbeb34e64c93863005 \
+    --hash=sha256:2406dc1dda01c7f6060ab586e4601f18affb7a6b965c50a8c90ff07569cf782a \
+    --hash=sha256:2858b2504c8697beb9357be01dc47ef86438cc1cb36ecb6991796d19475faa3e \
+    --hash=sha256:2a7b7640167ab536c3cb90cfc3977c7094f1c5890d7eeede8b273c175c3910fd \
+    --hash=sha256:3228b7a51e3ed533f5472f54f70fd0b0a64c48dc1649a0f0e809bec312934d7a \
+    --hash=sha256:328b552513d4f95b0a2eea4c8573e112866107227661834652a8984766aa7656 \
+    --hash=sha256:39f4b0a6ae22a1c567cb0630c30dd082481f95c13ca528dc501a7766b9c718c0 \
+    --hash=sha256:3b0036c978cbcc4a4512278e98e3e6d9e6b834dc973206162eddf98b586ef1c6 \
+    --hash=sha256:3ea8c252d8df5e9166bcf3d9edced2af132f4ead8ac422eac723c5781063709a \
+    --hash=sha256:41608c0acbe0899c852281978492f9ce2c6fbfaf60aff0cefc54a7c4516b822c \
+    --hash=sha256:59d11674964b74a81b149d4ceaff2b674b3b0e4d0f10f0be1533e49c4a28408b \
+    --hash=sha256:5e479df4b2d0f8f02133b7e4430098699450e1b2a826438af6bec9a400530957 \
+    --hash=sha256:684850fb1e3e55c9220aad007f8386d8e3e477c4ec9211ae54d968ecdca8c6f9 \
+    --hash=sha256:6ccc43d68b81c424e46192a778f97da94ee0630337c9bbe5b2ecc9b0c1c59001 \
+    --hash=sha256:6d42debaf55450643146fabe4b6817bb2a55b23698b0434107e892a43117285e \
+    --hash=sha256:710376bf67d8ff4500a31d0c207b8941ff4fba5de6890a701d71680474fe2a60 \
+    --hash=sha256:756ae7efddd68d4ea7d89c636b703e14a0c686688d42f588b90778a3c2fc0564 \
+    --hash=sha256:77149002d9386fae303a4a162e6bce75cc2161347ad2ba06c2f0182561875d45 \
+    --hash=sha256:78e2f18a82b88cbc37d22365cf8d2b879a492faedb3f2975adb4ed8dfe994d3a \
+    --hash=sha256:7d9b42127a6c0bdcc25c3dcf252bb3ddc70454fac593b1b6933ae091396deb13 \
+    --hash=sha256:8389d6044ee4e2037dca83e3f6994738550f6ee8cfb746762283fad9b932868f \
+    --hash=sha256:9c1a81af067e72261c9cbe33ea792893e83bc6aa987bfbd6fdc1e5e7b22777c4 \
+    --hash=sha256:c1e0920909d916d3375c7a1fdb0b1c78e46170e8bb42792312b6eb6676b2f87f \
+    --hash=sha256:c68fdf21c6f3573ae19c7ee65f9ff185649a060c9a06535e9c3a0ee0bbac9235 \
+    --hash=sha256:c733ef3bdcfe52a1a75564389bad4064352274036e7e234730526d155f04d914 \
+    --hash=sha256:c9c58b0b84055d8bc27b7df5a9d141df4ee6ff59821f922dd73155861282f6a3 \
+    --hash=sha256:d03abec50df423b026a5aa09656bd9d37f1e6a49271f123f31f9b8aed5dc3ea3 \
+    --hash=sha256:d2cfac21e31e841d60dc28c0ec7d4ec47a35c608cb8906435d47ef83ffb22150 \
+    --hash=sha256:dcc119db14757b0c7bce64042158307b9b1c76471e655751a61b57f5a0e4d78e \
+    --hash=sha256:df3a7b258cc230a65245167a202dd07320a5af05f3d41da1488ba0fa05bc9347 \
+    --hash=sha256:df48a623c58180874d7407b4d9ec06a19b84ed47f60a3884345b1a5099c1818b \
+    --hash=sha256:e1b95972a0ae3f248a899cdbac92ba2e01d731225f566569311043ce2226f5e7 \
+    --hash=sha256:f326b3c1bbfda5b9308252ee0dcb30b612ee92b0e105d4abec70335fab5b1245 \
+    --hash=sha256:f411cb22115cb15452d099fec0ee636b06cf81bfb40ed9c02d30c8dc2bc2e3d1
+aiohttp-socks==0.5.5 \
+    --hash=sha256:2eb2059756bde34c55bb429541cbf2eba3fd53e36ac80875b461221e2858b04a \
+    --hash=sha256:faaa25ed4dc34440ca888d23e089420f3b1918dc4ecf062c3fd9474827ad6a39
+aiorpcX==0.18.5 \
+    --hash=sha256:18eba632833b3ac75bbf7db67b32920129670b91919d7f54aeed35c813e8357a \
+    --hash=sha256:69d397babc1a6724770caa7f1394e5692a8aee19d5e769abb0c4a4566b837375
 async-timeout==3.0.1 \
     --hash=sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f \
     --hash=sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3
-attrs==19.3.0 \
-    --hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \
-    --hash=sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72
-bitstring==3.1.6 \
-    --hash=sha256:7b60b0c300d0d3d0a24ec84abfda4b0eaed3dc56dc90f6cbfe497166c9ad8443 \
-    --hash=sha256:c97a8e2a136e99b523b27da420736ae5cb68f83519d633794a6a11192f69f8bf \
-    --hash=sha256:e392819965e7e0246e3cf6a51d5a54e731890ae03ebbfa3cd0e4f74909072096
-certifi==2019.11.28 \
-    --hash=sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3 \
-    --hash=sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f
+attrs==20.3.0 \
+    --hash=sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6 \
+    --hash=sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700
+bitstring==3.1.7 \
+    --hash=sha256:fdf3eb72b229d2864fb507f8f42b1b2c57af7ce5fec035972f9566de440a864a
+certifi==2020.12.5 \
+    --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \
+    --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830
 chardet==3.0.4 \
     --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
     --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691
-click==6.7 \
-    --hash=sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d \
-    --hash=sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b
-dnspython==1.16.0 \
-    --hash=sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01 \
-    --hash=sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d
-ecdsa==0.14.1 \
-    --hash=sha256:64c613005f13efec6541bb0a33290d0d03c27abab5f15fbab20fb0ee162bdd8e \
-    --hash=sha256:e108a5fe92c67639abae3260e43561af914e7fd0d27bae6d2ec1312ae7934dfe
-idna==2.8 \
-    --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
-    --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c
+dnspython==2.0.0 \
+    --hash=sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7 \
+    --hash=sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d
+helpdev==0.7.1 \
+    --hash=sha256:779a761b18c2d96fb181aa699609f802347806125f2fee2f60dad875a625e38e \
+    --hash=sha256:bb62a79acbac141dadf42cadeb92bb7450dd18b9824a62043b6a0b149190db3d
+idna==3.1 \
+    --hash=sha256:5205d03e7bcbb919cc9c19885f9920d622ca52448306f2377daede5cf3faac16 \
+    --hash=sha256:c5b02147e01ea9920e6b0a3f1f7bb833612d507592c837a6c49552768f4054e1
 idna_ssl==1.1.0 \
     --hash=sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c
-importlib-metadata==1.1.0 \
-    --hash=sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21 \
-    --hash=sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742
-jsonrpcclient==3.3.4 \
-    --hash=sha256:c50860409b73af9f94b648439caae3b4af80d5ac937f2a8ac7783de3d1050ba9
-jsonrpcserver==4.0.5 \
-    --hash=sha256:240c517f49b0fdd3bfa428c9a7cc581126a0c43eca60d29762da124017d9d9f4
-jsonschema==3.2.0 \
-    --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163 \
-    --hash=sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a
-more-itertools==8.0.0 \
-    --hash=sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2 \
-    --hash=sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45
-multidict==4.6.1 \
-    --hash=sha256:07f9a6bf75ad675d53956b2c6a2d4ef2fa63132f33ecc99e9c24cf93beb0d10b \
-    --hash=sha256:0ffe4d4d28cbe9801952bfb52a8095dd9ffecebd93f84bdf973c76300de783c5 \
-    --hash=sha256:1b605272c558e4c659dbaf0fb32a53bfede44121bcf77b356e6e906867b958b7 \
-    --hash=sha256:205a011e636d885af6dd0029e41e3514a46e05bb2a43251a619a6e8348b96fc0 \
-    --hash=sha256:250632316295f2311e1ed43e6b26a63b0216b866b45c11441886ac1543ca96e1 \
-    --hash=sha256:2bc9c2579312c68a3552ee816311c8da76412e6f6a9cf33b15152e385a572d2a \
-    --hash=sha256:318aadf1cfb6741c555c7dd83d94f746dc95989f4f106b25b8a83dfb547f2756 \
-    --hash=sha256:42cdd649741a14b0602bf15985cad0dd4696a380081a3319cd1ead46fd0f0fab \
-    --hash=sha256:5159c4975931a1a78bf6602bbebaa366747fce0a56cb2111f44789d2c45e379f \
-    --hash=sha256:87e26d8b89127c25659e962c61a4c655ec7445d19150daea0759516884ecb8b4 \
-    --hash=sha256:891b7e142885e17a894d9d22b0349b92bb2da4769b4e675665d0331c08719be5 \
-    --hash=sha256:8d919034420378132d074bf89df148d0193e9780c9fe7c0e495e895b8af4d8a2 \
-    --hash=sha256:9c890978e2b37dd0dc1bd952da9a5d9f245d4807bee33e3517e4119c48d66f8c \
-    --hash=sha256:a37433ce8cdb35fc9e6e47e1606fa1bfd6d70440879038dca7d8dd023197eaa9 \
-    --hash=sha256:c626029841ada34c030b94a00c573a0c7575fe66489cde148785b6535397d675 \
-    --hash=sha256:cfec9d001a83dc73580143f3c77e898cf7ad78b27bb5e64dbe9652668fcafec7 \
-    --hash=sha256:efaf1b18ea6c1f577b1371c0159edbe4749558bfe983e13aa24d0a0c01e1ad7b
-pip==19.3.1 \
-    --hash=sha256:21207d76c1031e517668898a6b46a9fb1501c7a4710ef5dfd6a40ad9e6757ea7 \
-    --hash=sha256:6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7
-protobuf==3.11.1 \
-    --hash=sha256:0265379852b9e1f76af6d3d3fe4b3c383a595cc937594bda8565cf69a96baabd \
-    --hash=sha256:200b77e51f17fbc1d3049045f5835f60405dec3a00fe876b9b986592e46d908c \
-    --hash=sha256:29bd1ed46b2536ad8959401a2f02d2d7b5a309f8e97518e4f92ca6c5ba74dbed \
-    --hash=sha256:3175d45698edb9a07c1a78a1a4850e674ce8988f20596580158b1d0921d0f057 \
-    --hash=sha256:34a7270940f86da7a28be466ac541c89b6dbf144a6348b9cf7ac6f56b71006ce \
-    --hash=sha256:38cbc830a4a5ba9956763b0f37090bfd14dd74e72762be6225de2ceac55f4d03 \
-    --hash=sha256:665194f5ad386511ac8d8a0bd57b9ab37b8dd2cd71969458777318e774b9cd46 \
-    --hash=sha256:839bad7d115c77cdff29b488fae6a3ab503ce9a4192bd4c42302a6ea8e5d0f33 \
-    --hash=sha256:934a9869a7f3b0d84eca460e386fba1f7ba2a0c1a120a2648bc41fadf50efd1c \
-    --hash=sha256:aecdf12ef6dc7fd91713a6da93a86c2f2a8fe54840a3b1670853a2b7402e77c9 \
-    --hash=sha256:c4e90bc27c0691c76e09b5dc506133451e52caee1472b8b3c741b7c912ce43ef \
-    --hash=sha256:c65d135ea2d85d40309e268106dab02d3bea723db2db21c23ecad4163ced210b \
-    --hash=sha256:c98dea04a1ff41a70aff2489610f280004831798cb36a068013eed04c698903d \
-    --hash=sha256:d9049aa194378a426f0b2c784e2054565bf6f754d20fcafdee7102a6250556e8 \
-    --hash=sha256:e028fee51c96de4e81924484c77111dfdea14010ecfc906ea5b252209b0c4de6 \
-    --hash=sha256:e84ad26fb50091b1ea676403c0dd2bd47663099454aa6d88000b1dafecab0941 \
-    --hash=sha256:e88a924b591b06d0191620e9c8aa75297b3111066bb09d49a24bae1054a10c13
-pyaes==1.6.1 \
-    --hash=sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f
-pycryptodomex==3.9.4 \
-    --hash=sha256:0943b65fb41b7403a9def6214061fdd9ab9afd0bbc581e553c72eebe60bded36 \
-    --hash=sha256:0a1dbb5c4d975a4ea568fb7686550aa225d94023191fb0cca8747dc5b5d77857 \
-    --hash=sha256:0f43f1608518347fdcb9c8f443fa5cabedd33f94188b13e4196a3a7ba90d169c \
-    --hash=sha256:11ce5fec5990e34e3981ed14897ba601c83957b577d77d395f1f8f878a179f98 \
-    --hash=sha256:17a09e38fdc91e4857cf5a7ce82f3c0b229c3977490f2146513e366923fc256b \
-    --hash=sha256:22d970cee5c096b9123415e183ae03702b2cd4d3ba3f0ced25c4e1aba3967167 \
-    --hash=sha256:2a1793efcbae3a2264c5e0e492a2629eb10d895d6e5f17dbbd00eb8b489c6bda \
-    --hash=sha256:30a8a148a0fe482cec1aaf942bbd0ade56ec197c14fe058b2a94318c57e1f991 \
-    --hash=sha256:32fbbaf964c5184d3f3e349085b0536dd28184b02e2b014fc900f58bbc126339 \
-    --hash=sha256:347d67faee36d449dc9632da411cc318df52959079062627f1243001b10dc227 \
-    --hash=sha256:45f4b4e5461a041518baabc52340c249b60833aa84cea6377dc8016a2b33c666 \
-    --hash=sha256:4717daec0035034b002d31c42e55431c970e3e38a78211f43990e1b7eaf19e28 \
-    --hash=sha256:51a1ac9e7dda81da444fed8be558a60ec88dfc73b2aa4b0efa310e87acb75838 \
-    --hash=sha256:53e9dcc8f14783f6300b70da325a50ac1b0a3dbaee323bd9dc3f71d409c197a1 \
-    --hash=sha256:5519a2ed776e193688b7ddb61ab709303f6eb7d1237081e298283c72acc44271 \
-    --hash=sha256:583450e8e80a0885c453211ed2bd69ceea634d8c904f23ff8687f677fe810e95 \
-    --hash=sha256:60f862bd2a07133585a4fc2ce2b1a8ec24746b07ac44307d22ef2b767cb03435 \
-    --hash=sha256:612091f1d3c84e723bec7cb855cf77576e646045744794c9a3f75ba80737762f \
-    --hash=sha256:629a87b87c8203b8789ccefc7f2f2faecd2daaeb56bdd0b4e44cd89565f2db07 \
-    --hash=sha256:6e56ec4c8938fb388b6f250ddd5e21c15e8f25a76e0ad0e2abae9afee09e67b4 \
-    --hash=sha256:8e8092651844a11ec7fa534395f3dfe99256ce4edca06f128efc9d770d6e1dc1 \
-    --hash=sha256:8f5f260629876603e08f3ce95c8ccd9b6b83bf9a921c41409046796267f7adc5 \
-    --hash=sha256:9a6b74f38613f54c56bd759b411a352258f47489bbefd1d57c930a291498b35b \
-    --hash=sha256:a5a13ebb52c4cd065fb673d8c94f39f30823428a4de19e1f3f828b63a8882d1e \
-    --hash=sha256:a77ca778a476829876a3a70ae880073379160e4a465d057e3c4e1c79acdf1b8a \
-    --hash=sha256:a9f7be3d19f79429c2118fd61bc2ec4fa095e93b56fb3a5f3009822402c4380f \
-    --hash=sha256:dc15a467c4f9e4b43748ba2f97aea66f67812bfd581818284c47cadc81d4caec \
-    --hash=sha256:e13cdeea23059f7577c230fd580d2c8178e67ebe10e360041abe86c33c316f1c \
-    --hash=sha256:e45b85c8521bca6bdfaf57e4987743ade53e9f03529dd3adbc9524094c6d55c4 \
-    --hash=sha256:e87f17867b260f57c88487f943eb4d46c90532652bb37046e764842c3b66cbb1 \
-    --hash=sha256:ee40a5b156f6c1192bc3082e9d73d0479904433cdda83110546cd67f5a15a5be \
-    --hash=sha256:ef63ffde3b267043579af8830fc97fc3b9b8a526a24e3ba23af9989d4e9e689a
-pyrsistent==0.15.6 \
-    --hash=sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b
-QDarkStyle==2.6.8 \
-    --hash=sha256:037a54bf0aa5153f8055b65b8b36ac0d0f7648f2fd906c011a4da22eb0f582a2 \
-    --hash=sha256:fd1abae37d3a0a004089178da7c0b26ec5eb29f965b3e573853b8f280b614dea
+importlib-metadata==3.3.0 \
+    --hash=sha256:5c5a2720817414a6c41f0a49993908068243ae02c1635a228126519b509c8aed \
+    --hash=sha256:bf792d480abbd5eda85794e4afb09dd538393f7d6e6ffef6e9f03d2014cf9450
+multidict==5.1.0 \
+    --hash=sha256:018132dbd8688c7a69ad89c4a3f39ea2f9f33302ebe567a879da8f4ca73f0d0a \
+    --hash=sha256:051012ccee979b2b06be928a6150d237aec75dd6bf2d1eeeb190baf2b05abc93 \
+    --hash=sha256:05c20b68e512166fddba59a918773ba002fdd77800cad9f55b59790030bab632 \
+    --hash=sha256:07b42215124aedecc6083f1ce6b7e5ec5b50047afa701f3442054373a6deb656 \
+    --hash=sha256:0e3c84e6c67eba89c2dbcee08504ba8644ab4284863452450520dad8f1e89b79 \
+    --hash=sha256:0e929169f9c090dae0646a011c8b058e5e5fb391466016b39d21745b48817fd7 \
+    --hash=sha256:1ab820665e67373de5802acae069a6a05567ae234ddb129f31d290fc3d1aa56d \
+    --hash=sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5 \
+    --hash=sha256:2e68965192c4ea61fff1b81c14ff712fc7dc15d2bd120602e4a3494ea6584224 \
+    --hash=sha256:2f1a132f1c88724674271d636e6b7351477c27722f2ed789f719f9e3545a3d26 \
+    --hash=sha256:37e5438e1c78931df5d3c0c78ae049092877e5e9c02dd1ff5abb9cf27a5914ea \
+    --hash=sha256:3a041b76d13706b7fff23b9fc83117c7b8fe8d5fe9e6be45eee72b9baa75f348 \
+    --hash=sha256:3a4f32116f8f72ecf2a29dabfb27b23ab7cdc0ba807e8459e59a93a9be9506f6 \
+    --hash=sha256:46c73e09ad374a6d876c599f2328161bcd95e280f84d2060cf57991dec5cfe76 \
+    --hash=sha256:46dd362c2f045095c920162e9307de5ffd0a1bfbba0a6e990b344366f55a30c1 \
+    --hash=sha256:4b186eb7d6ae7c06eb4392411189469e6a820da81447f46c0072a41c748ab73f \
+    --hash=sha256:54fd1e83a184e19c598d5e70ba508196fd0bbdd676ce159feb412a4a6664f952 \
+    --hash=sha256:585fd452dd7782130d112f7ddf3473ffdd521414674c33876187e101b588738a \
+    --hash=sha256:5cf3443199b83ed9e955f511b5b241fd3ae004e3cb81c58ec10f4fe47c7dce37 \
+    --hash=sha256:6a4d5ce640e37b0efcc8441caeea8f43a06addace2335bd11151bc02d2ee31f9 \
+    --hash=sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359 \
+    --hash=sha256:806068d4f86cb06af37cd65821554f98240a19ce646d3cd24e1c33587f313eb8 \
+    --hash=sha256:830f57206cc96ed0ccf68304141fec9481a096c4d2e2831f311bde1c404401da \
+    --hash=sha256:929006d3c2d923788ba153ad0de8ed2e5ed39fdbe8e7be21e2f22ed06c6783d3 \
+    --hash=sha256:9436dc58c123f07b230383083855593550c4d301d2532045a17ccf6eca505f6d \
+    --hash=sha256:9dd6e9b1a913d096ac95d0399bd737e00f2af1e1594a787e00f7975778c8b2bf \
+    --hash=sha256:ace010325c787c378afd7f7c1ac66b26313b3344628652eacd149bdd23c68841 \
+    --hash=sha256:b47a43177a5e65b771b80db71e7be76c0ba23cc8aa73eeeb089ed5219cdbe27d \
+    --hash=sha256:b797515be8743b771aa868f83563f789bbd4b236659ba52243b735d80b29ed93 \
+    --hash=sha256:b7993704f1a4b204e71debe6095150d43b2ee6150fa4f44d6d966ec356a8d61f \
+    --hash=sha256:d5c65bdf4484872c4af3150aeebe101ba560dcfb34488d9a8ff8dbcd21079647 \
+    --hash=sha256:d81eddcb12d608cc08081fa88d046c78afb1bf8107e6feab5d43503fea74a635 \
+    --hash=sha256:dc862056f76443a0db4509116c5cd480fe1b6a2d45512a653f9a855cc0517456 \
+    --hash=sha256:ecc771ab628ea281517e24fd2c52e8f31c41e66652d07599ad8818abaad38cda \
+    --hash=sha256:f200755768dc19c6f4e2b672421e0ebb3dd54c38d5a4f262b872d8cfcc9e93b5 \
+    --hash=sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281 \
+    --hash=sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80
+pip==20.1.1 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
+protobuf==3.14.0 \
+    --hash=sha256:0e247612fadda953047f53301a7b0407cb0c3cb4ae25a6fde661597a04039b3c \
+    --hash=sha256:0fc96785262042e4863b3f3b5c429d4636f10d90061e1840fce1baaf59b1a836 \
+    --hash=sha256:1c51fda1bbc9634246e7be6016d860be01747354ed7015ebe38acf4452f470d2 \
+    --hash=sha256:1d63eb389347293d8915fb47bee0951c7b5dab522a4a60118b9a18f33e21f8ce \
+    --hash=sha256:22bcd2e284b3b1d969c12e84dc9b9a71701ec82d8ce975fdda19712e1cfd4e00 \
+    --hash=sha256:2a7e2fe101a7ace75e9327b9c946d247749e564a267b0515cf41dfe450b69bac \
+    --hash=sha256:43b554b9e73a07ba84ed6cf25db0ff88b1e06be610b37656e292e3cbb5437472 \
+    --hash=sha256:4b74301b30513b1a7494d3055d95c714b560fbb630d8fb9956b6f27992c9f980 \
+    --hash=sha256:4e75105c9dfe13719b7293f75bd53033108f4ba03d44e71db0ec2a0e8401eafd \
+    --hash=sha256:5b7a637212cc9b2bcf85dd828b1178d19efdf74dbfe1ddf8cd1b8e01fdaaa7f5 \
+    --hash=sha256:5e9806a43232a1fa0c9cf5da8dc06f6910d53e4390be1fa06f06454d888a9142 \
+    --hash=sha256:629b03fd3caae7f815b0c66b41273f6b1900a579e2ccb41ef4493a4f5fb84f3a \
+    --hash=sha256:72230ed56f026dd664c21d73c5db73ebba50d924d7ba6b7c0d81a121e390406e \
+    --hash=sha256:86a75477addde4918e9a1904e5c6af8d7b691f2a3f65587d73b16100fbe4c3b2 \
+    --hash=sha256:8971c421dbd7aad930c9bd2694122f332350b6ccb5202a8b7b06f3f1a5c41ed5 \
+    --hash=sha256:9616f0b65a30851e62f1713336c931fcd32c057202b7ff2cfbfca0fc7d5e3043 \
+    --hash=sha256:b0d5d35faeb07e22a1ddf8dce620860c8fe145426c02d1a0ae2688c6e8ede36d \
+    --hash=sha256:ecc33531a213eee22ad60e0e2aaea6c8ba0021f0cce35dbf0ab03dee6e2a23a1
+python-socks==1.1.2 \
+    --hash=sha256:4390882760ae60b14615f951aac3ef2e9eab45eb33ed8e7ed02d9b4dfb3b5640 \
+    --hash=sha256:fa7513c9293d95d90b1da9e10b84fa53afcb4c0f67e9c141d9f479cde2d8af1a
+QDarkStyle==2.8.1 \
+    --hash=sha256:7cead57817a8a1f38b48d76ef38986b6cc397d0315c0dd0431fcd06749556947 \
+    --hash=sha256:d53b0120bddd9e3efba9801731e22ef86ed798bb5fc6a802f5f7bb32dedf0321
 qrcode==6.1 \
     --hash=sha256:3996ee560fc39532910603704c82980ff6d4d5d629f9c3f25f34174ce8606cf5 \
     --hash=sha256:505253854f607f2abf4d16092c61d4e9d511a3b4392e60bff957a68592b04369
-setuptools==42.0.2 \
-    --hash=sha256:c5b372090d7c8709ce79a6a66872a91e518f7d65af97fca78135e1cb10d4b940 \
-    --hash=sha256:c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6
-six==1.13.0 \
-    --hash=sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd \
-    --hash=sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66
-typing-extensions==3.7.4.1 \
-    --hash=sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2 \
-    --hash=sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d \
-    --hash=sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575
-wheel==0.33.6 \
-    --hash=sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646 \
-    --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28
-yarl==1.4.1 \
-    --hash=sha256:031e8f56cf085d3b3df6b6bce756369ea7052b82d35ea07b6045f209c819e0e5 \
-    --hash=sha256:074958fe4578ef3a3d0bdaf96bbc25e4c4db82b7ff523594776fcf3d3f16c531 \
-    --hash=sha256:2db667ee21f620b446a54a793e467714fc5a446fcc82d93a47e8bde01d69afab \
-    --hash=sha256:326f2dbaaa17b858ae86f261ae73a266fd820a561fc5142cee9d0fc58448fbd7 \
-    --hash=sha256:32a3885f542f74d0f4f87057050c6b45529ebd79d0639f56582e741521575bfe \
-    --hash=sha256:56126ef061b913c3eefecace3404ca88917265d0550b8e32bbbeab29e5c830bf \
-    --hash=sha256:589ac1e82add13fbdedc04eb0a83400db728e5f1af2bd273392088ca90de7062 \
-    --hash=sha256:6076bce2ecc6ebf6c92919d77762f80f4c9c6ecc9c1fbaa16567ec59ad7d6f1d \
-    --hash=sha256:63be649c535d18ab6230efbc06a07f7779cd4336a687672defe70c025349a47b \
-    --hash=sha256:6642cbc92eaffa586180f669adc772f5c34977e9e849e93f33dc142351e98c9c \
-    --hash=sha256:6fa05a25f2280e78a514041d4609d39962e7d51525f2439db9ad7a2ae7aac163 \
-    --hash=sha256:7ed006a220422c33ff0889288be24db56ff0a3008ffe9eaead58a690715ad09b \
-    --hash=sha256:80c9c213803b50899460cc355f47e66778c3c868f448b7b7de5b1f1858c82c2a \
-    --hash=sha256:8bae18e2129850e76969b57869dacc72a66cccdbeebce1a28d7f3d439c21a7a3 \
-    --hash=sha256:ab112fba996a8f48f427e26969f2066d50080df0c24007a8cc6d7ae865e19013 \
-    --hash=sha256:b1c178ef813940c9a5cbad42ab7b8b76ac08b594b0a6bad91063c968e0466efc \
-    --hash=sha256:d6eff151c3b23a56a5e4f496805619bc3bdf4f749f63a7a95ad50e8267c17475
-zipp==0.6.0 \
-    --hash=sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e \
-    --hash=sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335
-colorama==0.4.1 \
-    --hash=sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d \
-    --hash=sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48
+QtPy==1.9.0 \
+    --hash=sha256:2db72c44b55d0fe1407be8fba35c838ad0d6d3bb81f23007886dc1fc0f459c8d \
+    --hash=sha256:fa0b8363b363e89b2a6f49eddc162a04c0699ae95e109a6be3bb145a913190ea
+setuptools==46.4.0 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+six==1.15.0 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+typing-extensions==3.7.4.3 \
+    --hash=sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918 \
+    --hash=sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c \
+    --hash=sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f
+wheel==0.34.2 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e
+yarl==1.6.3 \
+    --hash=sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e \
+    --hash=sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434 \
+    --hash=sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366 \
+    --hash=sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3 \
+    --hash=sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec \
+    --hash=sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959 \
+    --hash=sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e \
+    --hash=sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c \
+    --hash=sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6 \
+    --hash=sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a \
+    --hash=sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6 \
+    --hash=sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424 \
+    --hash=sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e \
+    --hash=sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f \
+    --hash=sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50 \
+    --hash=sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2 \
+    --hash=sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc \
+    --hash=sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4 \
+    --hash=sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970 \
+    --hash=sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10 \
+    --hash=sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0 \
+    --hash=sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406 \
+    --hash=sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896 \
+    --hash=sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643 \
+    --hash=sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721 \
+    --hash=sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478 \
+    --hash=sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724 \
+    --hash=sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e \
+    --hash=sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8 \
+    --hash=sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96 \
+    --hash=sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25 \
+    --hash=sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76 \
+    --hash=sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2 \
+    --hash=sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2 \
+    --hash=sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c \
+    --hash=sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a \
+    --hash=sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71
+zipp==3.4.0 \
+    --hash=sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108 \
+    --hash=sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb
+colorama==0.4.4 \
+    --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b \
+    --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2

contrib/freeze_packages.sh

@@ -6,33 +6,43 @@ set -e
 venv_dir=~/.electrum-venv
 contrib=$(dirname "$0")
 
-which virtualenv > /dev/null 2>&1 || { echo "Please install virtualenv" && exit 1; }
-python3 -m hashin -h > /dev/null 2>&1 || { python3 -m pip install hashin; }
-other_python=$(which python3)
+# note: we should not use a higher version of python than what the binaries bundle
+if [[ ! "$SYSTEM_PYTHON" ]] ; then
+    SYSTEM_PYTHON=$(which python3.6) || printf ""
+else
+    SYSTEM_PYTHON=$(which $SYSTEM_PYTHON) || printf ""
+fi
+if [[ ! "$SYSTEM_PYTHON" ]] ; then
+    echo "Please specify which python to use in \$SYSTEM_PYTHON" && exit 1;
+fi
 
-for i in '' '-hw' '-binaries' '-wine-build'; do
+which virtualenv > /dev/null 2>&1 || { echo "Please install virtualenv" && exit 1; }
+
+${SYSTEM_PYTHON} -m hashin -h > /dev/null 2>&1 || { ${SYSTEM_PYTHON} -m pip install hashin; }
+
+for i in '' '-hw' '-binaries' '-binaries-mac' '-build-wine' '-build-mac' '-build-sdist' '-build-appimage'; do
     rm -rf "$venv_dir"
-    virtualenv -p $(which python3) $venv_dir
+    virtualenv -p ${SYSTEM_PYTHON} $venv_dir
 
     source $venv_dir/bin/activate
 
-    echo "Installing $m dependencies"
+    echo "Installing dependencies... (requirements${i}.txt)"
 
-    python -m pip install -r $contrib/requirements/requirements${i}.txt --upgrade
+    python -m pip install -r "$contrib/requirements/requirements${i}.txt" --upgrade
 
     echo "OK."
 
     requirements=$(pip freeze --all)
-    restricted=$(echo $requirements | $other_python $contrib/deterministic-build/find_restricted_dependencies.py)
+    restricted=$(echo $requirements | ${SYSTEM_PYTHON} $contrib/deterministic-build/find_restricted_dependencies.py)
     requirements="$requirements $restricted"
 
-    echo "Generating package hashes..."
-    rm $contrib/deterministic-build/requirements${i}.txt
-    touch $contrib/deterministic-build/requirements${i}.txt
+    echo "Generating package hashes... (requirements${i}.txt)"
+    rm "$contrib/deterministic-build/requirements${i}.txt"
+    touch "$contrib/deterministic-build/requirements${i}.txt"
 
     for requirement in $requirements; do
         echo -e "\r  Hashing $requirement..."
-        $other_python -m hashin -r $contrib/deterministic-build/requirements${i}.txt ${requirement}
+        ${SYSTEM_PYTHON} -m hashin -r "$contrib/deterministic-build/requirements${i}.txt" "${requirement}"
     done
 
     echo "OK."

contrib/make_libsecp256k1.sh

@@ -1,6 +1,20 @@
 #!/bin/bash
 
-LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7"
+# This script was tested on Linux and MacOS hosts, where it can be used
+# to build native libsecp256k1 binaries.
+#
+# It can also be used to cross-compile to Windows:
+# $ sudo apt-get install mingw-w64
+# For a Windows x86 (32-bit) target, run:
+# $ GCC_TRIPLET_HOST="i686-w64-mingw32" ./contrib/make_libsecp256k1.sh
+# Or for a Windows x86_64 (64-bit) target, run:
+# $ GCC_TRIPLET_HOST="x86_64-w64-mingw32" ./contrib/make_libsecp256k1.sh
+#
+# To cross-compile to Linux x86:
+# sudo apt-get install gcc-multilib g++-multilib
+# $ AUTOCONF_FLAGS="--host=i686-linux-gnu CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32" ./contrib/make_libsecp256k1.sh
+
+LIBSECP_VERSION="dbd41db16a0e91b2566820898a3ab2d7dad4fe00"
 
 set -e
 
@@ -19,9 +33,13 @@ info "Building $pkgname..."
         git clone https://github.com/bitcoin-core/secp256k1.git
     fi
     cd secp256k1
+    if ! $(git cat-file -e ${LIBSECP_VERSION}) ; then
+        info "Could not find requested version $LIBSECP_VERSION in local clone; fetching..."
+        git fetch --all
+    fi
     git reset --hard
-    git clean -f -x -q
-    git checkout $LIBSECP_VERSION
+    git clean -dfxq
+    git checkout "${LIBSECP_VERSION}^{commit}"
 
     if ! [ -x configure ] ; then
         echo "libsecp256k1_la_LDFLAGS = -no-undefined" >> Makefile.am
@@ -35,8 +53,9 @@ info "Building $pkgname..."
             --enable-module-recovery \
             --enable-experimental \
             --enable-module-ecdh \
-            --disable-jni \
+            --disable-benchmark \
             --disable-tests \
+            --disable-exhaustive-tests \
             --disable-static \
             --enable-shared || fail "Could not configure $pkgname. Please make sure you have a C compiler installed and try again."
     fi

contrib/make_packages

@@ -6,5 +6,6 @@ test -n "$CONTRIB" -a -d "$CONTRIB" || exit
 rm "$CONTRIB"/../packages/ -r
 
 #Install pure python modules in electrum directory
-python3 -m pip install -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages
+python3 -m pip install --no-dependencies --no-binary :all: \
+    -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages
 

contrib/make_zbar.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+# This script can be used on Linux hosts to build native libzbar binaries.
+# sudo apt-get install pkg-config libx11-dev libx11-6 libv4l-dev libxv-dev libxext-dev libjpeg-dev
+#
+# It can also be used to cross-compile to Windows:
+# $ sudo apt-get install mingw-w64 mingw-w64-tools win-iconv-mingw-w64-dev
+# For a Windows x86 (32-bit) target, run:
+# $ GCC_TRIPLET_HOST="i686-w64-mingw32" BUILD_TYPE="wine" ./contrib/make_zbar.sh
+# Or for a Windows x86_64 (64-bit) target, run:
+# $ GCC_TRIPLET_HOST="x86_64-w64-mingw32" BUILD_TYPE="wine" ./contrib/make_zbar.sh
+
+ZBAR_VERSION="d2893738411be897a04caa42ffc13d1f6107d3c6"
+
+set -e
+
+. $(dirname "$0")/build_tools_util.sh || (echo "Could not source build_tools_util.sh" && exit 1)
+
+here=$(dirname $(realpath "$0" 2> /dev/null || grealpath "$0"))
+CONTRIB="$here"
+PROJECT_ROOT="$CONTRIB/.."
+
+pkgname="zbar"
+info "Building $pkgname..."
+
+(
+    cd $CONTRIB
+    if [ ! -d zbar ]; then
+        git clone https://github.com/mchehab/zbar.git
+    fi
+    cd zbar
+    if ! $(git cat-file -e ${ZBAR_VERSION}) ; then
+        info "Could not find requested version $ZBAR_VERSION in local clone; fetching..."
+        git fetch --all
+    fi
+    git reset --hard
+    git clean -dfxq
+    git checkout "${ZBAR_VERSION}^{commit}"
+
+    if [ "$BUILD_TYPE" = "wine" ] ; then
+        echo "libzbar_la_LDFLAGS += -Wc,-static" >> zbar/Makefile.am
+        echo "LDFLAGS += -Wc,-static" >> Makefile.am
+    fi
+    if ! [ -x configure ] ; then
+        autoreconf -vfi || fail "Could not run autoreconf for $pkgname. Please make sure you have automake and libtool installed, and try again."
+    fi
+    if ! [ -r config.status ] ; then
+        if [ "$BUILD_TYPE" = "wine" ] ; then
+            # windows target
+            ./configure \
+                $AUTOCONF_FLAGS \
+                --prefix="$here/$pkgname/dist" \
+                --with-x=no \
+                --enable-pthread=no \
+                --enable-doc=no \
+                --enable-video=yes \
+                --with-directshow=yes \
+                --with-jpeg=no \
+                --with-python=no \
+                --with-gtk=no \
+                --with-qt=no \
+                --with-java=no \
+                --with-imagemagick=no \
+                --with-dbus=no \
+                --enable-codes=qrcode \
+                --disable-dependency-tracking \
+                --disable-static \
+                --enable-shared || fail "Could not configure $pkgname. Please make sure you have a C compiler installed and try again."
+        else
+            # linux target
+            ./configure \
+                $AUTOCONF_FLAGS \
+                --prefix="$here/$pkgname/dist" \
+                --with-x=yes \
+                --enable-pthread=no \
+                --enable-doc=no \
+                --enable-video=yes \
+                --with-jpeg=yes \
+                --with-python=no \
+                --with-gtk=no \
+                --with-qt=no \
+                --with-java=no \
+                --with-imagemagick=no \
+                --with-dbus=no \
+                --enable-codes=qrcode \
+                --disable-static \
+                --enable-shared || fail "Could not configure $pkgname. Please make sure you have a C compiler installed and try again."
+        fi
+    fi
+    make -j4 || fail "Could not build $pkgname"
+    make install || fail "Could not install $pkgname"
+    . "$here/$pkgname/dist/lib/libzbar.la"
+    host_strip "$here/$pkgname/dist/lib/$dlname"
+    cp -fpv "$here/$pkgname/dist/lib/$dlname" "$PROJECT_ROOT/electrum" || fail "Could not copy the $pkgname binary to its destination"
+    info "$dlname has been placed in the inner 'electrum' folder."
+)

contrib/osx/README.md

@@ -1,5 +1,5 @@
-Building Mac OS binaries
-========================
+Building macOS binaries
+=======================
 
 ✗ _This script does not produce reproducible output (yet!).
    Please help us remedy this._
@@ -7,36 +7,47 @@ Building Mac OS binaries
 This guide explains how to build Electrum binaries for macOS systems.
 
 
-## 1. Building the binary
+## Building the binary
 
-This needs to be done on a system running macOS or OS X. We use El Capitan (10.11.6) as building it
-on High Sierra (or later)
-makes the binaries [incompatible with older versions](https://github.com/pyinstaller/pyinstaller/issues/1191).
+This needs to be done on a system running macOS or OS X.
 
-Another factor for the minimum supported macOS version is the
-[bundled Qt version](https://github.com/spesmilo/electrum/issues/3685).
+Notes about compatibility with different macOS versions:
+- In general the binary is not guaranteed to run on an older version of macOS
+  than what the build machine has. This is due to bundling the compiled Python into
+  the [PyInstaller binary](https://github.com/pyinstaller/pyinstaller/issues/1191).
+- The [bundled version of Qt](https://github.com/spesmilo/electrum/issues/3685) also
+  imposes a minimum supported macOS version.
+- If you want to build binaries that conform to the macOS "Gatekeeper", so as to
+  minimise the warnings users get, the binaries need to be codesigned with a
+  certificate issued by Apple, and starting with macOS 10.15 the binaries also
+  need to be notarized by Apple's central server. The catch is that to be able to build
+  binaries that Apple will notarise (due to the requirements on the binaries themselves,
+  e.g. hardened runtime) the build machine needs at least macOS 10.14.
+  See [#6128](https://github.com/spesmilo/electrum/issues/6128).
+
+We currently build the release binaries on macOS 10.14.6, and these seem to run on
+10.13 or newer.
 
 Before starting, make sure that the Xcode command line tools are installed (e.g. you have `git`).
 
-#### 1.1a Get Xcode
+#### 1.a Get Xcode
 
 Building the QR scanner (CalinsQRReader) requires full Xcode (not just command line tools).
 
-The last Xcode version compatible with El Capitan is Xcode 8.2.1
-
 Get it from [here](https://developer.apple.com/download/more/).
-
 Unfortunately, you need an "Apple ID" account.
 
+(note: the last Xcode that runs on macOS 10.14.6 is Xcode 11.3.1)
+
 After downloading, uncompress it.
 
 Make sure it is the "selected" xcode (e.g.):
 
     sudo xcode-select -s $HOME/Downloads/Xcode.app/Contents/Developer/
 
-#### 1.1b Build QR scanner separately on newer Mac
+#### 1.b Build QR scanner separately on another Mac
 
-Alternatively, you can try building just the QR scanner on newer macOS.
+Alternatively, you can try building just the QR scanner on another Mac.
 
 On newer Mac, run:
 
@@ -46,27 +57,17 @@ On newer Mac, run:
 Move `prebuilt_qr` to El Capitan: `contrib/osx/CalinsQRReader/prebuilt_qr`.
 
 
-#### 1.2 Build Electrum
+#### 2. Build Electrum
 
     cd electrum
     ./contrib/osx/make_osx
-    
+
 This creates both a folder named Electrum.app and the .dmg file.
 
+If you want the binaries codesigned for MacOS and notarised by Apple's central server,
+provide these env vars to the `make_osx` script:
 
-## 2. Building the image deterministically (WIP)
-The usual way to distribute macOS applications is to use image files containing the 
-application. Although these images can be created on a Mac with the built-in `hdiutil`,
-they are not deterministic.
-
-Instead, we use the toolchain that Bitcoin uses: genisoimage and libdmg-hfsplus.
-These tools do not work on macOS, so you need a separate Linux machine (or VM).
-
-Copy the Electrum.app directory over and install the dependencies, e.g.:
-
-    apt install libcap-dev cmake make gcc faketime
-    
-Then you can just invoke `package.sh` with the path to the app:
-
-    cd electrum
-    ./contrib/osx/package.sh ~/Electrum.app/
+    CODESIGN_CERT="Developer ID Application: Electrum Technologies GmbH (L6P37P7P56)" \
+    APPLE_ID_USER="me@email.com" \
+    APPLE_ID_PASSWORD="1234" \
+    ./contrib/osx/make_osx

contrib/osx/base.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-. $(dirname "$0")/../build_tools_util.sh
-
-
-function DoCodeSignMaybe { # ARGS: infoName fileOrDirName codesignIdentity
-    infoName="$1"
-    file="$2"
-    identity="$3"
-    deep=""
-    if [ -z "$identity" ]; then
-        # we are ok with them not passing anything; master script calls us unconditionally even if no identity is specified
-        return
-    fi
-    if [ -d "$file" ]; then
-        deep="--deep"
-    fi
-    if [ -z "$infoName" ] || [ -z "$file" ] || [ -z "$identity" ] || [ ! -e "$file" ]; then
-        fail "Argument error to internal function DoCodeSignMaybe()"
-    fi
-    info "Code signing ${infoName}..."
-    codesign -f -v $deep -s "$identity" "$file" || fail "Could not code sign ${infoName}"
-}

contrib/osx/entitlements.plist

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <!-- These are required for binaries built by PyInstaller -->
+    <!-- see pyinstaller/pyinstaller#4629 -->
+    <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+    <true/>
+    <key>com.apple.security.cs.disable-library-validation</key>
+    <true/>
+
+    <!-- These are required for USB HID access (hw wallets). -->
+    <!-- see https://github.com/Electron-Cash/Electron-Cash/commit/5abec73eee0cdeb725e3c5a989621ec4ccfb92a0 -->
+    <key>com.apple.security.cs.allow-dyld-environment-variables</key>
+    <true/>
+    <key>com.apple.security.cs.allow-jit</key>
+    <true/>
+
+    <!-- Camera access, to read QR codes -->
+    <key>com.apple.security.device.camera</key>
+    <true/>
+</dict>
+</plist>

contrib/osx/make_osx

@@ -1,15 +1,16 @@
 #!/usr/bin/env bash
 
 # Parameterize
-PYTHON_VERSION=3.7.6
+PYTHON_VERSION=3.7.9
 BUILDDIR=/tmp/electrum-build
 PACKAGE=Electrum
 GIT_REPO=https://github.com/spesmilo/electrum
-LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7"
 
 export GCC_STRIP_BINARIES="1"
 
-. $(dirname "$0")/base.sh
+
+. $(dirname "$0")/../build_tools_util.sh
+
 
 CONTRIB_OSX="$(dirname "$(realpath "$0")")"
 CONTRIB="$CONTRIB_OSX/.."
@@ -18,32 +19,51 @@ ROOT_FOLDER="$CONTRIB/.."
 src_dir=$(dirname "$0")
 cd $src_dir/../..
 
-VERSION=`git describe --tags --dirty --always`
 
 which brew > /dev/null 2>&1 || fail "Please install brew from https://brew.sh/ to continue"
 which xcodebuild > /dev/null 2>&1 || fail "Please install Xcode and xcode command line tools to continue"
 
 # Code Signing: See https://developer.apple.com/library/archive/documentation/Security/Conceptual/CodeSigningGuide/Procedures/Procedures.html
-APP_SIGN=""
-if [ -n "$1" ]; then
+if [ -n "$CODESIGN_CERT" ]; then
     # Test the identity is valid for signing by doing this hack. There is no other way to do this.
     cp -f /bin/ls ./CODESIGN_TEST
-    codesign -s "$1" --dryrun -f ./CODESIGN_TEST > /dev/null 2>&1
+    codesign -s "$CODESIGN_CERT" --dryrun -f ./CODESIGN_TEST > /dev/null 2>&1
     res=$?
     rm -f ./CODESIGN_TEST
     if ((res)); then
-        fail "Code signing identity \"$1\" appears to be invalid."
+        fail "Code signing identity \"$CODESIGN_CERT\" appears to be invalid."
     fi
     unset res
-    APP_SIGN="$1"
-    info "Code signing enabled using identity \"$APP_SIGN\""
+    info "Code signing enabled using identity \"$CODESIGN_CERT\""
 else
-    warn "Code signing DISABLED. Specify a valid macOS Developer identity installed on the system as the first argument to this script to enable signing."
+    warn "Code signing DISABLED. Specify a valid macOS Developer identity installed on the system to enable signing."
 fi
 
+
+function DoCodeSignMaybe { # ARGS: infoName fileOrDirName
+    infoName="$1"
+    file="$2"
+    deep=""
+    if [ -z "$CODESIGN_CERT" ]; then
+        # no cert -> we won't codesign
+        return
+    fi
+    if [ -d "$file" ]; then
+        deep="--deep"
+    fi
+    if [ -z "$infoName" ] || [ -z "$file" ] || [ ! -e "$file" ]; then
+        fail "Argument error to internal function DoCodeSignMaybe()"
+    fi
+    hardened_arg="--entitlements=${CONTRIB_OSX}/entitlements.plist -o runtime"
+
+    info "Code signing ${infoName}..."
+    codesign -f -v $deep -s "$CODESIGN_CERT" $hardened_arg "$file" || fail "Could not code sign ${infoName}"
+}
+
+
 info "Installing Python $PYTHON_VERSION"
 export PATH="~/.pyenv/bin:~/.pyenv/shims:~/Library/Python/3.7/bin:$PATH"
-if [ -d "~/.pyenv" ]; then
+if [ -d "${HOME}/.pyenv" ]; then
   pyenv update
 else
   curl -L https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash > /dev/null 2>&1
@@ -52,15 +72,18 @@ PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install -s $PYTHON_VERSION && \
 pyenv global $PYTHON_VERSION || \
 fail "Unable to use Python $PYTHON_VERSION"
 
+break_legacy_easy_install
 
-info "install dependencies specific to binaries"
-# note that this also installs pinned versions of both pip and setuptools
-python3 -m pip install --no-dependencies -Ir ./contrib/deterministic-build/requirements-binaries.txt --user \
-    || fail "Could not install pyinstaller"
+# create a fresh virtualenv
+# This helps to avoid older versions of pip-installed dependencies interfering with the build.
+VENV_DIR="$CONTRIB_OSX/build-venv"
+rm -rf "$VENV_DIR"
+python3 -m venv $VENV_DIR
+source $VENV_DIR/bin/activate
 
-
-info "Installing pyinstaller"
-python3 -m pip install -I --user pyinstaller==3.6 || fail "Could not install pyinstaller"
+info "Installing build dependencies"
+python3 -m pip install --no-dependencies --no-warn-script-location -Ir ./contrib/deterministic-build/requirements-build-mac.txt \
+    || fail "Could not install build dependencies"
 
 info "Using these versions for building $PACKAGE:"
 sw_vers
@@ -91,10 +114,10 @@ info "generating locale"
 
 
 info "Downloading libusb..."
-curl https://homebrew.bintray.com/bottles/libusb-1.0.22.el_capitan.bottle.tar.gz | \
+curl https://homebrew.bintray.com/bottles/libusb-1.0.23.high_sierra.bottle.tar.gz | \
 tar xz --directory $BUILDDIR
-cp $BUILDDIR/libusb/1.0.22/lib/libusb-1.0.dylib contrib/osx
-echo "82c368dfd4da017ceb32b12ca885576f325503428a4966cc09302cbd62702493  contrib/osx/libusb-1.0.dylib" | \
+cp $BUILDDIR/libusb/1.0.23/lib/libusb-1.0.dylib contrib/osx
+echo "caea266f3fc3982adc55d6cb8d9bad10f6e61f0c24ce5901aa1804618e08e14d  contrib/osx/libusb-1.0.dylib" | \
     shasum -a 256 -c || fail "libusb checksum mismatched"
 
 info "Preparing for building libsecp256k1"
@@ -109,19 +132,23 @@ rm -fr build
 # prefer building using xcode ourselves. otherwise fallback to prebuilt binary
 xcodebuild || cp -r prebuilt_qr build || fail "Could not build CalinsQRReader"
 popd
-DoCodeSignMaybe "CalinsQRReader.app" "${d}/build/Release/CalinsQRReader.app" "$APP_SIGN" # If APP_SIGN is empty will be a noop
+DoCodeSignMaybe "CalinsQRReader.app" "${d}/build/Release/CalinsQRReader.app"
 
 
 info "Installing requirements..."
-python3 -m pip install --no-dependencies -Ir ./contrib/deterministic-build/requirements.txt --user || \
-fail "Could not install requirements"
+python3 -m pip install --no-dependencies --no-warn-script-location -Ir ./contrib/deterministic-build/requirements.txt \
+    || fail "Could not install requirements"
 
 info "Installing hardware wallet requirements..."
-python3 -m pip install --no-dependencies -Ir ./contrib/deterministic-build/requirements-hw.txt --user || \
-fail "Could not install hardware wallet requirements"
+python3 -m pip install --no-dependencies --no-warn-script-location -Ir ./contrib/deterministic-build/requirements-hw.txt \
+    || fail "Could not install hardware wallet requirements"
+
+info "Installing dependencies specific to binaries..."
+python3 -m pip install --no-dependencies --no-warn-script-location -Ir ./contrib/deterministic-build/requirements-binaries-mac.txt \
+    || fail "Could not install dependencies specific to binaries"
 
 info "Building $PACKAGE..."
-python3 -m pip install --no-dependencies --user . > /dev/null || fail "Could not build $PACKAGE"
+python3 -m pip install --no-dependencies --no-warn-script-location . > /dev/null || fail "Could not build $PACKAGE"
 
 info "Faking timestamps..."
 for d in ~/Library/Python/ ~/.pyenv .; do
@@ -130,8 +157,10 @@ for d in ~/Library/Python/ ~/.pyenv .; do
   popd
 done
 
+VERSION=`git describe --tags --dirty --always`
+
 info "Building binary"
-APP_SIGN="$APP_SIGN" pyinstaller --noconfirm --ascii --clean --name $VERSION contrib/osx/osx.spec || fail "Could not build binary"
+APP_SIGN="$CODESIGN_CERT" pyinstaller --noconfirm --ascii --clean --name $VERSION contrib/osx/osx.spec || fail "Could not build binary"
 
 info "Adding bitcoin URI types to Info.plist"
 plutil -insert 'CFBundleURLTypes' \
@@ -139,14 +168,23 @@ plutil -insert 'CFBundleURLTypes' \
 	-- dist/$PACKAGE.app/Contents/Info.plist \
 	|| fail "Could not add keys to Info.plist. Make sure the program 'plutil' exists and is installed."
 
-DoCodeSignMaybe "app bundle" "dist/${PACKAGE}.app" "$APP_SIGN" # If APP_SIGN is empty will be a noop
+DoCodeSignMaybe "app bundle" "dist/${PACKAGE}.app"
+
+if [ ! -z "$CODESIGN_CERT" ]; then
+    if [ ! -z "$APPLE_ID_USER" ]; then
+        info "Notarizing .app with Apple's central server..."
+        "${CONTRIB_OSX}/notarize_app.sh" "dist/${PACKAGE}.app" || fail "Could not notarize binary."
+    else
+        warn "AppleID details not set! Skipping Apple notarization."
+    fi
+fi
 
 info "Creating .DMG"
 hdiutil create -fs HFS+ -volname $PACKAGE -srcfolder dist/$PACKAGE.app dist/electrum-$VERSION.dmg || fail "Could not create .DMG"
 
-DoCodeSignMaybe ".DMG" "dist/electrum-${VERSION}.dmg" "$APP_SIGN" # If APP_SIGN is empty will be a noop
+DoCodeSignMaybe ".DMG" "dist/electrum-${VERSION}.dmg"
 
-if [ -z "$APP_SIGN" ]; then
+if [ -z "$CODESIGN_CERT" ]; then
     warn "App was built successfully but was not code signed. Users may get security warnings from macOS."
-    warn "Specify a valid code signing identity as the first argument to this script to enable code signing."
+    warn "Specify a valid code signing identity to enable code signing."
 fi

contrib/osx/notarize_app.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# from https://github.com/metabrainz/picard/blob/e1354632d2db305b7a7624282701d34d73afa225/scripts/package/macos-notarize-app.sh
+
+
+if [ -z "$1" ]; then
+    echo "Specify app bundle as first parameter"
+    exit 1
+fi
+
+if [ -z "$APPLE_ID_USER" ] || [ -z "$APPLE_ID_PASSWORD" ]; then
+    echo "You need to set your Apple ID credentials with \$APPLE_ID_USER and \$APPLE_ID_PASSWORD."
+    exit 1
+fi
+
+APP_BUNDLE=$(basename "$1")
+APP_BUNDLE_DIR=$(dirname "$1")
+
+cd "$APP_BUNDLE_DIR" || exit 1
+
+# Package app for submission
+echo "Generating ZIP archive ${APP_BUNDLE}.zip..."
+ditto -c -k --rsrc --keepParent "$APP_BUNDLE" "${APP_BUNDLE}.zip"
+
+# Submit for notarization
+echo "Submitting $APP_BUNDLE for notarization..."
+RESULT=$(xcrun altool --notarize-app --type osx \
+  --file "${APP_BUNDLE}.zip" \
+  --primary-bundle-id org.electrum.electrum \
+  --username $APPLE_ID_USER \
+  --password @env:APPLE_ID_PASSWORD \
+  --output-format xml)
+
+if [ $? -ne 0 ]; then
+  echo "Submitting $APP_BUNDLE failed:"
+  echo "$RESULT"
+  exit 1
+fi
+
+REQUEST_UUID=$(echo "$RESULT" | xpath \
+  "//key[normalize-space(text()) = 'RequestUUID']/following-sibling::string[1]/text()" 2> /dev/null)
+
+if [ -z "$REQUEST_UUID" ]; then
+  echo "Submitting $APP_BUNDLE failed:"
+  echo "$RESULT"
+  exit 1
+fi
+
+echo "$(echo "$RESULT" | xpath \
+  "//key[normalize-space(text()) = 'success-message']/following-sibling::string[1]/text()" 2> /dev/null)"
+
+# Poll for notarization status
+echo "Submitted notarization request $REQUEST_UUID, waiting for response..."
+sleep 60
+while :
+do
+  RESULT=$(xcrun altool --notarization-info "$REQUEST_UUID" \
+    --username "$APPLE_ID_USER" \
+    --password @env:APPLE_ID_PASSWORD \
+    --output-format xml)
+  STATUS=$(echo "$RESULT" | xpath \
+    "//key[normalize-space(text()) = 'Status']/following-sibling::string[1]/text()" 2> /dev/null)
+
+  if [ "$STATUS" = "success" ]; then
+    echo "Notarization of $APP_BUNDLE succeeded!"
+    break
+  elif [ "$STATUS" = "in progress" ]; then
+    echo "Notarization in progress..."
+    sleep 20
+  else
+    echo "Notarization of $APP_BUNDLE failed:"
+    echo "$RESULT"
+    exit 1
+  fi
+done
+
+# Staple the notary ticket
+xcrun stapler staple "$APP_BUNDLE"

contrib/osx/osx.spec

@@ -59,16 +59,19 @@ block_cipher = None
 
 # see https://github.com/pyinstaller/pyinstaller/issues/2005
 hiddenimports = []
+hiddenimports += collect_submodules('pkg_resources')  # workaround for https://github.com/pypa/setuptools/issues/1963
 hiddenimports += collect_submodules('trezorlib')
 hiddenimports += collect_submodules('safetlib')
 hiddenimports += collect_submodules('btchip')
 hiddenimports += collect_submodules('keepkeylib')
 hiddenimports += collect_submodules('websocket')
 hiddenimports += collect_submodules('ckcc')
+hiddenimports += collect_submodules('bitbox02')
 hiddenimports += ['PyQt5.QtPrintSupport']  # needed by Revealer
 
 datas = [
     (electrum + PYPKG + '/*.json', PYPKG),
+    (electrum + PYPKG + '/lnwire/*.csv', PYPKG + '/lnwire'),
     (electrum + PYPKG + '/wordlist/english.txt', PYPKG + '/wordlist'),
     (electrum + PYPKG + '/locale', PYPKG + '/locale'),
     (electrum + PYPKG + '/plugins', PYPKG + '/plugins'),
@@ -79,8 +82,7 @@ datas += collect_data_files('safetlib')
 datas += collect_data_files('btchip')
 datas += collect_data_files('keepkeylib')
 datas += collect_data_files('ckcc')
-datas += collect_data_files('jsonrpcserver')
-datas += collect_data_files('jsonrpcclient')
+datas += collect_data_files('bitbox02')
 
 # Add the QR Scanner helper app
 datas += [(electrum + "contrib/osx/CalinsQRReader/build/Release/CalinsQRReader.app", "./contrib/osx/CalinsQRReader/build/Release/CalinsQRReader.app")]
@@ -137,24 +139,29 @@ if APP_SIGN:
 
 pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
 
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.datas,
-          name=PACKAGE,
-          debug=False,
-          strip=False,
-          upx=True,
-          icon=electrum+ICONS_FILE,
-          console=False)
-
-app = BUNDLE(exe,
-             version = VERSION,
-             name=PACKAGE + '.app',
-             icon=electrum+ICONS_FILE,
-             bundle_identifier=None,
-             info_plist={
-                'NSHighResolutionCapable': 'True',
-                'NSSupportsAutomaticGraphicsSwitching': 'True'
-             }
+exe = EXE(
+    pyz,
+    a.scripts,
+    exclude_binaries=True,
+    name=MAIN_SCRIPT,
+    debug=False,
+    strip=False,
+    upx=True,
+    icon=electrum+ICONS_FILE,
+    console=False,
+)
+
+app = BUNDLE(
+    exe,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    version = VERSION,
+    name=PACKAGE + '.app',
+    icon=electrum+ICONS_FILE,
+    bundle_identifier=None,
+    info_plist={
+        'NSHighResolutionCapable': 'True',
+        'NSSupportsAutomaticGraphicsSwitching': 'True'
+    },
 )

contrib/requirements/requirements-binaries-mac.txt

@@ -0,0 +1,2 @@
+PyQt5>=5.15.2
+cryptography>=2.6

contrib/requirements/requirements-binaries.txt

@@ -1,2 +1,5 @@
-PyQt5<5.12
-PyQt5-sip<=4.19.13
+PyQt5
+
+# we need at least cryptography>=2.1 for electrum.crypto,
+# and at least cryptography>=2.6 for dnspython[DNSSEC]
+cryptography>=2.6

contrib/requirements/requirements-build-appimage.txt

@@ -0,0 +1,10 @@
+pip
+setuptools
+wheel
+
+# Note: hidapi requires Cython at build-time (not needed at runtime).
+# For reproducible builds, the version of Cython must be pinned down.
+# The pinned Cython must be installed before hidapi is built;
+# otherwise when installing hidapi, pip just downloads the latest Cython.
+# see https://github.com/spesmilo/electrum/issues/5859
+Cython>=0.27

contrib/requirements/requirements-build-mac.txt

@@ -0,0 +1,15 @@
+pip
+setuptools<50.0.0   # 50.0.0 might break pyinstaller. see https://github.com/pyinstaller/pyinstaller/commit/e9f9d79d6b23c767512156323d0a5d28c6386a57
+wheel
+
+pyinstaller>=3.6
+
+# needed by pyinstaller:
+macholib
+
+# Note: hidapi requires Cython at build-time (not needed at runtime).
+# For reproducible builds, the version of Cython must be pinned down.
+# The pinned Cython must be installed before hidapi is built;
+# otherwise when installing hidapi, pip just downloads the latest Cython.
+# see https://github.com/spesmilo/electrum/issues/5859
+Cython>=0.27

contrib/requirements/requirements-build-sdist.txt

@@ -0,0 +1,4 @@
+# need modern versions of pip (and maybe other build tools), the one in apt had issues
+pip
+setuptools
+wheel

contrib/requirements/requirements-build-wine.txt

@@ -0,0 +1,9 @@
+pip
+setuptools<50.0.0   # 50.0.0 might break pyinstaller. see https://github.com/pyinstaller/pyinstaller/commit/e9f9d79d6b23c767512156323d0a5d28c6386a57
+wheel
+
+# needed by pyinstaller:
+pefile>=2017.8.1
+altgraph
+pywin32-ctypes>=0.2.0
+pyinstaller-hooks-contrib>=2020.6

contrib/requirements/requirements-hw.txt

@@ -1,16 +1,7 @@
-# Note: hidapi requires Cython as a build-time dependency (it is not needed at runtime).
-# For reproducible builds, the version of Cython must be pinned down.
-# Further, the pinned Cython must be installed before hidapi is built;
-# otherwise hidapi just downloads the latest Cython. To enforce order,
-# Cython must be listed before hidapi. Notably this also applies to
-# deterministic-build/requirements-hw.txt where items are lexicographically sorted.
-# Hence, we rely on "Cython" preceding "hidapi" lexicographically... :/
-# see https://github.com/spesmilo/electrum/issues/5859
-Cython>=0.27
-
-trezor[hidapi]>=0.11.5
+hidapi
+trezor[hidapi]>=0.12.0
 safet>=0.1.5
 keepkey>=6.3.1
-btchip-python>=0.1.26
+btchip-python>=0.1.30
 ckcc-protocol>=0.7.7
-hidapi
+bitbox02>=5.0.0

contrib/requirements/requirements-travis.txt

@@ -1,3 +1,3 @@
 tox
-python-coveralls
+coveralls
 tox-travis

contrib/requirements/requirements-wine-build.txt

@@ -1,7 +0,0 @@
-pip
-setuptools
-
-# needed by pyinstaller:
-pefile>=2017.8.1
-altgraph
-pywin32-ctypes>=0.2.0

contrib/requirements/requirements.txt

@@ -1,15 +1,13 @@
-pyaes>=0.1a1
-ecdsa>=0.14
 qrcode
-protobuf
-dnspython
-qdarkstyle<2.7
+protobuf>=3.12
+qdarkstyle<2.9
 aiorpcx>=0.18,<0.19
 aiohttp>=3.3.0,<4.0.0
-aiohttp_socks
+aiohttp_socks>=0.3
 certifi
 bitstring
-pycryptodomex>=3.7
-jsonrpcserver
-jsonrpcclient
-attrs
+attrs>=19.2.0
+
+# Note that we also need the dnspython[DNSSEC] extra which pulls in cryptography,
+# but as that is not pure-python it cannot be listed in this file!
+dnspython>=2.0,<2.1

contrib/sign_packages

@@ -1,5 +1,4 @@
-#!/usr/bin/python2
-
+#!/usr/bin/env python3
 import os
 import getpass
 

contrib/sign_version

@@ -1,4 +1,4 @@
 #!/bin/bash
 version=`python3 -c "import electrum; print(electrum.version.ELECTRUM_VERSION)"`
-sig=`./run_electrum -w $SIGNING_WALLET signmessage $SIGNING_ADDRESS $version`
+sig=`./run_electrum -o signmessage $SIGNING_ADDRESS $version -w $SIGNING_WALLET`
 echo "{ \"version\":\"$version\", \"signatures\":{ \"$SIGNING_ADDRESS\":\"$sig\"}}"

contrib/udev/53-hid-bitbox02.rules

@@ -0,0 +1 @@
+SUBSYSTEM=="usb", TAG+="uaccess", TAG+="udev-acl", SYMLINK+="bitbox02_%n", ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="2403"

contrib/udev/54-hid-bitbox02.rules

@@ -0,0 +1 @@
+KERNEL=="hidraw*", SUBSYSTEM=="hidraw", ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="2403", TAG+="uaccess", TAG+="udev-acl", SYMLINK+="bitbox02-%n"

contrib/udev/README.md

@@ -6,7 +6,8 @@ These are necessary for the devices to be usable on Linux environments.
 
  - `20-hw1.rules` (Ledger): https://github.com/LedgerHQ/udev-rules/blob/master/20-hw1.rules
  - `51-coinkite.rules` (Coldcard): https://github.com/Coldcard/ckcc-protocol/blob/master/51-coinkite.rules
- - `51-hid-digitalbitbox.rules`, `52-hid-digitalbitbox.rules` (Digital Bitbox): https://shiftcrypto.ch/start_linux
+ - `51-hid-digitalbitbox.rules`, `52-hid-digitalbitbox.rules` (Digital Bitbox): https://github.com/digitalbitbox/bitbox-wallet-app/blob/master/frontends/qt/resources/deb-afterinstall.sh
+ - `53-hid-bitbox02.rules`, `54-hid-bitbox02.rules` (BitBox02): https://github.com/digitalbitbox/bitbox-wallet-app/blob/master/frontends/qt/resources/deb-afterinstall.sh
  - `51-trezor.rules` (Trezor): https://github.com/trezor/trezor-common/blob/master/udev/51-trezor.rules
  - `51-usb-keepkey.rules` (Keepkey): https://github.com/keepkey/udev-rules/blob/master/51-usb-keepkey.rules
  - `51-safe-t.rules` (Archos): https://github.com/archos-safe-t/safe-t-common/blob/master/udev/51-safe-t.rules

electrum-env

@@ -4,24 +4,36 @@
 # python dependencies before activating the env and running Electrum.
 # If 'env' already exists, it is activated and Electrum is started
 # without any installations. Additionally, the PYTHONPATH environment
-# variable is set properly before running Electrum.
+# variable is set so that system packages such as e.g. apt installed
+# PyQt5 will also be visible.
 #
-# python-qt and its dependencies will still need to be installed with
-# your package manager.
+# By default, only pure python dependencies are installed.
+# If you would like more extras to be installed, do e.g.:
+# $ source ./env/bin/activate
+# $ pip install -e '.[crypto,gui,hardware]'
+# $ deactivate
+
+set -e
 
 PYTHON_VER="$(python3 -c 'import sys; print(sys.version[:3])')"
 
 cd $(dirname $0)
 if [ -e ./env/bin/activate ]; then
     source ./env/bin/activate
+    # FIXME what if this is an old directory and our requirements
+    #       changed in the meantime? should run "pip install -e . --upgrade"
 else
-    virtualenv env -p `which python3`
+    python3 -m venv env
     source ./env/bin/activate
-    python3 -m pip install .[fast]
+    pip install -e .
 fi
 
-export PYTHONPATH="/usr/local/lib/python${PYTHON_VER}/site-packages:$PYTHONPATH"
+export PYTHONPATH="$PYTHONPATH:"\
+"/usr/local/lib/python${PYTHON_VER}/site-packages:"\
+"/usr/local/lib/python${PYTHON_VER}/dist-packages:"\
+"/usr/lib/python3/dist-packages:"\
+"/usr/lib/python${PYTHON_VER}/site-packages:"\
+"${HOME}/.local/lib/python${PYTHON_VER}/site-packages"
+
 
 ./run_electrum "$@"
-
-deactivate

electrum/address_synchronizer.py

@@ -28,9 +28,9 @@ import itertools
 from collections import defaultdict
 from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple, NamedTuple, Sequence, List
 
-from . import bitcoin
+from . import bitcoin, util
 from .bitcoin import COINBASE_MATURITY
-from .util import profiler, bfh, TxMinedInfo
+from .util import profiler, bfh, TxMinedInfo, UnrelatedTransactionException
 from .transaction import Transaction, TxOutput, TxInput, PartialTxInput, TxOutpoint, PartialTransaction
 from .synchronizer import Synchronizer
 from .verifier import SPV
@@ -48,21 +48,21 @@ TX_HEIGHT_LOCAL = -2
 TX_HEIGHT_UNCONF_PARENT = -1
 TX_HEIGHT_UNCONFIRMED = 0
 
-class AddTransactionException(Exception):
-    pass
-
-
-class UnrelatedTransactionException(AddTransactionException):
-    def __str__(self):
-        return _("Transaction is unrelated to this wallet.")
-
 
 class HistoryItem(NamedTuple):
     txid: str
     tx_mined_status: TxMinedInfo
-    delta: Optional[int]
+    delta: int
+    fee: Optional[int]
+    balance: int
+
+
+class TxWalletDelta(NamedTuple):
+    is_relevant: bool  # "related to wallet?"
+    is_any_input_ismine: bool
+    is_all_input_ismine: bool
+    delta: int
     fee: Optional[int]
-    balance: Optional[int]
 
 
 class AddressSynchronizer(Logger):
@@ -70,13 +70,17 @@ class AddressSynchronizer(Logger):
     inherited by wallet
     """
 
+    network: Optional['Network']
+    synchronizer: Optional['Synchronizer']
+    verifier: Optional['SPV']
+
     def __init__(self, db: 'WalletDB'):
         self.db = db
-        self.network = None  # type: Network
+        self.network = None
         Logger.__init__(self)
         # verifier (SPV) and synchronizer are started in start_network
-        self.synchronizer = None  # type: Synchronizer
-        self.verifier = None  # type: SPV
+        self.synchronizer = None
+        self.verifier = None
         # locks: if you need to take multiple ones, acquire them in the order they are defined here!
         self.lock = threading.RLock()
         self.transaction_lock = threading.RLock()
@@ -92,8 +96,14 @@ class AddressSynchronizer(Logger):
 
         self.load_and_cleanup()
 
+    def with_lock(func):
+        def func_wrapper(self: 'AddressSynchronizer', *args, **kwargs):
+            with self.lock:
+                return func(self, *args, **kwargs)
+        return func_wrapper
+
     def with_transaction_lock(func):
-        def func_wrapper(self, *args, **kwargs):
+        def func_wrapper(self: 'AddressSynchronizer', *args, **kwargs):
             with self.transaction_lock:
                 return func(self, *args, **kwargs)
         return func_wrapper
@@ -139,10 +149,31 @@ class AddressSynchronizer(Logger):
         prevout_hash = txin.prevout.txid.hex()
         prevout_n = txin.prevout.out_idx
         for addr in self.db.get_txo_addresses(prevout_hash):
-            l = self.db.get_txo_addr(prevout_hash, addr)
-            for n, v, is_cb in l:
-                if n == prevout_n:
-                    return addr
+            d = self.db.get_txo_addr(prevout_hash, addr)
+            if prevout_n in d:
+                return addr
+        tx = self.db.get_transaction(prevout_hash)
+        if tx:
+            return tx.outputs()[prevout_n].address
+        return None
+
+    def get_txin_value(self, txin: TxInput, *, address: str = None) -> Optional[int]:
+        if txin.value_sats() is not None:
+            return txin.value_sats()
+        prevout_hash = txin.prevout.txid.hex()
+        prevout_n = txin.prevout.out_idx
+        if address is None:
+            address = self.get_txin_address(txin)
+        if address:
+            d = self.db.get_txo_addr(prevout_hash, address)
+            try:
+                v, cb = d[prevout_n]
+                return v
+            except KeyError:
+                pass
+        tx = self.db.get_transaction(prevout_hash)
+        if tx:
+            return tx.outputs()[prevout_n].value
         return None
 
     def get_txout_address(self, txo: TxOutput) -> Optional[str]:
@@ -156,17 +187,17 @@ class AddressSynchronizer(Logger):
                 # add it in case it was previously unconfirmed
                 self.add_unverified_tx(tx_hash, tx_height)
 
-    def start_network(self, network):
+    def start_network(self, network: Optional['Network']) -> None:
         self.network = network
         if self.network is not None:
             self.synchronizer = Synchronizer(self)
             self.verifier = SPV(self.network, self)
-            self.network.register_callback(self.on_blockchain_updated, ['blockchain_updated'])
+            util.register_callback(self.on_blockchain_updated, ['blockchain_updated'])
 
     def on_blockchain_updated(self, event, *args):
         self._get_addr_balance_cache = {}  # invalidate cache
 
-    def stop_threads(self):
+    def stop(self):
         if self.network:
             if self.synchronizer:
                 asyncio.run_coroutine_threadsafe(self.synchronizer.stop(), self.network.asyncio_loop)
@@ -174,7 +205,7 @@ class AddressSynchronizer(Logger):
             if self.verifier:
                 asyncio.run_coroutine_threadsafe(self.verifier.stop(), self.network.asyncio_loop)
                 self.verifier = None
-            self.network.unregister_callback(self.on_blockchain_updated)
+            util.unregister_callback(self.on_blockchain_updated)
             self.db.put('stored_height', self.get_local_height())
 
     def add_address(self, address):
@@ -269,16 +300,17 @@ class AddressSynchronizer(Logger):
                     self.remove_transaction(tx_hash2)
             # add inputs
             def add_value_from_prev_output():
-                # note: this nested loop takes linear time in num is_mine outputs of prev_tx
-                for addr in self.db.get_txo_addresses(prevout_hash):
+                # note: this takes linear time in num is_mine outputs of prev_tx
+                addr = self.get_txin_address(txi)
+                if addr and self.is_mine(addr):
                     outputs = self.db.get_txo_addr(prevout_hash, addr)
-                    # note: instead of [(n, v, is_cb), ...]; we could store: {n -> (v, is_cb)}
-                    for n, v, is_cb in outputs:
-                        if n == prevout_n:
-                            if addr and self.is_mine(addr):
-                                self.db.add_txi_addr(tx_hash, addr, ser, v)
-                                self._get_addr_balance_cache.pop(addr, None)  # invalidate cache
-                            return
+                    try:
+                        v, is_cb = outputs[prevout_n]
+                    except KeyError:
+                        pass
+                    else:
+                        self.db.add_txi_addr(tx_hash, addr, ser, v)
+                        self._get_addr_balance_cache.pop(addr, None)  # invalidate cache
             for txi in tx.inputs():
                 if txi.is_coinbase_input():
                     continue
@@ -345,7 +377,7 @@ class AddressSynchronizer(Logger):
                     prevout = TxOutpoint(bfh(tx_hash), idx)
                     self.db.remove_prevout_by_scripthash(scripthash, prevout=prevout, value=txo.value)
 
-    def get_depending_transactions(self, tx_hash):
+    def get_depending_transactions(self, tx_hash: str) -> Set[str]:
         """Returns all (grand-)children of tx_hash in this wallet."""
         with self.transaction_lock:
             children = set()
@@ -416,6 +448,7 @@ class AddressSynchronizer(Logger):
         with self.lock:
             with self.transaction_lock:
                 self.db.clear_history()
+                self._history_local.clear()
 
     def get_txpos(self, tx_hash):
         """Returns (height, txpos) tuple, even if the tx is unverified."""
@@ -441,6 +474,8 @@ class AddressSynchronizer(Logger):
                 self.threadlocal_cache.local_height = orig_val
         return f
 
+    @with_lock
+    @with_transaction_lock
     @with_local_height_cached
     def get_history(self, *, domain=None) -> Sequence[HistoryItem]:
         # get domain
@@ -449,15 +484,11 @@ class AddressSynchronizer(Logger):
         domain = set(domain)
         # 1. Get the history of each address in the domain, maintain the
         #    delta of a tx as the sum of its deltas on domain addresses
-        tx_deltas = defaultdict(int)
+        tx_deltas = defaultdict(int)  # type: Dict[str, int]
         for addr in domain:
             h = self.get_address_history(addr)
             for tx_hash, height in h:
-                delta = self.get_tx_delta(tx_hash, addr)
-                if delta is None or tx_deltas[tx_hash] is None:
-                    tx_deltas[tx_hash] = None
-                else:
-                    tx_deltas[tx_hash] += delta
+                tx_deltas[tx_hash] += self.get_tx_delta(tx_hash, addr)
         # 2. create sorted history
         history = []
         for tx_hash in tx_deltas:
@@ -476,15 +507,11 @@ class AddressSynchronizer(Logger):
                                   delta=delta,
                                   fee=fee,
                                   balance=balance))
-            if balance is None or delta is None:
-                balance = None
-            else:
-                balance -= delta
+            balance -= delta
         h2.reverse()
-        # fixme: this may happen if history is incomplete
-        if balance not in [None, 0]:
-            self.logger.warning("history not synchronized")
-            return []
+
+        if balance != 0:
+            raise Exception("wallet.get_history() failed balance sanity-check")
 
         return h2
 
@@ -546,7 +573,7 @@ class AddressSynchronizer(Logger):
             self.unverified_tx.pop(tx_hash, None)
             self.db.add_verified_tx(tx_hash, info)
         tx_mined_status = self.get_tx_height(tx_hash)
-        self.network.trigger_callback('verified', self, tx_hash, tx_mined_status)
+        util.trigger_callback('verified', self, tx_hash, tx_mined_status)
 
     def get_unverified_txs(self):
         '''Returns a map from tx hash to transaction height'''
@@ -584,7 +611,7 @@ class AddressSynchronizer(Logger):
             return cached_local_height
         return self.network.get_local_height() if self.network else self.db.get('stored_height', 0)
 
-    def add_future_tx(self, tx: Transaction, num_blocks: int) -> None:
+    def add_future_tx(self, tx: Transaction, num_blocks: int) -> bool:
         assert num_blocks > 0, num_blocks
         with self.lock:
             tx_was_added = self.add_transaction(tx)
@@ -593,6 +620,8 @@ class AddressSynchronizer(Logger):
             return tx_was_added
 
     def get_tx_height(self, tx_hash: str) -> TxMinedInfo:
+        if tx_hash is None:  # ugly backwards compat...
+            return TxMinedInfo(height=TX_HEIGHT_LOCAL, conf=0)
         with self.lock:
             verified_tx_mined_info = self.db.get_verified_tx(tx_hash)
             if verified_tx_mined_info:
@@ -611,9 +640,12 @@ class AddressSynchronizer(Logger):
 
     def set_up_to_date(self, up_to_date):
         with self.lock:
+            status_changed = self.up_to_date != up_to_date
             self.up_to_date = up_to_date
         if self.network:
             self.network.notify('status')
+        if status_changed:
+            self.logger.info(f'set_up_to_date: {up_to_date}')
 
     def is_up_to_date(self):
         with self.lock: return self.up_to_date
@@ -625,86 +657,56 @@ class AddressSynchronizer(Logger):
             return 0, 0
 
     @with_transaction_lock
-    def get_tx_delta(self, tx_hash, address):
+    def get_tx_delta(self, tx_hash: str, address: str) -> int:
         """effect of tx on address"""
         delta = 0
-        # substract the value of coins sent from address
+        # subtract the value of coins sent from address
         d = self.db.get_txi_addr(tx_hash, address)
         for n, v in d:
             delta -= v
         # add the value of the coins received at address
         d = self.db.get_txo_addr(tx_hash, address)
-        for n, v, cb in d:
+        for n, (v, cb) in d.items():
             delta += v
         return delta
 
-    @with_transaction_lock
-    def get_tx_value(self, txid):
-        """effect of tx on the entire domain"""
-        delta = 0
-        for addr in self.db.get_txi_addresses(txid):
-            d = self.db.get_txi_addr(txid, addr)
-            for n, v in d:
-                delta -= v
-        for addr in self.db.get_txo_addresses(txid):
-            d = self.db.get_txo_addr(txid, addr)
-            for n, v, cb in d:
-                delta += v
-        return delta
-
-    def get_wallet_delta(self, tx: Transaction):
-        """ effect of tx on wallet """
+    def get_wallet_delta(self, tx: Transaction) -> TxWalletDelta:
+        """effect of tx on wallet"""
         is_relevant = False  # "related to wallet?"
-        is_mine = False
-        is_pruned = False
-        is_partial = False
-        v_in = v_out = v_out_mine = 0
-        for txin in tx.inputs():
-            addr = self.get_txin_address(txin)
-            if self.is_mine(addr):
-                is_mine = True
-                is_relevant = True
-                d = self.db.get_txo_addr(txin.prevout.txid.hex(), addr)
-                for n, v, cb in d:
-                    if n == txin.prevout.out_idx:
-                        value = v
-                        break
-                else:
-                    value = None
+        num_input_ismine = 0
+        v_in = v_in_mine = v_out = v_out_mine = 0
+        with self.lock, self.transaction_lock:
+            for txin in tx.inputs():
+                addr = self.get_txin_address(txin)
+                value = self.get_txin_value(txin, address=addr)
+                if self.is_mine(addr):
+                    num_input_ismine += 1
+                    is_relevant = True
+                    assert value is not None
+                    v_in_mine += value
                 if value is None:
-                    value = txin.value_sats()
-                if value is None:
-                    is_pruned = True
-                else:
+                    v_in = None
+                elif v_in is not None:
                     v_in += value
-            else:
-                is_partial = True
-        if not is_mine:
-            is_partial = False
-        for o in tx.outputs():
-            v_out += o.value
-            if self.is_mine(o.address):
-                v_out_mine += o.value
-                is_relevant = True
-        if is_pruned:
-            # some inputs are mine:
-            fee = None
-            if is_mine:
-                v = v_out_mine - v_out
-            else:
-                # no input is mine
-                v = v_out_mine
+            for txout in tx.outputs():
+                v_out += txout.value
+                if self.is_mine(txout.address):
+                    v_out_mine += txout.value
+                    is_relevant = True
+        delta = v_out_mine - v_in_mine
+        if v_in is not None:
+            fee = v_in - v_out
         else:
-            v = v_out_mine - v_in
-            if is_partial:
-                # some inputs are mine, but not all
-                fee = None
-            else:
-                # all inputs are mine
-                fee = v_in - v_out
-        if not is_mine:
             fee = None
-        return is_relevant, is_mine, v, fee
+        if fee is None and isinstance(tx, PartialTransaction):
+            fee = tx.get_fee()
+        return TxWalletDelta(
+            is_relevant=is_relevant,
+            is_any_input_ismine=num_input_ismine > 0,
+            is_all_input_ismine=num_input_ismine == len(tx.inputs()),
+            delta=delta,
+            fee=fee,
+        )
 
     def get_tx_fee(self, txid: str) -> Optional[int]:
         """ Returns tx_fee or None. Use server fee only if tx is unconfirmed and not mine"""
@@ -731,8 +733,7 @@ class AddressSynchronizer(Logger):
         tx = self.db.get_transaction(txid)
         if not tx:
             return None
-        with self.lock, self.transaction_lock:
-            is_relevant, is_mine, v, fee = self.get_wallet_delta(tx)
+        fee = self.get_wallet_delta(tx).fee
         # save result
         self.db.add_tx_fee_we_calculated(txid, fee)
         self.db.add_num_inputs_to_tx(txid, len(tx.inputs()))
@@ -744,8 +745,8 @@ class AddressSynchronizer(Logger):
             received = {}
             sent = {}
             for tx_hash, height in h:
-                l = self.db.get_txo_addr(tx_hash, address)
-                for n, v, is_cb in l:
+                d = self.db.get_txo_addr(tx_hash, address)
+                for n, (v, is_cb) in d.items():
                     received[tx_hash + ':%d'%n] = (height, v, is_cb)
             for tx_hash, height in h:
                 l = self.db.get_txi_addr(tx_hash, address)
@@ -753,29 +754,35 @@ class AddressSynchronizer(Logger):
                     sent[txi] = height
         return received, sent
 
-    def get_addr_utxo(self, address: str) -> Dict[TxOutpoint, PartialTxInput]:
+
+    def get_addr_outputs(self, address: str) -> Dict[TxOutpoint, PartialTxInput]:
         coins, spent = self.get_addr_io(address)
-        for txi in spent:
-            coins.pop(txi)
         out = {}
         for prevout_str, v in coins.items():
             tx_height, value, is_cb = v
             prevout = TxOutpoint.from_str(prevout_str)
-            utxo = PartialTxInput(prevout=prevout,
-                                  is_coinbase_output=is_cb)
+            utxo = PartialTxInput(prevout=prevout, is_coinbase_output=is_cb)
             utxo._trusted_address = address
             utxo._trusted_value_sats = value
             utxo.block_height = tx_height
+            utxo.spent_height = spent.get(prevout_str, None)
             out[prevout] = utxo
         return out
 
+    def get_addr_utxo(self, address: str) -> Dict[TxOutpoint, PartialTxInput]:
+        out = self.get_addr_outputs(address)
+        for k, v in list(out.items()):
+            if v.spent_height is not None:
+                out.pop(k)
+        return out
+
     # return the total amount ever received by an address
     def get_addr_received(self, address):
         received, sent = self.get_addr_io(address)
         return sum([v for height, v, is_cb in received.values()])
 
     @with_local_height_cached
-    def get_addr_balance(self, address, *, excluded_coins: Set[str] = None):
+    def get_addr_balance(self, address, *, excluded_coins: Set[str] = None) -> Tuple[int, int, int]:
         """Return the balance of a bitcoin address:
         confirmed and matured, unconfirmed, unmatured
         """

electrum/base_crash_reporter.py

@@ -75,7 +75,7 @@ class BaseCrashReporter(Logger):
 
     async def do_post(self, proxy, url, data):
         async with make_aiohttp_session(proxy) as session:
-            async with session.post(url, data=data) as resp:
+            async with session.post(url, data=data, raise_for_status=True) as resp:
                 return await resp.text()
 
     def get_traceback_info(self):
@@ -121,15 +121,18 @@ class BaseCrashReporter(Logger):
             ['git', 'describe', '--always', '--dirty'], cwd=dir)
         return str(version, "utf8").strip()
 
+    def _get_traceback_str(self) -> str:
+        return "".join(traceback.format_exception(*self.exc_args))
+
     def get_report_string(self):
         info = self.get_additional_info()
-        info["traceback"] = "".join(traceback.format_exception(*self.exc_args))
+        info["traceback"] = self._get_traceback_str()
         return self.issue_template.format(**info)
 
     def get_user_description(self):
         raise NotImplementedError
 
-    def get_wallet_type(self):
+    def get_wallet_type(self) -> str:
         raise NotImplementedError
 
 

electrum/base_wizard.py

@@ -28,20 +28,19 @@ import sys
 import copy
 import traceback
 from functools import partial
-from typing import List, TYPE_CHECKING, Tuple, NamedTuple, Any, Dict, Optional
+from typing import List, TYPE_CHECKING, Tuple, NamedTuple, Any, Dict, Optional, Union
 
 from . import bitcoin
 from . import keystore
 from . import mnemonic
 from .bip32 import is_bip32_derivation, xpub_type, normalize_bip32_derivation, BIP32Node
-from .keystore import bip44_derivation, purpose48_derivation, Hardware_KeyStore, KeyStore
+from .keystore import bip44_derivation, purpose48_derivation, Hardware_KeyStore, KeyStore, bip39_to_seed
 from .wallet import (Imported_Wallet, Standard_Wallet, Multisig_Wallet,
                      wallet_types, Wallet, Abstract_Wallet)
-from .storage import (WalletStorage, StorageEncryptionVersion,
-                      get_derivation_used_for_hw_device_encryption)
+from .storage import WalletStorage, StorageEncryptionVersion
 from .wallet_db import WalletDB
 from .i18n import _
-from .util import UserCancelled, InvalidPassword, WalletFileException
+from .util import UserCancelled, InvalidPassword, WalletFileException, UserFacingException
 from .simple_config import SimpleConfig
 from .plugin import Plugins, HardwarePluginLibraryUnavailable
 from .logging import Logger
@@ -61,6 +60,12 @@ class ScriptTypeNotSupported(Exception): pass
 class GoBack(Exception): pass
 
 
+class ReRunDialog(Exception): pass
+
+
+class ChooseHwDeviceAgain(Exception): pass
+
+
 class WizardStackItem(NamedTuple):
     action: Any
     args: Any
@@ -114,18 +119,21 @@ class BaseWizard(Logger):
     def can_go_back(self):
         return len(self._stack) > 1
 
-    def go_back(self):
+    def go_back(self, *, rerun_previous: bool = True) -> None:
         if not self.can_go_back():
             return
         # pop 'current' frame
         self._stack.pop()
-        # pop 'previous' frame
-        stack_item = self._stack.pop()
+        prev_frame = self._stack[-1]
         # try to undo side effects since we last entered 'previous' frame
-        # FIXME only self.storage is properly restored
-        self.data = copy.deepcopy(stack_item.db_data)
-        # rerun 'previous' frame
-        self.run(stack_item.action, *stack_item.args, **stack_item.kwargs)
+        # FIXME only self.data is properly restored
+        self.data = copy.deepcopy(prev_frame.db_data)
+
+        if rerun_previous:
+            # pop 'previous' frame
+            self._stack.pop()
+            # rerun 'previous' frame
+            self.run(prev_frame.action, *prev_frame.args, **prev_frame.kwargs)
 
     def reset_stack(self):
         self._stack = []
@@ -145,7 +153,7 @@ class BaseWizard(Logger):
         self.choice_dialog(title=title, message=message, choices=choices, run_next=self.on_wallet_type)
 
     def upgrade_db(self, storage, db):
-        exc = None
+        exc = None  # type: Optional[Exception]
         def on_finished():
             if exc is None:
                 self.terminate(storage=storage, db=db)
@@ -159,6 +167,13 @@ class BaseWizard(Logger):
                 exc = e
         self.waiting_dialog(do_upgrade, _('Upgrading wallet format...'), on_finished=on_finished)
 
+    def run_task_without_blocking_gui(self, task, *, msg: str = None) -> Any:
+        """Perform a task in a thread without blocking the GUI.
+        Returns the result of 'task', or raises the same exception.
+        This method blocks until 'task' is finished.
+        """
+        raise NotImplementedError()
+
     def load_2fa(self):
         self.data['wallet_type'] = '2fa'
         self.data['use_trustedcoin'] = True
@@ -234,7 +249,7 @@ class BaseWizard(Logger):
                 self.data['addresses'][addr] = {'type':txin_type, 'pubkey':pubkey}
             self.keystores.append(k)
         else:
-            return self.terminate()
+            return self.terminate(aborted=True)
         return self.run('create_wallet')
 
     def restore_from_key(self):
@@ -254,7 +269,16 @@ class BaseWizard(Logger):
         k = keystore.from_master_key(text)
         self.on_keystore(k)
 
-    def choose_hw_device(self, purpose=HWD_SETUP_NEW_WALLET, *, storage=None):
+    def choose_hw_device(self, purpose=HWD_SETUP_NEW_WALLET, *, storage: WalletStorage = None):
+        while True:
+            try:
+                self._choose_hw_device(purpose=purpose, storage=storage)
+            except ChooseHwDeviceAgain:
+                pass
+            else:
+                break
+
+    def _choose_hw_device(self, *, purpose, storage: WalletStorage = None):
         title = _('Hardware Keystore')
         # check available plugins
         supported_plugins = self.plugins.get_hardware_support()
@@ -271,7 +295,8 @@ class BaseWizard(Logger):
 
         # scan devices
         try:
-            scanned_devices = devmgr.scan_devices()
+            scanned_devices = self.run_task_without_blocking_gui(task=devmgr.scan_devices,
+                                                                 msg=_("Scanning devices..."))
         except BaseException as e:
             self.logger.info('error scanning devices: {}'.format(repr(e)))
             debug_msg = '  {}:\n    {}'.format(_('Error scanning devices'), e)
@@ -317,8 +342,8 @@ class BaseWizard(Logger):
             msg += '\n\n'
             msg += _('Debug message') + '\n' + debug_msg
             self.confirm_dialog(title=title, message=msg,
-                                run_next=lambda x: self.choose_hw_device(purpose, storage=storage))
-            return
+                                run_next=lambda x: None)
+            raise ChooseHwDeviceAgain()
         # select device
         self.devices = devices
         choices = []
@@ -327,68 +352,65 @@ class BaseWizard(Logger):
             label = info.label or _("An unnamed {}").format(name)
             try: transport_str = info.device.transport_ui_string[:20]
             except: transport_str = 'unknown transport'
-            descr = f"{label} [{name}, {state}, {transport_str}]"
+            descr = f"{label} [{info.model_name or name}, {state}, {transport_str}]"
             choices.append(((name, info), descr))
         msg = _('Select a device') + ':'
         self.choice_dialog(title=title, message=msg, choices=choices,
                            run_next=lambda *args: self.on_device(*args, purpose=purpose, storage=storage))
 
-    def on_device(self, name, device_info, *, purpose, storage=None):
-        self.plugin = self.plugins.get_plugin(name)  # type: HW_PluginBase
+    def on_device(self, name, device_info: 'DeviceInfo', *, purpose, storage: WalletStorage = None):
+        self.plugin = self.plugins.get_plugin(name)
+        assert isinstance(self.plugin, HW_PluginBase)
+        devmgr = self.plugins.device_manager
         try:
-            self.plugin.setup_device(device_info, self, purpose)
+            client = self.plugin.setup_device(device_info, self, purpose)
         except OSError as e:
             self.show_error(_('We encountered an error while connecting to your device:')
                             + '\n' + str(e) + '\n'
                             + _('To try to fix this, we will now re-pair with your device.') + '\n'
                             + _('Please try again.'))
-            devmgr = self.plugins.device_manager
             devmgr.unpair_id(device_info.device.id_)
-            self.choose_hw_device(purpose, storage=storage)
-            return
+            raise ChooseHwDeviceAgain()
         except OutdatedHwFirmwareException as e:
             if self.question(e.text_ignore_old_fw_and_continue(), title=_("Outdated device firmware")):
                 self.plugin.set_ignore_outdated_fw()
                 # will need to re-pair
-                devmgr = self.plugins.device_manager
                 devmgr.unpair_id(device_info.device.id_)
-            self.choose_hw_device(purpose, storage=storage)
-            return
+            raise ChooseHwDeviceAgain()
         except (UserCancelled, GoBack):
-            self.choose_hw_device(purpose, storage=storage)
-            return
+            raise ChooseHwDeviceAgain()
+        except UserFacingException as e:
+            self.show_error(str(e))
+            raise ChooseHwDeviceAgain()
         except BaseException as e:
             self.logger.exception('')
             self.show_error(str(e))
-            self.choose_hw_device(purpose, storage=storage)
-            return
+            raise ChooseHwDeviceAgain()
+
         if purpose == HWD_SETUP_NEW_WALLET:
             def f(derivation, script_type):
                 derivation = normalize_bip32_derivation(derivation)
                 self.run('on_hw_derivation', name, device_info, derivation, script_type)
             self.derivation_and_script_type_dialog(f)
         elif purpose == HWD_SETUP_DECRYPT_WALLET:
-            derivation = get_derivation_used_for_hw_device_encryption()
-            xpub = self.plugin.get_xpub(device_info.device.id_, derivation, 'standard', self)
-            password = keystore.Xpub.get_pubkey_from_xpub(xpub, ()).hex()
+            password = client.get_password_for_storage_encryption()
             try:
                 storage.decrypt(password)
             except InvalidPassword:
                 # try to clear session so that user can type another passphrase
-                devmgr = self.plugins.device_manager
-                client = devmgr.client_by_id(device_info.device.id_)
                 if hasattr(client, 'clear_session'):  # FIXME not all hw wallet plugins have this
                     client.clear_session()
                 raise
         else:
             raise Exception('unknown purpose: %s' % purpose)
 
-    def derivation_and_script_type_dialog(self, f):
+    def derivation_and_script_type_dialog(self, f, *, get_account_xpub=None):
         message1 = _('Choose the type of addresses in your wallet.')
         message2 = ' '.join([
             _('You can override the suggested derivation path.'),
             _('If you are not sure what this is, leave this field unchanged.')
         ])
+        hide_choices = False
         if self.wallet_type == 'multisig':
             # There is no general standard for HD multisig.
             # For legacy, this is partially compatible with BIP45; assumes index=0
@@ -399,6 +421,14 @@ class BaseWizard(Logger):
                 ('p2wsh-p2sh', 'p2sh-segwit multisig (p2wsh-p2sh)', purpose48_derivation(0, xtype='p2wsh-p2sh')),
                 ('p2wsh',      'native segwit multisig (p2wsh)',    purpose48_derivation(0, xtype='p2wsh')),
             ]
+            # if this is not the first cosigner, pre-select the expected script type,
+            # and hide the choices
+            script_type = self.get_script_type_of_wallet()
+            if script_type is not None:
+                script_types = [*zip(*choices)][0]
+                chosen_idx = script_types.index(script_type)
+                default_choice_idx = chosen_idx
+                hide_choices = True
         else:
             default_choice_idx = 2
             choices = [
@@ -408,37 +438,54 @@ class BaseWizard(Logger):
             ]
         while True:
             try:
-                self.choice_and_line_dialog(
-                    run_next=f, title=_('Script type and Derivation path'), message1=message1,
-                    message2=message2, choices=choices, test_text=is_bip32_derivation,
-                    default_choice_idx=default_choice_idx)
+                self.derivation_and_script_type_gui_specific_dialog(
+                    run_next=f,
+                    title=_('Script type and Derivation path'),
+                    message1=message1,
+                    message2=message2,
+                    choices=choices,
+                    test_text=is_bip32_derivation,
+                    default_choice_idx=default_choice_idx,
+                    get_account_xpub=get_account_xpub,
+                    hide_choices=hide_choices,
+                )
                 return
             except ScriptTypeNotSupported as e:
                 self.show_error(e)
                 # let the user choose again
 
-    def on_hw_derivation(self, name, device_info, derivation, xtype):
+    def on_hw_derivation(self, name, device_info: 'DeviceInfo', derivation, xtype):
         from .keystore import hardware_keystore
         devmgr = self.plugins.device_manager
+        assert isinstance(self.plugin, HW_PluginBase)
         try:
             xpub = self.plugin.get_xpub(device_info.device.id_, derivation, xtype, self)
-            client = devmgr.client_by_id(device_info.device.id_)
+            client = devmgr.client_by_id(device_info.device.id_, scan_now=False)
             if not client: raise Exception("failed to find client for device id")
             root_fingerprint = client.request_root_fingerprint_from_device()
+            label = client.label()  # use this as device_info.label might be outdated!
+            soft_device_id = client.get_soft_device_id()  # use this as device_info.device_id might be outdated!
         except ScriptTypeNotSupported:
             raise  # this is handled in derivation_dialog
         except BaseException as e:
             self.logger.exception('')
             self.show_error(e)
-            return
+            raise ChooseHwDeviceAgain()
         d = {
             'type': 'hardware',
             'hw_type': name,
             'derivation': derivation,
             'root_fingerprint': root_fingerprint,
             'xpub': xpub,
-            'label': device_info.label,
+            'label': label,
+            'soft_device_id': soft_device_id,
         }
+        try:
+            client.manipulate_keystore_dict_during_wizard_setup(d)
+        except Exception as e:
+            self.logger.exception('')
+            self.show_error(e)
+            raise ChooseHwDeviceAgain()
         k = hardware_keystore(d)
         self.on_keystore(k)
 
@@ -462,12 +509,13 @@ class BaseWizard(Logger):
         self.opt_ext = True
         is_cosigning_seed = lambda x: mnemonic.seed_type(x) in ['standard', 'segwit']
         test = mnemonic.is_seed if self.wallet_type == 'standard' else is_cosigning_seed
-        self.restore_seed_dialog(run_next=self.on_restore_seed, test=test)
+        f = lambda *args: self.run('on_restore_seed', *args)
+        self.restore_seed_dialog(run_next=f, test=test)
 
     def on_restore_seed(self, seed, is_bip39, is_ext):
         self.seed_type = 'bip39' if is_bip39 else mnemonic.seed_type(seed)
         if self.seed_type == 'bip39':
-            f = lambda passphrase: self.on_restore_bip39(seed, passphrase)
+            f = lambda passphrase: self.run('on_restore_bip39', seed, passphrase)
             self.passphrase_dialog(run_next=f, is_restoring=True) if is_ext else f('')
         elif self.seed_type in ['standard', 'segwit']:
             f = lambda passphrase: self.run('create_keystore', seed, passphrase)
@@ -484,7 +532,16 @@ class BaseWizard(Logger):
         def f(derivation, script_type):
             derivation = normalize_bip32_derivation(derivation)
             self.run('on_bip43', seed, passphrase, derivation, script_type)
-        self.derivation_and_script_type_dialog(f)
+        if self.wallet_type == 'standard':
+            def get_account_xpub(account_path):
+                root_seed = bip39_to_seed(seed, passphrase)
+                root_node = BIP32Node.from_rootseed(root_seed, xtype="standard")
+                account_node = root_node.subkey_at_private_derivation(account_path)
+                account_xpub = account_node.to_xpub()
+                return account_xpub
+        else:
+            get_account_xpub = None
+        self.derivation_and_script_type_dialog(f, get_account_xpub=get_account_xpub)
 
     def create_keystore(self, seed, passphrase):
         k = keystore.from_seed(seed, passphrase, self.wallet_type == 'multisig')
@@ -494,7 +551,14 @@ class BaseWizard(Logger):
         k = keystore.from_bip39_seed(seed, passphrase, derivation, xtype=script_type)
         self.on_keystore(k)
 
-    def on_keystore(self, k):
+    def get_script_type_of_wallet(self) -> Optional[str]:
+        if len(self.keystores) > 0:
+            ks = self.keystores[0]
+            if isinstance(ks, keystore.Xpub):
+                return xpub_type(ks.xpub)
+        return None
+
+    def on_keystore(self, k: KeyStore):
         has_xpub = isinstance(k, keystore.Xpub)
         if has_xpub:
             t1 = xpub_type(k.xpub)
@@ -521,12 +585,15 @@ class BaseWizard(Logger):
                     self.show_error(_('Cannot add this cosigner:') + '\n' + "Their key type is '%s', we are '%s'"%(t1, t2))
                     self.run('choose_keystore')
                     return
-            self.keystores.append(k)
-            if len(self.keystores) == 1:
+            if len(self.keystores) == 0:
                 xpub = k.get_master_public_key()
                 self.reset_stack()
+                self.keystores.append(k)
                 self.run('show_xpub_and_add_cosigners', xpub)
-            elif len(self.keystores) < self.n:
+                return
+            self.reset_stack()
+            self.keystores.append(k)
+            if len(self.keystores) < self.n:
                 self.run('choose_keystore')
             else:
                 self.run('create_wallet')
@@ -538,18 +605,18 @@ class BaseWizard(Logger):
         if self.wallet_type == 'standard' and isinstance(self.keystores[0], Hardware_KeyStore):
             # offer encrypting with a pw derived from the hw device
             k = self.keystores[0]  # type: Hardware_KeyStore
+            assert isinstance(self.plugin, HW_PluginBase)
             try:
                 k.handler = self.plugin.create_handler(self)
                 password = k.get_password_for_storage_encryption()
             except UserCancelled:
                 devmgr = self.plugins.device_manager
                 devmgr.unpair_xpub(k.xpub)
-                self.choose_hw_device()
-                return
+                raise ChooseHwDeviceAgain()
             except BaseException as e:
                 self.logger.exception('')
                 self.show_error(str(e))
-                return
+                raise ChooseHwDeviceAgain()
             self.request_storage_encryption(
                 run_next=lambda encrypt_storage: self.on_password(
                     password,
@@ -593,12 +660,10 @@ class BaseWizard(Logger):
                                                    encrypt_keystore=encrypt_keystore)
         self.terminate()
 
-
-    def create_storage(self, path):
+    def create_storage(self, path) -> Tuple[WalletStorage, WalletDB]:
         if os.path.exists(path):
             raise Exception('file already exists at path')
-        if not self.pw_args:
-            return
+        assert self.pw_args, f"pw_args not set?!"
         pw_args = self.pw_args
         self.pw_args = None  # clean-up so that it can get GC-ed
         storage = WalletStorage(path)
@@ -612,7 +677,9 @@ class BaseWizard(Logger):
         db.write(storage)
         return storage, db
 
-    def terminate(self, *, storage: Optional[WalletStorage], db: Optional[WalletDB] = None):
+    def terminate(self, *, storage: WalletStorage = None,
+                  db: WalletDB = None,
+                  aborted: bool = False) -> None:
         raise NotImplementedError()  # implemented by subclasses
 
     def show_xpub_and_add_cosigners(self, xpub):
@@ -640,7 +707,7 @@ class BaseWizard(Logger):
     def create_seed(self, seed_type):
         from . import mnemonic
         self.seed_type = seed_type
-        seed = mnemonic.Mnemonic('en').make_seed(self.seed_type)
+        seed = mnemonic.Mnemonic('en').make_seed(seed_type=self.seed_type)
         self.opt_bip39 = False
         f = lambda x: self.request_passphrase(seed, x)
         self.show_seed_dialog(run_next=f, seed_text=seed)
@@ -654,7 +721,7 @@ class BaseWizard(Logger):
 
     def confirm_seed(self, seed, passphrase):
         f = lambda x: self.confirm_passphrase(seed, passphrase)
-        self.confirm_seed_dialog(run_next=f, test=lambda x: x==seed)
+        self.confirm_seed_dialog(run_next=f, seed=seed if self.config.get('debug_seed') else '', test=lambda x: x==seed)
 
     def confirm_passphrase(self, seed, passphrase):
         f = lambda x: self.run('create_keystore', seed, x)
@@ -667,3 +734,6 @@ class BaseWizard(Logger):
             self.line_dialog(run_next=f, title=title, message=message, default='', test=lambda x: x==passphrase)
         else:
             f('')
+
+    def show_error(self, msg: Union[str, BaseException]) -> None:
+        raise NotImplementedError()

electrum/bip32.py

@@ -401,3 +401,26 @@ def root_fp_and_der_prefix_from_xkey(xkey: str) -> Tuple[Optional[str], Optional
         derivation_prefix = convert_bip32_intpath_to_strpath([child_number_int])
         root_fingerprint = node.fingerprint.hex()
     return root_fingerprint, derivation_prefix
+
+
+def is_xkey_consistent_with_key_origin_info(xkey: str, *,
+                                            derivation_prefix: str = None,
+                                            root_fingerprint: str = None) -> bool:
+    bip32node = BIP32Node.from_xkey(xkey)
+    int_path = None
+    if derivation_prefix is not None:
+        int_path = convert_bip32_path_to_list_of_uint32(derivation_prefix)
+    if int_path is not None and len(int_path) != bip32node.depth:
+        return False
+    if bip32node.depth == 0:
+        if bfh(root_fingerprint) != bip32node.calc_fingerprint_of_this_node():
+            return False
+        if bip32node.child_number != bytes(4):
+            return False
+    if int_path is not None and bip32node.depth > 0:
+        if int.from_bytes(bip32node.child_number, 'big') != int_path[-1]:
+            return False
+    if bip32node.depth == 1:
+        if bfh(root_fingerprint) != bip32node.fingerprint:
+            return False
+    return True

electrum/bip39_recovery.py

@@ -0,0 +1,75 @@
+# Copyright (C) 2020 The Electrum developers
+# Distributed under the MIT software license, see the accompanying
+# file LICENCE or http://www.opensource.org/licenses/mit-license.php
+
+from typing import TYPE_CHECKING
+
+from aiorpcx import TaskGroup
+
+from . import bitcoin
+from .constants import BIP39_WALLET_FORMATS
+from .bip32 import BIP32_PRIME, BIP32Node
+from .bip32 import convert_bip32_path_to_list_of_uint32 as bip32_str_to_ints
+from .bip32 import convert_bip32_intpath_to_strpath as bip32_ints_to_str
+
+if TYPE_CHECKING:
+    from .network import Network
+
+
+async def account_discovery(network: 'Network', get_account_xpub):
+    async with TaskGroup() as group:
+        account_scan_tasks = []
+        for wallet_format in BIP39_WALLET_FORMATS:
+            account_scan = scan_for_active_accounts(network, get_account_xpub, wallet_format)
+            account_scan_tasks.append(await group.spawn(account_scan))
+    active_accounts = []
+    for task in account_scan_tasks:
+        active_accounts.extend(task.result())
+    return active_accounts
+
+
+async def scan_for_active_accounts(network: 'Network', get_account_xpub, wallet_format):
+    active_accounts = []
+    account_path = bip32_str_to_ints(wallet_format["derivation_path"])
+    while True:
+        account_xpub = get_account_xpub(account_path)
+        account_node = BIP32Node.from_xkey(account_xpub)
+        has_history = await account_has_history(network, account_node, wallet_format["script_type"])
+        if has_history:
+            account = format_account(wallet_format, account_path)
+            active_accounts.append(account)
+        if not has_history or not wallet_format["iterate_accounts"]:
+            break
+        account_path[-1] = account_path[-1] + 1
+    return active_accounts
+
+
+async def account_has_history(network: 'Network', account_node: BIP32Node, script_type: str) -> bool:
+    gap_limit = 20
+    async with TaskGroup() as group:
+        get_history_tasks = []
+        for address_index in range(gap_limit):
+            address_node = account_node.subkey_at_public_derivation("0/" + str(address_index))
+            pubkey = address_node.eckey.get_public_key_hex()
+            address = bitcoin.pubkey_to_address(script_type, pubkey)
+            script = bitcoin.address_to_script(address)
+            scripthash = bitcoin.script_to_scripthash(script)
+            get_history = network.get_history_for_scripthash(scripthash)
+            get_history_tasks.append(await group.spawn(get_history))
+    for task in get_history_tasks:
+        history = task.result()
+        if len(history) > 0:
+            return True
+    return False
+
+
+def format_account(wallet_format, account_path):
+    description = wallet_format["description"]
+    if wallet_format["iterate_accounts"]:
+        account_index = account_path[-1] % BIP32_PRIME
+        description = f'{description} (Account {account_index})'
+    return {
+        "description": description,
+        "derivation_path": bip32_ints_to_str(account_path),
+        "script_type": wallet_format["script_type"],
+    }

electrum/bip39_wallet_formats.json

@@ -0,0 +1,80 @@
+[
+    {
+        "description": "Standard BIP44 legacy",
+        "derivation_path": "m/44'/0'/0'",
+        "script_type": "p2pkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Standard BIP49 compatibility segwit",
+        "derivation_path": "m/49'/0'/0'",
+        "script_type": "p2wpkh-p2sh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Standard BIP84 native segwit",
+        "derivation_path": "m/84'/0'/0'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Non-standard legacy",
+        "derivation_path": "m/0'",
+        "script_type": "p2pkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Non-standard compatibility segwit",
+        "derivation_path": "m/0'",
+        "script_type": "p2wpkh-p2sh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Non-standard native segwit",
+        "derivation_path": "m/0'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Copay native segwit",
+        "derivation_path": "m/44'/0'/0'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Samourai Bad Bank (toxic change)",
+        "derivation_path": "m/84'/0'/2147483644'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Whirlpool Pre Mix",
+        "derivation_path": "m/84'/0'/2147483645'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Whirlpool Post Mix",
+        "derivation_path": "m/84'/0'/2147483646'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Ricochet legacy",
+        "derivation_path": "m/44'/0'/2147483647'",
+        "script_type": "p2pkh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Ricochet compatibility segwit",
+        "derivation_path": "m/49'/0'/2147483647'",
+        "script_type": "p2wpkh-p2sh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Ricochet native segwit",
+        "derivation_path": "m/84'/0'/2147483647'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": false
+    }
+]

electrum/bitcoin.py

@@ -24,10 +24,11 @@
 # SOFTWARE.
 
 import hashlib
-from typing import List, Tuple, TYPE_CHECKING, Optional, Union
-from enum import IntEnum
+from typing import List, Tuple, TYPE_CHECKING, Optional, Union, Sequence
+import enum
+from enum import IntEnum, Enum
 
-from .util import bfh, bh2u, BitcoinException, assert_bytes, to_bytes, inv_dict
+from .util import bfh, bh2u, BitcoinException, assert_bytes, to_bytes, inv_dict, is_hex_str
 from . import version
 from . import segwit_addr
 from . import constants
@@ -44,6 +45,10 @@ COINBASE_MATURITY = 100
 COIN = 100000000
 TOTAL_COIN_SUPPLY_LIMIT_IN_BTC = 21000000
 
+NLOCKTIME_MIN = 0
+NLOCKTIME_BLOCKHEIGHT_MAX = 500_000_000 - 1
+NLOCKTIME_MAX = 2 ** 32 - 1
+
 # supported types of transaction outputs
 # TODO kill these with fire
 TYPE_ADDRESS = 0
@@ -294,20 +299,62 @@ def add_number_to_script(i: int) -> bytes:
     return bfh(push_script(script_num_to_hex(i)))
 
 
+def construct_witness(items: Sequence[Union[str, int, bytes]]) -> str:
+    """Constructs a witness from the given stack items."""
+    witness = var_int(len(items))
+    for item in items:
+        if type(item) is int:
+            item = script_num_to_hex(item)
+        elif isinstance(item, (bytes, bytearray)):
+            item = bh2u(item)
+        else:
+            assert is_hex_str(item)
+        witness += witness_push(item)
+    return witness
+
+
+def construct_script(items: Sequence[Union[str, int, bytes, opcodes]]) -> str:
+    """Constructs bitcoin script from given items."""
+    script = ''
+    for item in items:
+        if isinstance(item, opcodes):
+            script += item.hex()
+        elif type(item) is int:
+            script += add_number_to_script(item).hex()
+        elif isinstance(item, (bytes, bytearray)):
+            script += push_script(item.hex())
+        elif isinstance(item, str):
+            assert is_hex_str(item)
+            script += push_script(item)
+        else:
+            raise Exception(f'unexpected item for script: {item!r}')
+    return script
+
+
 def relayfee(network: 'Network' = None) -> int:
+    """Returns feerate in sat/kbyte."""
     from .simple_config import FEERATE_DEFAULT_RELAY, FEERATE_MAX_RELAY
     if network and network.relay_fee is not None:
         fee = network.relay_fee
     else:
         fee = FEERATE_DEFAULT_RELAY
+    # sanity safeguards, as network.relay_fee is coming from a server:
     fee = min(fee, FEERATE_MAX_RELAY)
-    fee = max(fee, 0)
+    fee = max(fee, FEERATE_DEFAULT_RELAY)
     return fee
 
 
-def dust_threshold(network: 'Network'=None) -> int:
+# see https://github.com/bitcoin/bitcoin/blob/a62f0ed64f8bbbdfe6467ac5ce92ef5b5222d1bd/src/policy/policy.cpp#L14
+DUST_LIMIT_DEFAULT_SAT_LEGACY = 546
+DUST_LIMIT_DEFAULT_SAT_SEGWIT = 294
+
+
+def dust_threshold(network: 'Network' = None) -> int:
+    """Returns the dust limit in satoshis."""
     # Change <= dust threshold is added to the tx fee
-    return 182 * 3 * relayfee(network) // 1000
+    dust_lim = 182 * 3 * relayfee(network)  # in msat
+    # convert to sat, but round up:
+    return (dust_lim // 1000) + (dust_lim % 1000 > 0)
 
 
 def hash_encode(x: bytes) -> str:
@@ -359,12 +406,12 @@ def script_to_p2wsh(script: str, *, net=None) -> str:
     return hash_to_segwit_addr(sha256(bfh(script)), witver=0, net=net)
 
 def p2wpkh_nested_script(pubkey: str) -> str:
-    pkh = bh2u(hash_160(bfh(pubkey)))
-    return '00' + push_script(pkh)
+    pkh = hash_160(bfh(pubkey))
+    return construct_script([0, pkh])
 
 def p2wsh_nested_script(witness_script: str) -> str:
-    wsh = bh2u(sha256(bfh(witness_script)))
-    return '00' + push_script(wsh)
+    wsh = sha256(bfh(witness_script))
+    return construct_script([0, wsh])
 
 def pubkey_to_address(txin_type: str, pubkey: str, *, net=None) -> str:
     if net is None: net = constants.net
@@ -409,20 +456,50 @@ def address_to_script(addr: str, *, net=None) -> str:
     if witprog is not None:
         if not (0 <= witver <= 16):
             raise BitcoinException(f'impossible witness version: {witver}')
-        script = bh2u(add_number_to_script(witver))
-        script += push_script(bh2u(bytes(witprog)))
-        return script
+        return construct_script([witver, bytes(witprog)])
     addrtype, hash_160_ = b58_address_to_hash160(addr)
     if addrtype == net.ADDRTYPE_P2PKH:
         script = pubkeyhash_to_p2pkh_script(bh2u(hash_160_))
     elif addrtype == net.ADDRTYPE_P2SH:
-        script = opcodes.OP_HASH160.hex()
-        script += push_script(bh2u(hash_160_))
-        script += opcodes.OP_EQUAL.hex()
+        script = construct_script([opcodes.OP_HASH160, hash_160_, opcodes.OP_EQUAL])
     else:
         raise BitcoinException(f'unknown address type: {addrtype}')
     return script
 
+
+class OnchainOutputType(Enum):
+    """Opaque types of scriptPubKeys.
+    In case of p2sh, p2wsh and similar, no knowledge of redeem script, etc.
+    """
+    P2PKH = enum.auto()
+    P2SH = enum.auto()
+    WITVER0_P2WPKH = enum.auto()
+    WITVER0_P2WSH = enum.auto()
+
+
+def address_to_hash(addr: str, *, net=None) -> Tuple[OnchainOutputType, bytes]:
+    """Return (type, pubkey hash / witness program) for an address."""
+    if net is None: net = constants.net
+    if not is_address(addr, net=net):
+        raise BitcoinException(f"invalid bitcoin address: {addr}")
+    witver, witprog = segwit_addr.decode(net.SEGWIT_HRP, addr)
+    if witprog is not None:
+        if witver != 0:
+            raise BitcoinException(f"not implemented handling for witver={witver}")
+        if len(witprog) == 20:
+            return OnchainOutputType.WITVER0_P2WPKH, bytes(witprog)
+        elif len(witprog) == 32:
+            return OnchainOutputType.WITVER0_P2WSH, bytes(witprog)
+        else:
+            raise BitcoinException(f"unexpected length for segwit witver=0 witprog: len={len(witprog)}")
+    addrtype, hash_160_ = b58_address_to_hash160(addr)
+    if addrtype == net.ADDRTYPE_P2PKH:
+        return OnchainOutputType.P2PKH, hash_160_
+    elif addrtype == net.ADDRTYPE_P2SH:
+        return OnchainOutputType.P2SH, hash_160_
+    raise BitcoinException(f"unknown address type: {addrtype}")
+
+
 def address_to_scripthash(addr: str) -> str:
     script = address_to_script(addr)
     return script_to_scripthash(script)
@@ -432,13 +509,16 @@ def script_to_scripthash(script: str) -> str:
     return bh2u(bytes(reversed(h)))
 
 def public_key_to_p2pk_script(pubkey: str) -> str:
-    return push_script(pubkey) + opcodes.OP_CHECKSIG.hex()
+    return construct_script([pubkey, opcodes.OP_CHECKSIG])
 
 def pubkeyhash_to_p2pkh_script(pubkey_hash160: str) -> str:
-    script = bytes([opcodes.OP_DUP, opcodes.OP_HASH160]).hex()
-    script += push_script(pubkey_hash160)
-    script += bytes([opcodes.OP_EQUALVERIFY, opcodes.OP_CHECKSIG]).hex()
-    return script
+    return construct_script([
+        opcodes.OP_DUP,
+        opcodes.OP_HASH160,
+        pubkey_hash160,
+        opcodes.OP_EQUALVERIFY,
+        opcodes.OP_CHECKSIG
+    ])
 
 
 __b58chars = b'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
@@ -448,6 +528,9 @@ __b43chars = b'0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ$*+-./:'
 assert len(__b43chars) == 43
 
 
+class BaseDecodeError(BitcoinException): pass
+
+
 def base_encode(v: bytes, *, base: int) -> str:
     """ encode v, which is a string of bytes, to base58."""
     assert_bytes(v)
@@ -495,7 +578,7 @@ def base_decode(v: Union[bytes, str], *, base: int, length: int = None) -> Optio
     for c in v[::-1]:
         digit = chars.find(bytes([c]))
         if digit == -1:
-            raise ValueError('Forbidden character {} for base {}'.format(c, base))
+            raise BaseDecodeError('Forbidden character {} for base {}'.format(c, base))
         # naive but slow variant:   long_value += digit * (base**i)
         long_value += digit * power_of_base
         power_of_base *= base
@@ -518,7 +601,7 @@ def base_decode(v: Union[bytes, str], *, base: int, length: int = None) -> Optio
     return bytes(result)
 
 
-class InvalidChecksum(Exception):
+class InvalidChecksum(BaseDecodeError):
     pass
 
 
@@ -556,8 +639,8 @@ def is_segwit_script_type(txin_type: str) -> bool:
     return txin_type in ('p2wpkh', 'p2wpkh-p2sh', 'p2wsh', 'p2wsh-p2sh')
 
 
-def serialize_privkey(secret: bytes, compressed: bool, txin_type: str,
-                      internal_use: bool=False) -> str:
+def serialize_privkey(secret: bytes, compressed: bool, txin_type: str, *,
+                      internal_use: bool = False) -> str:
     # we only export secrets inside curve range
     secret = ecc.ECPrivkey.normalize_secret_bytes(secret)
     if internal_use:
@@ -584,18 +667,17 @@ def deserialize_privkey(key: str) -> Tuple[str, bytes, bool]:
             raise BitcoinException('unknown script type: {}'.format(txin_type))
     try:
         vch = DecodeBase58Check(key)
-    except BaseException:
+    except Exception as e:
         neutered_privkey = str(key)[:3] + '..' + str(key)[-2:]
-        raise BitcoinException("cannot deserialize privkey {}"
-                               .format(neutered_privkey))
+        raise BaseDecodeError(f"cannot deserialize privkey {neutered_privkey}") from e
 
     if txin_type is None:
         # keys exported in version 3.0.x encoded script type in first byte
         prefix_value = vch[0] - constants.net.WIF_PREFIX
         try:
             txin_type = WIF_SCRIPT_TYPES_INV[prefix_value]
-        except KeyError:
-            raise BitcoinException('invalid prefix ({}) for WIF key (1)'.format(vch[0]))
+        except KeyError as e:
+            raise BitcoinException('invalid prefix ({}) for WIF key (1)'.format(vch[0])) from None
     else:
         # all other keys must have a fixed first byte
         if vch[0] != constants.net.WIF_PREFIX:

electrum/blockchain.py

@@ -22,6 +22,7 @@
 # SOFTWARE.
 import os
 import threading
+import time
 from typing import Optional, Dict, Mapping, Sequence
 
 from . import util
@@ -85,7 +86,7 @@ def hash_raw_header(header: str) -> str:
 # key: blockhash hex at forkpoint
 # the chain at some key is the best chain that includes the given hash
 blockchains = {}  # type: Dict[str, Blockchain]
-blockchains_lock = threading.RLock()
+blockchains_lock = threading.RLock()  # lock order: take this last; so after Blockchain.lock
 
 
 def read_blockchains(config: 'SimpleConfig'):
@@ -159,6 +160,20 @@ _CHAINWORK_CACHE = {
 }  # type: Dict[str, int]
 
 
+def init_headers_file_for_best_chain():
+    b = get_best_chain()
+    filename = b.path()
+    length = HEADER_SIZE * len(constants.net.CHECKPOINTS) * 2016
+    if not os.path.exists(filename) or os.path.getsize(filename) < length:
+        with open(filename, 'wb') as f:
+            if length > 0:
+                f.seek(length - 1)
+                f.write(b'\x00')
+        util.ensure_sparse_file(filename)
+    with b.lock:
+        b.update_size()
+
+
 class Blockchain(Logger):
     """
     Manages blockchain headers and their verification
@@ -207,7 +222,7 @@ class Blockchain(Logger):
 
     def get_parent_heights(self) -> Mapping['Blockchain', int]:
         """Returns map: (parent chain -> height of last common block)"""
-        with blockchains_lock:
+        with self.lock, blockchains_lock:
             result = {self: self.height()}
             chain = self
             while True:
@@ -470,6 +485,20 @@ class Blockchain(Logger):
         height = self.height()
         return self.read_header(height)
 
+    def is_tip_stale(self) -> bool:
+        STALE_DELAY = 8 * 60 * 60  # in seconds
+        header = self.header_at_tip()
+        if not header:
+            return True
+        # note: We check the timestamp only in the latest header.
+        #       The Bitcoin consensus has a lot of leeway here:
+        #       - needs to be greater than the median of the timestamps of the past 11 blocks, and
+        #       - up to at most 2 hours into the future compared to local clock
+        #       so there is ~2 hours of leeway in either direction
+        if header['timestamp'] + STALE_DELAY < time.time():
+            return True
+        return False
+
     def get_hash(self, height: int) -> str:
         def is_height_checkpoint():
             within_cp_range = height <= constants.net.max_checkpoint()
@@ -617,6 +646,7 @@ class Blockchain(Logger):
 
 
 def check_header(header: dict) -> Optional[Blockchain]:
+    """Returns any Blockchain that contains header, or None."""
     if type(header) is not dict:
         return None
     with blockchains_lock: chains = list(blockchains.values())
@@ -627,8 +657,20 @@ def check_header(header: dict) -> Optional[Blockchain]:
 
 
 def can_connect(header: dict) -> Optional[Blockchain]:
+    """Returns the Blockchain that has a tip that directly links up
+    with header, or None.
+    """
     with blockchains_lock: chains = list(blockchains.values())
     for b in chains:
         if b.can_connect(header):
             return b
     return None
+
+
+def get_chains_that_contain_header(height: int, header_hash: str) -> Sequence[Blockchain]:
+    """Returns a list of Blockchains that contain header, best chain first."""
+    with blockchains_lock: chains = list(blockchains.values())
+    chains = [chain for chain in chains
+              if chain.check_hash(height=height, header_hash=header_hash)]
+    chains = sorted(chains, key=lambda x: x.get_chainwork(), reverse=True)
+    return chains

electrum/channel_db.py

@@ -31,13 +31,16 @@ from typing import Sequence, List, Tuple, Optional, Dict, NamedTuple, TYPE_CHECK
 import binascii
 import base64
 import asyncio
+import threading
+from enum import IntEnum
 
 
 from .sql_db import SqlDB, sql
-from . import constants
-from .util import bh2u, profiler, get_headers_dir, bfh, is_ip_address, list_enabled_bits
+from . import constants, util
+from .util import bh2u, profiler, get_headers_dir, is_ip_address, json_normalize
 from .logging import Logger
-from .lnutil import LN_GLOBAL_FEATURES_KNOWN_SET, LNPeerAddr, format_short_channel_id, ShortChannelID
+from .lnutil import (LNPeerAddr, format_short_channel_id, ShortChannelID,
+                     validate_features, IncompatibleOrInsaneFeatures)
 from .lnverifier import LNChannelVerifier, verify_sig_for_channel_update
 from .lnmsg import decode_msg
 
@@ -46,18 +49,18 @@ if TYPE_CHECKING:
     from .lnchannel import Channel
 
 
-class UnknownEvenFeatureBits(Exception): pass
-
-def validate_features(features : int):
-    enabled_features = list_enabled_bits(features)
-    for fbit in enabled_features:
-        if (1 << fbit) not in LN_GLOBAL_FEATURES_KNOWN_SET and fbit % 2 == 0:
-            raise UnknownEvenFeatureBits()
-
-
 FLAG_DISABLE   = 1 << 1
 FLAG_DIRECTION = 1 << 0
 
+
+class NodeAddress(NamedTuple):
+    """Holds address information of Lightning nodes
+    and how up to date this info is."""
+    host: str
+    port: int
+    timestamp: int
+
+
 class ChannelInfo(NamedTuple):
     short_channel_id: ShortChannelID
     node1_id: bytes
@@ -101,16 +104,22 @@ class Policy(NamedTuple):
     def from_msg(payload: dict) -> 'Policy':
         return Policy(
             key                         = payload['short_channel_id'] + payload['start_node'],
-            cltv_expiry_delta           = int.from_bytes(payload['cltv_expiry_delta'], "big"),
-            htlc_minimum_msat           = int.from_bytes(payload['htlc_minimum_msat'], "big"),
-            htlc_maximum_msat           = int.from_bytes(payload['htlc_maximum_msat'], "big") if 'htlc_maximum_msat' in payload else None,
-            fee_base_msat               = int.from_bytes(payload['fee_base_msat'], "big"),
-            fee_proportional_millionths = int.from_bytes(payload['fee_proportional_millionths'], "big"),
+            cltv_expiry_delta           = payload['cltv_expiry_delta'],
+            htlc_minimum_msat           = payload['htlc_minimum_msat'],
+            htlc_maximum_msat           = payload.get('htlc_maximum_msat', None),
+            fee_base_msat               = payload['fee_base_msat'],
+            fee_proportional_millionths = payload['fee_proportional_millionths'],
             message_flags               = int.from_bytes(payload['message_flags'], "big"),
             channel_flags               = int.from_bytes(payload['channel_flags'], "big"),
-            timestamp                   = int.from_bytes(payload['timestamp'], "big")
+            timestamp                   = payload['timestamp'],
         )
 
+    @staticmethod
+    def from_raw_msg(key:bytes, raw: bytes) -> 'Policy':
+        payload = decode_msg(raw)[1]
+        payload['start_node'] = key[8:]
+        return Policy.from_msg(payload)
+
     def is_disabled(self):
         return self.channel_flags & FLAG_DISABLE
 
@@ -119,11 +128,10 @@ class Policy(NamedTuple):
         return ShortChannelID.normalize(self.key[0:8])
 
     @property
-    def start_node(self):
+    def start_node(self) -> bytes:
         return self.key[8:]
 
 
-
 class NodeInfo(NamedTuple):
     node_id: bytes
     features: int
@@ -131,15 +139,30 @@ class NodeInfo(NamedTuple):
     alias: str
 
     @staticmethod
-    def from_msg(payload):
+    def from_msg(payload) -> Tuple['NodeInfo', Sequence['LNPeerAddr']]:
         node_id = payload['node_id']
         features = int.from_bytes(payload['features'], "big")
         validate_features(features)
         addresses = NodeInfo.parse_addresses_field(payload['addresses'])
+        peer_addrs = []
+        for host, port in addresses:
+            try:
+                peer_addrs.append(LNPeerAddr(host=host, port=port, pubkey=node_id))
+            except ValueError:
+                pass
         alias = payload['alias'].rstrip(b'\x00')
-        timestamp = int.from_bytes(payload['timestamp'], "big")
-        return NodeInfo(node_id=node_id, features=features, timestamp=timestamp, alias=alias), [
-            Address(host=host, port=port, node_id=node_id, last_connected_date=None) for host, port in addresses]
+        try:
+            alias = alias.decode('utf8')
+        except:
+            alias = ''
+        timestamp = payload['timestamp']
+        node_info = NodeInfo(node_id=node_id, features=features, timestamp=timestamp, alias=alias)
+        return node_info, peer_addrs
+
+    @staticmethod
+    def from_raw_msg(raw: bytes) -> Tuple['NodeInfo', Sequence['LNPeerAddr']]:
+        payload_dict = decode_msg(raw)[1]
+        return NodeInfo.from_msg(payload_dict)
 
     @staticmethod
     def parse_addresses_field(addresses_field):
@@ -182,50 +205,64 @@ class NodeInfo(NamedTuple):
         return addresses
 
 
-class Address(NamedTuple):
-    node_id: bytes
-    host: str
-    port: int
-    last_connected_date: Optional[int]
-
+class UpdateStatus(IntEnum):
+    ORPHANED   = 0
+    EXPIRED    = 1
+    DEPRECATED = 2
+    UNCHANGED  = 3
+    GOOD       = 4
 
 class CategorizedChannelUpdates(NamedTuple):
     orphaned: List    # no channel announcement for channel update
     expired: List     # update older than two weeks
     deprecated: List  # update older than database entry
+    unchanged: List   # unchanged policies
     good: List        # good updates
-    to_delete: List   # database entries to delete
 
 
-# TODO It would make more sense to store the raw gossip messages in the db.
-#      That is pretty much a pre-requisite of actively participating in gossip.
+def get_mychannel_info(short_channel_id: ShortChannelID,
+                       my_channels: Dict[ShortChannelID, 'Channel']) -> Optional[ChannelInfo]:
+    chan = my_channels.get(short_channel_id)
+    ci = ChannelInfo.from_raw_msg(chan.construct_channel_announcement_without_sigs())
+    return ci._replace(capacity_sat=chan.constraints.capacity)
+
+def get_mychannel_policy(short_channel_id: bytes, node_id: bytes,
+                         my_channels: Dict[ShortChannelID, 'Channel']) -> Optional[Policy]:
+    chan = my_channels.get(short_channel_id)  # type: Optional[Channel]
+    if not chan:
+        return
+    if node_id == chan.node_id:  # incoming direction (to us)
+        remote_update_raw = chan.get_remote_update()
+        if not remote_update_raw:
+            return
+        now = int(time.time())
+        remote_update_decoded = decode_msg(remote_update_raw)[1]
+        remote_update_decoded['timestamp'] = now
+        remote_update_decoded['start_node'] = node_id
+        return Policy.from_msg(remote_update_decoded)
+    elif node_id == chan.get_local_pubkey():  # outgoing direction (from us)
+        local_update_decoded = decode_msg(chan.get_outgoing_gossip_channel_update())[1]
+        local_update_decoded['start_node'] = node_id
+        return Policy.from_msg(local_update_decoded)
+
 
 create_channel_info = """
 CREATE TABLE IF NOT EXISTS channel_info (
-short_channel_id VARCHAR(64),
-node1_id VARCHAR(66),
-node2_id VARCHAR(66),
-capacity_sat INTEGER,
+short_channel_id BLOB(8),
+msg BLOB,
 PRIMARY KEY(short_channel_id)
 )"""
 
 create_policy = """
 CREATE TABLE IF NOT EXISTS policy (
-key VARCHAR(66),
-cltv_expiry_delta INTEGER NOT NULL,
-htlc_minimum_msat INTEGER NOT NULL,
-htlc_maximum_msat INTEGER,
-fee_base_msat INTEGER NOT NULL,
-fee_proportional_millionths INTEGER NOT NULL,
-channel_flags INTEGER NOT NULL,
-message_flags INTEGER NOT NULL,
-timestamp INTEGER NOT NULL,
+key BLOB(41),
+msg BLOB,
 PRIMARY KEY(key)
 )"""
 
 create_address = """
 CREATE TABLE IF NOT EXISTS address (
-node_id VARCHAR(66),
+node_id BLOB(33),
 host STRING(256),
 port INTEGER NOT NULL,
 timestamp INTEGER,
@@ -234,10 +271,8 @@ PRIMARY KEY(node_id, host, port)
 
 create_node_info = """
 CREATE TABLE IF NOT EXISTS node_info (
-node_id VARCHAR(66),
-features INTEGER NOT NULL,
-timestamp INTEGER NOT NULL,
-alias STRING(64),
+node_id BLOB(33),
+msg BLOB,
 PRIMARY KEY(node_id)
 )"""
 
@@ -247,19 +282,27 @@ class ChannelDB(SqlDB):
     NUM_MAX_RECENT_PEERS = 20
 
     def __init__(self, network: 'Network'):
-        path = os.path.join(get_headers_dir(network.config), 'channel_db')
-        super().__init__(network, path, commit_interval=100)
+        path = os.path.join(get_headers_dir(network.config), 'gossip_db')
+        super().__init__(network.asyncio_loop, path, commit_interval=100)
+        self.lock = threading.RLock()
         self.num_nodes = 0
         self.num_channels = 0
         self._channel_updates_for_private_channels = {}  # type: Dict[Tuple[bytes, bytes], dict]
         self.ca_verifier = LNChannelVerifier(network, self)
+
         # initialized in load_data
-        self._channels = {}  # type: Dict[bytes, ChannelInfo]
-        self._policies = {}  # type: Dict[Tuple[bytes, bytes], Policy]  # (node_id, scid) -> Policy
-        self._nodes = {}
+        # note: modify/iterate needs self.lock
+        self._channels = {}  # type: Dict[ShortChannelID, ChannelInfo]
+        self._policies = {}  # type: Dict[Tuple[bytes, ShortChannelID], Policy]  # (node_id, scid) -> Policy
+        self._nodes = {}  # type: Dict[bytes, NodeInfo]  # node_id -> NodeInfo
         # node_id -> (host, port, ts)
-        self._addresses = defaultdict(set)  # type: Dict[bytes, Set[Tuple[str, int, int]]]
+        self._addresses = defaultdict(set)  # type: Dict[bytes, Set[NodeAddress]]
         self._channels_for_node = defaultdict(set)  # type: Dict[bytes, Set[ShortChannelID]]
+        self._recent_peers = []  # type: List[bytes]  # list of node_ids
+        self._chans_with_0_policies = set()  # type: Set[ShortChannelID]
+        self._chans_with_1_policies = set()  # type: Set[ShortChannelID]
+        self._chans_with_2_policies = set()  # type: Set[ShortChannelID]
+
         self.data_loaded = asyncio.Event()
         self.network = network # only for callback
 
@@ -267,47 +310,55 @@ class ChannelDB(SqlDB):
         self.num_nodes = len(self._nodes)
         self.num_channels = len(self._channels)
         self.num_policies = len(self._policies)
-        self.network.trigger_callback('channel_db', self.num_nodes, self.num_channels, self.num_policies)
+        util.trigger_callback('channel_db', self.num_nodes, self.num_channels, self.num_policies)
+        util.trigger_callback('ln_gossip_sync_progress')
 
     def get_channel_ids(self):
-        return set(self._channels.keys())
+        with self.lock:
+            return set(self._channels.keys())
 
     def add_recent_peer(self, peer: LNPeerAddr):
         now = int(time.time())
         node_id = peer.pubkey
-        self._addresses[node_id].add((peer.host, peer.port, now))
-        self.save_node_address(node_id, peer, now)
+        with self.lock:
+            self._addresses[node_id].add(NodeAddress(peer.host, peer.port, now))
+            # list is ordered
+            if node_id in self._recent_peers:
+                self._recent_peers.remove(node_id)
+            self._recent_peers.insert(0, node_id)
+            self._recent_peers = self._recent_peers[:self.NUM_MAX_RECENT_PEERS]
+        self._db_save_node_address(peer, now)
 
     def get_200_randomly_sorted_nodes_not_in(self, node_ids):
-        unshuffled = set(self._nodes.keys()) - node_ids
+        with self.lock:
+            unshuffled = set(self._nodes.keys()) - node_ids
         return random.sample(unshuffled, min(200, len(unshuffled)))
 
     def get_last_good_address(self, node_id) -> Optional[LNPeerAddr]:
         r = self._addresses.get(node_id)
         if not r:
             return None
-        addr = sorted(list(r), key=lambda x: x[2])[0]
-        host, port, timestamp = addr
+        addr = sorted(list(r), key=lambda x: x.timestamp)[0]
         try:
-            return LNPeerAddr(host, port, node_id)
+            return LNPeerAddr(addr.host, addr.port, node_id)
         except ValueError:
             return None
 
     def get_recent_peers(self):
-        assert self.data_loaded.is_set(), "channelDB load_data did not finish yet!"
-        # FIXME this does not reliably return "recent" peers...
-        #       Also, the list() cast over the whole dict (thousands of elements),
-        #       is really inefficient.
-        r = [self.get_last_good_address(node_id)
-             for node_id in list(self._addresses.keys())[-self.NUM_MAX_RECENT_PEERS:]]
-        return list(reversed(r))
+        if not self.data_loaded.is_set():
+            raise Exception("channelDB data not loaded yet!")
+        with self.lock:
+            ret = [self.get_last_good_address(node_id)
+                   for node_id in self._recent_peers]
+            return ret
 
     # note: currently channel announcements are trusted by default (trusted=True);
-    #       they are not verified. Verifying them would make the gossip sync
+    #       they are not SPV-verified. Verifying them would make the gossip sync
     #       even slower; especially as servers will start throttling us.
     #       It would probably put significant strain on servers if all clients
     #       verified the complete gossip.
     def add_channel_announcement(self, msg_payloads, *, trusted=True):
+        # note: signatures have already been verified.
         if type(msg_payloads) is dict:
             msg_payloads = [msg_payloads]
         added = 0
@@ -320,8 +371,8 @@ class ChannelDB(SqlDB):
                 continue
             try:
                 channel_info = ChannelInfo.from_msg(msg)
-            except UnknownEvenFeatureBits:
-                self.logger.info("unknown feature bits")
+            except IncompatibleOrInsaneFeatures as e:
+                self.logger.info(f"unknown or insane feature bits: {e!r}")
                 continue
             if trusted:
                 added += 1
@@ -335,84 +386,113 @@ class ChannelDB(SqlDB):
     def add_verified_channel_info(self, msg: dict, *, capacity_sat: int = None) -> None:
         try:
             channel_info = ChannelInfo.from_msg(msg)
-        except UnknownEvenFeatureBits:
+        except IncompatibleOrInsaneFeatures:
             return
         channel_info = channel_info._replace(capacity_sat=capacity_sat)
-        self._channels[channel_info.short_channel_id] = channel_info
-        self._channels_for_node[channel_info.node1_id].add(channel_info.short_channel_id)
-        self._channels_for_node[channel_info.node2_id].add(channel_info.short_channel_id)
-        self.save_channel(channel_info)
+        with self.lock:
+            self._channels[channel_info.short_channel_id] = channel_info
+            self._channels_for_node[channel_info.node1_id].add(channel_info.short_channel_id)
+            self._channels_for_node[channel_info.node2_id].add(channel_info.short_channel_id)
+        self._update_num_policies_for_chan(channel_info.short_channel_id)
+        if 'raw' in msg:
+            self._db_save_channel(channel_info.short_channel_id, msg['raw'])
 
-    def print_change(self, old_policy: Policy, new_policy: Policy):
-        # print what changed between policies
+    def policy_changed(self, old_policy: Policy, new_policy: Policy, verbose: bool) -> bool:
+        changed = False
         if old_policy.cltv_expiry_delta != new_policy.cltv_expiry_delta:
-            self.logger.info(f'cltv_expiry_delta: {old_policy.cltv_expiry_delta} -> {new_policy.cltv_expiry_delta}')
+            changed |= True
+            if verbose:
+                self.logger.info(f'cltv_expiry_delta: {old_policy.cltv_expiry_delta} -> {new_policy.cltv_expiry_delta}')
         if old_policy.htlc_minimum_msat != new_policy.htlc_minimum_msat:
-            self.logger.info(f'htlc_minimum_msat: {old_policy.htlc_minimum_msat} -> {new_policy.htlc_minimum_msat}')
+            changed |= True
+            if verbose:
+                self.logger.info(f'htlc_minimum_msat: {old_policy.htlc_minimum_msat} -> {new_policy.htlc_minimum_msat}')
         if old_policy.htlc_maximum_msat != new_policy.htlc_maximum_msat:
-            self.logger.info(f'htlc_maximum_msat: {old_policy.htlc_maximum_msat} -> {new_policy.htlc_maximum_msat}')
+            changed |= True
+            if verbose:
+                self.logger.info(f'htlc_maximum_msat: {old_policy.htlc_maximum_msat} -> {new_policy.htlc_maximum_msat}')
         if old_policy.fee_base_msat != new_policy.fee_base_msat:
-            self.logger.info(f'fee_base_msat: {old_policy.fee_base_msat} -> {new_policy.fee_base_msat}')
+            changed |= True
+            if verbose:
+                self.logger.info(f'fee_base_msat: {old_policy.fee_base_msat} -> {new_policy.fee_base_msat}')
         if old_policy.fee_proportional_millionths != new_policy.fee_proportional_millionths:
-            self.logger.info(f'fee_proportional_millionths: {old_policy.fee_proportional_millionths} -> {new_policy.fee_proportional_millionths}')
+            changed |= True
+            if verbose:
+                self.logger.info(f'fee_proportional_millionths: {old_policy.fee_proportional_millionths} -> {new_policy.fee_proportional_millionths}')
         if old_policy.channel_flags != new_policy.channel_flags:
-            self.logger.info(f'channel_flags: {old_policy.channel_flags} -> {new_policy.channel_flags}')
+            changed |= True
+            if verbose:
+                self.logger.info(f'channel_flags: {old_policy.channel_flags} -> {new_policy.channel_flags}')
         if old_policy.message_flags != new_policy.message_flags:
-            self.logger.info(f'message_flags: {old_policy.message_flags} -> {new_policy.message_flags}')
+            changed |= True
+            if verbose:
+                self.logger.info(f'message_flags: {old_policy.message_flags} -> {new_policy.message_flags}')
+        if not changed and verbose:
+            self.logger.info(f'policy unchanged: {old_policy.timestamp} -> {new_policy.timestamp}')
+        return changed
 
-    def add_channel_updates(self, payloads, max_age=None, verify=True) -> CategorizedChannelUpdates:
+    def add_channel_update(self, payload, max_age=None, verify=False, verbose=True):
+        now = int(time.time())
+        short_channel_id = ShortChannelID(payload['short_channel_id'])
+        timestamp = payload['timestamp']
+        if max_age and now - timestamp > max_age:
+            return UpdateStatus.EXPIRED
+        if timestamp - now > 60:
+            return UpdateStatus.DEPRECATED
+        channel_info = self._channels.get(short_channel_id)
+        if not channel_info:
+            return UpdateStatus.ORPHANED
+        flags = int.from_bytes(payload['channel_flags'], 'big')
+        direction = flags & FLAG_DIRECTION
+        start_node = channel_info.node1_id if direction == 0 else channel_info.node2_id
+        payload['start_node'] = start_node
+        # compare updates to existing database entries
+        timestamp = payload['timestamp']
+        start_node = payload['start_node']
+        short_channel_id = ShortChannelID(payload['short_channel_id'])
+        key = (start_node, short_channel_id)
+        old_policy = self._policies.get(key)
+        if old_policy and timestamp <= old_policy.timestamp + 60:
+            return UpdateStatus.DEPRECATED
+        if verify:
+            self.verify_channel_update(payload)
+        policy = Policy.from_msg(payload)
+        with self.lock:
+            self._policies[key] = policy
+        self._update_num_policies_for_chan(short_channel_id)
+        if 'raw' in payload:
+            self._db_save_policy(policy.key, payload['raw'])
+        if old_policy and not self.policy_changed(old_policy, policy, verbose):
+            return UpdateStatus.UNCHANGED
+        else:
+            return UpdateStatus.GOOD
+
+    def add_channel_updates(self, payloads, max_age=None) -> CategorizedChannelUpdates:
         orphaned = []
         expired = []
         deprecated = []
+        unchanged = []
         good = []
-        to_delete = []
-        # filter orphaned and expired first
-        known = []
-        now = int(time.time())
         for payload in payloads:
-            short_channel_id = ShortChannelID(payload['short_channel_id'])
-            timestamp = int.from_bytes(payload['timestamp'], "big")
-            if max_age and now - timestamp > max_age:
-                expired.append(payload)
-                continue
-            channel_info = self._channels.get(short_channel_id)
-            if not channel_info:
+            r = self.add_channel_update(payload, max_age=max_age, verbose=False)
+            if r == UpdateStatus.ORPHANED:
                 orphaned.append(payload)
-                continue
-            flags = int.from_bytes(payload['channel_flags'], 'big')
-            direction = flags & FLAG_DIRECTION
-            start_node = channel_info.node1_id if direction == 0 else channel_info.node2_id
-            payload['start_node'] = start_node
-            known.append(payload)
-        # compare updates to existing database entries
-        for payload in known:
-            timestamp = int.from_bytes(payload['timestamp'], "big")
-            start_node = payload['start_node']
-            short_channel_id = ShortChannelID(payload['short_channel_id'])
-            key = (start_node, short_channel_id)
-            old_policy = self._policies.get(key)
-            if old_policy and timestamp <= old_policy.timestamp:
+            elif r == UpdateStatus.EXPIRED:
+                expired.append(payload)
+            elif r == UpdateStatus.DEPRECATED:
                 deprecated.append(payload)
-                continue
-            good.append(payload)
-            if verify:
-                self.verify_channel_update(payload)
-            policy = Policy.from_msg(payload)
-            self._policies[key] = policy
-            self.save_policy(policy)
-        #
+            elif r == UpdateStatus.UNCHANGED:
+                unchanged.append(payload)
+            elif r == UpdateStatus.GOOD:
+                good.append(payload)
         self.update_counts()
         return CategorizedChannelUpdates(
             orphaned=orphaned,
             expired=expired,
             deprecated=deprecated,
-            good=good,
-            to_delete=to_delete,
-        )
+            unchanged=unchanged,
+            good=good)
 
-    def add_channel_update(self, payload):
-        # called from tests
-        self.add_channel_updates([payload], verify=False)
 
     def create_database(self):
         c = self.conn.cursor()
@@ -423,44 +503,48 @@ class ChannelDB(SqlDB):
         self.conn.commit()
 
     @sql
-    def save_policy(self, policy):
+    def _db_save_policy(self, key: bytes, msg: bytes):
+        # 'msg' is a 'channel_update' message
         c = self.conn.cursor()
-        c.execute("""REPLACE INTO policy (key, cltv_expiry_delta, htlc_minimum_msat, htlc_maximum_msat, fee_base_msat, fee_proportional_millionths, channel_flags, message_flags, timestamp) VALUES (?,?,?,?,?,?,?,?,?)""", list(policy))
+        c.execute("""REPLACE INTO policy (key, msg) VALUES (?,?)""", [key, msg])
 
     @sql
-    def delete_policy(self, node_id, short_channel_id):
+    def _db_delete_policy(self, node_id: bytes, short_channel_id: ShortChannelID):
         key = short_channel_id + node_id
         c = self.conn.cursor()
         c.execute("""DELETE FROM policy WHERE key=?""", (key,))
 
     @sql
-    def save_channel(self, channel_info):
+    def _db_save_channel(self, short_channel_id: ShortChannelID, msg: bytes):
+        # 'msg' is a 'channel_announcement' message
         c = self.conn.cursor()
-        c.execute("REPLACE INTO channel_info (short_channel_id, node1_id, node2_id, capacity_sat) VALUES (?,?,?,?)", list(channel_info))
+        c.execute("REPLACE INTO channel_info (short_channel_id, msg) VALUES (?,?)", [short_channel_id, msg])
 
     @sql
-    def delete_channel(self, short_channel_id):
+    def _db_delete_channel(self, short_channel_id: ShortChannelID):
         c = self.conn.cursor()
         c.execute("""DELETE FROM channel_info WHERE short_channel_id=?""", (short_channel_id,))
 
     @sql
-    def save_node(self, node_info):
+    def _db_save_node_info(self, node_id: bytes, msg: bytes):
+        # 'msg' is a 'node_announcement' message
         c = self.conn.cursor()
-        c.execute("REPLACE INTO node_info (node_id, features, timestamp, alias) VALUES (?,?,?,?)", list(node_info))
+        c.execute("REPLACE INTO node_info (node_id, msg) VALUES (?,?)", [node_id, msg])
 
     @sql
-    def save_node_address(self, node_id, peer, now):
+    def _db_save_node_address(self, peer: LNPeerAddr, timestamp: int):
         c = self.conn.cursor()
-        c.execute("REPLACE INTO address (node_id, host, port, timestamp) VALUES (?,?,?,?)", (node_id, peer.host, peer.port, now))
+        c.execute("REPLACE INTO address (node_id, host, port, timestamp) VALUES (?,?,?,?)",
+                  (peer.pubkey, peer.host, peer.port, timestamp))
 
     @sql
-    def save_node_addresses(self, node_id, node_addresses):
+    def _db_save_node_addresses(self, node_addresses: Sequence[LNPeerAddr]):
         c = self.conn.cursor()
         for addr in node_addresses:
-            c.execute("SELECT * FROM address WHERE node_id=? AND host=? AND port=?", (addr.node_id, addr.host, addr.port))
+            c.execute("SELECT * FROM address WHERE node_id=? AND host=? AND port=?", (addr.pubkey, addr.host, addr.port))
             r = c.fetchall()
             if r == []:
-                c.execute("INSERT INTO address (node_id, host, port, timestamp) VALUES (?,?,?,?)", (addr.node_id, addr.host, addr.port, 0))
+                c.execute("INSERT INTO address (node_id, host, port, timestamp) VALUES (?,?,?,?)", (addr.pubkey, addr.host, addr.port, 0))
 
     def verify_channel_update(self, payload):
         short_channel_id = payload['short_channel_id']
@@ -471,14 +555,14 @@ class ChannelDB(SqlDB):
             raise Exception(f'failed verifying channel update for {short_channel_id}')
 
     def add_node_announcement(self, msg_payloads):
+        # note: signatures have already been verified.
         if type(msg_payloads) is dict:
             msg_payloads = [msg_payloads]
-        old_addr = None
         new_nodes = {}
         for msg_payload in msg_payloads:
             try:
                 node_info, node_addresses = NodeInfo.from_msg(msg_payload)
-            except UnknownEvenFeatureBits:
+            except IncompatibleOrInsaneFeatures:
                 continue
             node_id = node_info.node_id
             # Ignore node if it has no associated channel (DoS protection)
@@ -492,39 +576,44 @@ class ChannelDB(SqlDB):
             if node and node.timestamp >= node_info.timestamp:
                 continue
             # save
-            self._nodes[node_id] = node_info
-            self.save_node(node_info)
-            for addr in node_addresses:
-                self._addresses[node_id].add((addr.host, addr.port, 0))
-            self.save_node_addresses(node_id, node_addresses)
+            with self.lock:
+                self._nodes[node_id] = node_info
+            if 'raw' in msg_payload:
+                self._db_save_node_info(node_id, msg_payload['raw'])
+            with self.lock:
+                for addr in node_addresses:
+                    self._addresses[node_id].add(NodeAddress(addr.host, addr.port, 0))
+            self._db_save_node_addresses(node_addresses)
 
         self.logger.debug("on_node_announcement: %d/%d"%(len(new_nodes), len(msg_payloads)))
         self.update_counts()
 
-    def get_old_policies(self, delta):
+    def get_old_policies(self, delta) -> Sequence[Tuple[bytes, ShortChannelID]]:
+        with self.lock:
+            _policies = self._policies.copy()
         now = int(time.time())
-        return list(k for k, v in list(self._policies.items()) if v.timestamp <= now - delta)
+        return list(k for k, v in _policies.items() if v.timestamp <= now - delta)
 
     def prune_old_policies(self, delta):
-        l = self.get_old_policies(delta)
-        if l:
-            for k in l:
-                self._policies.pop(k)
-                self.delete_policy(*k)
+        old_policies = self.get_old_policies(delta)
+        if old_policies:
+            for key in old_policies:
+                node_id, scid = key
+                with self.lock:
+                    self._policies.pop(key)
+                self._db_delete_policy(*key)
+                self._update_num_policies_for_chan(scid)
             self.update_counts()
-            self.logger.info(f'Deleting {len(l)} old policies')
-
-    def get_orphaned_channels(self):
-        ids = set(x[1] for x in self._policies.keys())
-        return list(x for x in self._channels.keys() if x not in ids)
+            self.logger.info(f'Deleting {len(old_policies)} old policies')
 
     def prune_orphaned_channels(self):
-        l = self.get_orphaned_channels()
-        if l:
-            for short_channel_id in l:
+        with self.lock:
+            orphaned_chans = self._chans_with_0_policies.copy()
+        if orphaned_chans:
+            for short_channel_id in orphaned_chans:
                 self.remove_channel(short_channel_id)
             self.update_counts()
-            self.logger.info(f'Deleting {len(l)} orphaned channels')
+            self.logger.info(f'Deleting {len(orphaned_chans)} orphaned channels')
 
     def add_channel_update_for_private_channel(self, msg_payload: dict, start_node_id: bytes):
         if not verify_sig_for_channel_update(msg_payload, start_node_id):
@@ -534,12 +623,15 @@ class ChannelDB(SqlDB):
         self._channel_updates_for_private_channels[(start_node_id, short_channel_id)] = msg_payload
 
     def remove_channel(self, short_channel_id: ShortChannelID):
-        channel_info = self._channels.pop(short_channel_id, None)
-        if channel_info:
-            self._channels_for_node[channel_info.node1_id].remove(channel_info.short_channel_id)
-            self._channels_for_node[channel_info.node2_id].remove(channel_info.short_channel_id)
+        # FIXME what about rm-ing policies?
+        with self.lock:
+            channel_info = self._channels.pop(short_channel_id, None)
+            if channel_info:
+                self._channels_for_node[channel_info.node1_id].remove(channel_info.short_channel_id)
+                self._channels_for_node[channel_info.node2_id].remove(channel_info.short_channel_id)
+        self._update_num_policies_for_chan(short_channel_id)
         # delete from database
-        self.delete_channel(short_channel_id)
+        self._db_delete_channel(short_channel_id)
 
     def get_node_addresses(self, node_id):
         return self._addresses.get(node_id)
@@ -547,40 +639,80 @@ class ChannelDB(SqlDB):
     @sql
     @profiler
     def load_data(self):
+        if self.data_loaded.is_set():
+            return
+        # Note: this method takes several seconds... mostly due to lnmsg.decode_msg being slow.
+        #       I believe lnmsg (and lightning.json) will need a rewrite anyway, so instead of tweaking
+        #       load_data() here, that should be done. see #6006
         c = self.conn.cursor()
         c.execute("""SELECT * FROM address""")
         for x in c:
             node_id, host, port, timestamp = x
-            self._addresses[node_id].add((str(host), int(port), int(timestamp or 0)))
+            self._addresses[node_id].add(NodeAddress(str(host), int(port), int(timestamp or 0)))
+        def newest_ts_for_node_id(node_id):
+            newest_ts = 0
+            for addr in self._addresses[node_id]:
+                newest_ts = max(newest_ts, addr.timestamp)
+            return newest_ts
+        sorted_node_ids = sorted(self._addresses.keys(), key=newest_ts_for_node_id, reverse=True)
+        self._recent_peers = sorted_node_ids[:self.NUM_MAX_RECENT_PEERS]
         c.execute("""SELECT * FROM channel_info""")
-        for x in c:
-            x = (ShortChannelID.normalize(x[0]), *x[1:])
-            ci = ChannelInfo(*x)
-            self._channels[ci.short_channel_id] = ci
+        for short_channel_id, msg in c:
+            try:
+                ci = ChannelInfo.from_raw_msg(msg)
+            except IncompatibleOrInsaneFeatures:
+                continue
+            self._channels[ShortChannelID.normalize(short_channel_id)] = ci
         c.execute("""SELECT * FROM node_info""")
-        for x in c:
-            ni = NodeInfo(*x)
-            self._nodes[ni.node_id] = ni
+        for node_id, msg in c:
+            try:
+                node_info, node_addresses = NodeInfo.from_raw_msg(msg)
+            except IncompatibleOrInsaneFeatures:
+                continue
+            # don't load node_addresses because they dont have timestamps
+            self._nodes[node_id] = node_info
         c.execute("""SELECT * FROM policy""")
-        for x in c:
-            p = Policy(*x)
+        for key, msg in c:
+            p = Policy.from_raw_msg(key, msg)
             self._policies[(p.start_node, p.short_channel_id)] = p
         for channel_info in self._channels.values():
             self._channels_for_node[channel_info.node1_id].add(channel_info.short_channel_id)
             self._channels_for_node[channel_info.node2_id].add(channel_info.short_channel_id)
+            self._update_num_policies_for_chan(channel_info.short_channel_id)
         self.logger.info(f'load data {len(self._channels)} {len(self._policies)} {len(self._channels_for_node)}')
         self.update_counts()
-        self.count_incomplete_channels()
+        (nchans_with_0p, nchans_with_1p, nchans_with_2p) = self.get_num_channels_partitioned_by_policy_count()
+        self.logger.info(f'num_channels_partitioned_by_policy_count. '
+                         f'0p: {nchans_with_0p}, 1p: {nchans_with_1p}, 2p: {nchans_with_2p}')
         self.data_loaded.set()
+        util.trigger_callback('gossip_db_loaded')
 
-    def count_incomplete_channels(self):
-        out = set()
-        for short_channel_id, ci in self._channels.items():
-            p1 = self.get_policy_for_node(short_channel_id, ci.node1_id)
-            p2 = self.get_policy_for_node(short_channel_id, ci.node2_id)
-            if p1 is None or p2 is not None:
-                out.add(short_channel_id)
-        self.logger.info(f'semi-orphaned: {len(out)}')
+    def _update_num_policies_for_chan(self, short_channel_id: ShortChannelID) -> None:
+        channel_info = self.get_channel_info(short_channel_id)
+        if channel_info is None:
+            with self.lock:
+                self._chans_with_0_policies.discard(short_channel_id)
+                self._chans_with_1_policies.discard(short_channel_id)
+                self._chans_with_2_policies.discard(short_channel_id)
+            return
+        p1 = self.get_policy_for_node(short_channel_id, channel_info.node1_id)
+        p2 = self.get_policy_for_node(short_channel_id, channel_info.node2_id)
+        with self.lock:
+            self._chans_with_0_policies.discard(short_channel_id)
+            self._chans_with_1_policies.discard(short_channel_id)
+            self._chans_with_2_policies.discard(short_channel_id)
+            if p1 is not None and p2 is not None:
+                self._chans_with_2_policies.add(short_channel_id)
+            elif p1 is None and p2 is None:
+                self._chans_with_0_policies.add(short_channel_id)
+            else:
+                self._chans_with_1_policies.add(short_channel_id)
+
+    def get_num_channels_partitioned_by_policy_count(self) -> Tuple[int, int, int]:
+        nchans_with_0p = len(self._chans_with_0_policies)
+        nchans_with_1p = len(self._chans_with_1_policies)
+        nchans_with_2p = len(self._chans_with_2_policies)
+        return nchans_with_0p, nchans_with_1p, nchans_with_2p
 
     def get_policy_for_node(self, short_channel_id: bytes, node_id: bytes, *,
                             my_channels: Dict[ShortChannelID, 'Channel'] = None) -> Optional['Policy']:
@@ -594,40 +726,23 @@ class ChannelDB(SqlDB):
             if chan_upd_dict:
                 return Policy.from_msg(chan_upd_dict)
         # check if it's one of our own channels
-        if not my_channels:
-            return
-        chan = my_channels.get(short_channel_id)  # type: Optional[Channel]
-        if not chan:
-            return
-        if node_id == chan.node_id:  # incoming direction (to us)
-            remote_update_raw = chan.get_remote_update()
-            if not remote_update_raw:
-                return
-            now = int(time.time())
-            remote_update_decoded = decode_msg(remote_update_raw)[1]
-            remote_update_decoded['timestamp'] = now.to_bytes(4, byteorder="big")
-            remote_update_decoded['start_node'] = node_id
-            return Policy.from_msg(remote_update_decoded)
-        elif node_id == chan.get_local_pubkey():  # outgoing direction (from us)
-            local_update_decoded = decode_msg(chan.get_outgoing_gossip_channel_update())[1]
-            local_update_decoded['start_node'] = node_id
-            return Policy.from_msg(local_update_decoded)
+        if my_channels:
+            return get_mychannel_policy(short_channel_id, node_id, my_channels)
 
-    def get_channel_info(self, short_channel_id: bytes, *,
+    def get_channel_info(self, short_channel_id: ShortChannelID, *,
                          my_channels: Dict[ShortChannelID, 'Channel'] = None) -> Optional[ChannelInfo]:
         ret = self._channels.get(short_channel_id)
         if ret:
             return ret
         # check if it's one of our own channels
-        if not my_channels:
-            return
-        chan = my_channels.get(short_channel_id)  # type: Optional[Channel]
-        ci = ChannelInfo.from_raw_msg(chan.construct_channel_announcement_without_sigs())
-        return ci._replace(capacity_sat=chan.constraints.capacity)
+        if my_channels:
+            return get_mychannel_info(short_channel_id, my_channels)
 
     def get_channels_for_node(self, node_id: bytes, *,
                               my_channels: Dict[ShortChannelID, 'Channel'] = None) -> Set[bytes]:
         """Returns the set of short channel IDs where node_id is one of the channel participants."""
+        if not self.data_loaded.is_set():
+            raise Exception("channelDB data not loaded yet!")
         relevant_channels = self._channels_for_node.get(node_id) or set()
         relevant_channels = set(relevant_channels)  # copy
         # add our own channels  # TODO maybe slow?
@@ -635,3 +750,60 @@ class ChannelDB(SqlDB):
             if node_id in (chan.node_id, chan.get_local_pubkey()):
                 relevant_channels.add(chan.short_channel_id)
         return relevant_channels
+
+    def get_endnodes_for_chan(self, short_channel_id: ShortChannelID, *,
+                              my_channels: Dict[ShortChannelID, 'Channel'] = None) -> Optional[Tuple[bytes, bytes]]:
+        channel_info = self.get_channel_info(short_channel_id)
+        if channel_info is not None:  # publicly announced channel
+            return channel_info.node1_id, channel_info.node2_id
+        # check if it's one of our own channels
+        if not my_channels:
+            return
+        chan = my_channels.get(short_channel_id)  # type: Optional[Channel]
+        if not chan:
+            return
+        return chan.get_local_pubkey(), chan.node_id
+
+    def get_node_info_for_node_id(self, node_id: bytes) -> Optional['NodeInfo']:
+        return self._nodes.get(node_id)
+
+    def get_node_infos(self) -> Dict[bytes, NodeInfo]:
+        with self.lock:
+            return self._nodes.copy()
+
+    def get_node_policies(self) -> Dict[Tuple[bytes, ShortChannelID], Policy]:
+        with self.lock:
+            return self._policies.copy()
+
+    def to_dict(self) -> dict:
+        """ Generates a graph representation in terms of a dictionary.
+
+        The dictionary contains only native python types and can be encoded
+        to json.
+        """
+        with self.lock:
+            graph = {'nodes': [], 'channels': []}
+
+            # gather nodes
+            for pk, nodeinfo in self._nodes.items():
+                # use _asdict() to convert NamedTuples to json encodable dicts
+                graph['nodes'].append(
+                    nodeinfo._asdict(),
+                )
+                graph['nodes'][-1]['addresses'] = [addr._asdict() for addr in self._addresses[pk]]
+
+            # gather channels
+            for cid, channelinfo in self._channels.items():
+                graph['channels'].append(
+                    channelinfo._asdict(),
+                )
+                policy1 = self._policies.get(
+                    (channelinfo.node1_id, channelinfo.short_channel_id))
+                policy2 = self._policies.get(
+                    (channelinfo.node2_id, channelinfo.short_channel_id))
+                graph['channels'][-1]['policy1'] = policy1._asdict() if policy1 else None
+                graph['channels'][-1]['policy2'] = policy2._asdict() if policy2 else None
+
+        # need to use json_normalize otherwise json encoding in rpc server fails
+        graph = json_normalize(graph)
+        return graph

electrum/checkpoints.json

@@ -1166,5 +1166,145 @@
     [
         "000000000000000000096b8d24db6471fb5871e9ae8bd1d7384fbee9c80a6052",
         2699909434228155498652331786772923585210445951064342528
+    ],
+    [
+        "00000000000000000016e0dd8fe86bf34feaa611b4c52180b6822b5ad31b68ff",
+        2647377219375933524160418539145769508351933111739613184
+    ],
+    [
+        "00000000000000000011e20e47a868d12a2bf3de814ebd067e83514aa2725745",
+        2502742632840755378666227277045667991877723059489079296
+    ],
+    [
+        "0000000000000000000c48f6bed594da7bb5e75731b4e78501670e834d426e87",
+        2267299103571658911252368261549572946260211294613274624
+    ],
+    [
+        "0000000000000000000f7871dc40f51b1ecd6343a6d9fd614d0e2235a7d9e3fd",
+        2112846149036891759953684644743283440459952687539027968
+    ],
+    [
+        "0000000000000000001558c0f33a360d105b52a749103eb2abd4a66a68d52664",
+        2072520395859657486634608572838975759381606196813234176
+    ],
+    [
+        "0000000000000000000676463abf3771ea01e0f8c948d1c93658a1d82d95df5a",
+        1969073848467738847181233556694484530967339635488849920
+    ],
+    [
+        "0000000000000000000e24396612da4ec125ee6c0b4507e854c5cfed1884cd30",
+        2119459443945814095658556318611324621123895782295994368
+    ],
+    [
+        "00000000000000000002fb021eeb13e47021920faf6e5daa3c40bc552c4d248e",
+        2078088717097888226752964612051624797686495299801972736
+    ],
+    [
+        "000000000000000000067b904af747b653ba448a79779f7846bf1ea5537b8a4d",
+        2093644940525638357414324633411056914147713045789409280
+    ],
+    [
+        "000000000000000000080ae07ccf2f1b6d1d089f5dcbc1fac50a6b93d005f1e0",
+        2082043540528505650049623783208955059537684253263265792
+    ],
+    [
+        "00000000000000000008f9ddf24dbec1459689fc399329e9738b2795860e4361",
+        1953761695813422977307213550702116033770404430236090368
+    ],
+    [
+        "0000000000000000000aacba541ebb7b56b0831e4ae33faf20ff1e528bb9a657",
+        1824503568004603261415443256727022530945994444270206976
+    ],
+    [
+        "00000000000000000010fe23dd08a4b6465c4850984bb538e9dfcb93995a23cc",
+        1743137387349479903250289511035208906392689711805104128
+    ],
+    [
+        "0000000000000000001166c174a9d34b0743953e724162fe44388e38d078204c",
+        1734095076719313606895363312975193263350078457161711616
+    ],
+    [
+        "00000000000000000006da92c61b6b63ea910be27cab5fd951137105314f2969",
+        1740794600224838465872409004248364704712181251938713600
+    ],
+    [
+        "000000000000000000043f26353c41c2343a277ad72f115171fb49d3be52dbbc",
+        1628687194130096895725758951785196783123433634364653568
+    ],
+    [
+        "0000000000000000000bc6800858a1b3be08fb26b55d4b989c95e06ad50a350c",
+        1937788944419033539314165479165359776648584743473905664
+    ],
+    [
+        "0000000000000000000c799dc0e36302db7fbb471711f140dc308508ef19e343",
+        1832085838499075985755083973639154607251969422303166464
+    ],
+    [
+        "0000000000000000000de98650125747f239134cf7e2b7362033e325a8003a14",
+        1689336589076054705025375464973257095873115523033071616
+    ],
+    [
+        "0000000000000000001138f586983520b0de3645c0873164f4b214b90cf3aedc",
+        1674005436900453533413418811078063286996924790657253376
+    ],
+    [
+        "0000000000000000000e87ecbff47d9ab75e78d92328d5951351f9702597dace",
+        1780912820169571750977100152906426673601736600243404800
+    ],
+    [
+        "00000000000000000007c4dac98234149700771e9d1756956660b63cca88c36b",
+        1963213226902041926479236780515292236058519345991516160
+    ],
+    [
+        "00000000000000000003030a3de58b57be352e2ca79016cefe19777e02ba0520",
+        1707948812427463753688699391317898960128433823967870976
+    ],
+    [
+        "0000000000000000000cfd1300625612513c6cd1413245fcdaf1eeb766e33a93",
+        1708005810991319658902509335026374895166200405337047040
+    ],
+    [
+        "0000000000000000000830b0a5ac4b78b5eb99209ebb4790be1fae1428c7f77c",
+        1554226608711362053849117616927967595838003183165112320
+    ],
+    [
+        "0000000000000000000ed5cf2e86791b44abce69e178e58613e64ed47e1c02a3",
+        1600203988720154928752887338080389143353359165034594304
+    ],
+    [
+        "0000000000000000000aac5c93f7945c60d82828990448cde97d3d7128830a6d",
+        1590739304116800001454600275103718494518067345886281728
+    ],
+    [
+        "000000000000000000049a66ca322371799e1cb51d85c8937764ba6a2abb8ed9",
+        1535456543183121267670627692621392373016562041515671552
+    ],
+    [
+        "0000000000000000000657c7aa925caa49d18e0c02cab9992be315012d8fab06",
+        1554222224206450061140363005873469446988944215367483392
+    ],
+    [
+        "000000000000000000061250f1186194229157967d10a01a2b36ab19d4304da5",
+        1395807138732878832030429199485686097922398375169228800
+    ],
+    [
+        "0000000000000000000d2e17e6d3179b4182518bd678f20bbda8b29e5e494d54",
+        1397005570075490172423356221048513449998516239854469120
+    ],
+    [
+        "00000000000000000005e2dea23567cb4fe092a354e7d1b50b59571715de22f6",
+        1348156339349342073285316259199804406349536350538039296
+    ],
+    [
+        "00000000000000000005e17383e25f65b531d50060b99ed66f673ea251949e4b",
+        1605902383604108119230963505243149930846997646019657728
+    ],
+    [
+        "000000000000000000090386439b3e1c7dc56d2e450694e910b366895f05b9ef",
+        1532070243889425565609149754863988745260019245813596160
+    ],
+    [
+        "000000000000000000046f183ba323cfceb2d11660376c59fb55e8521c4d32a5",
+        1407282849589201081744164532792174352192736757496676352
     ]
 ]

electrum/checkpoints_testnet.json

@@ -3118,5 +3118,645 @@
     [
         "000000000000006b039683c36b18ec712346521edce4dc5b81cdaf6475d89bd7",
         0
+    ],
+    [
+        "00000000000000525de83fba2439549ef0ed78d6d08516a0513abb972b0fca95",
+        0
+    ],
+    [
+        "000000000000006c5403ae9c42acf37362885c75c1a71a6b7fe20f9cfc5304a7",
+        0
+    ],
+    [
+        "000000000000006f881a62bc5ec9d4c4da83ddc6619a7eee82617e26e2c7ef3c",
+        0
+    ],
+    [
+        "000000000000012941300197c5b6627a66f9cf48ae9c6791b36c63c0218a1be9",
+        0
+    ],
+    [
+        "00000000000002cd7ec2e00992a4dc6c5e0a56cfbc19b5afa9730bd94f174b5b",
+        0
+    ],
+    [
+        "000000000022e09ee2ee7b3fd223cb9ccfe11058cca5ad0c705fe5a0c26b28dc",
+        0
+    ],
+    [
+        "0000000007d35ebaf81412d40d1224bdc5792bfbc70827c09f05dc5fb168e67f",
+        0
+    ],
+    [
+        "00000000328e1b1aecf68947ad53fb11c58a383704ddbb8b29704669e22225bd",
+        0
+    ],
+    [
+        "000000000003d3b3f171fd10fda1be9d4464b1438bb9443081c2c224a047cc4e",
+        0
+    ],
+    [
+        "000000000001e3c5dcea0586d3c8f69c0f35658fae283d29f64df9b5301bc721",
+        0
+    ],
+    [
+        "00000000000ce5f3757a0cab09a8cb131b3f2c63303375ad1c84fe423866d33f",
+        0
+    ],
+    [
+        "00000000000ca01b96070fb643bcebbc862cff4da78dcd52de1418c940d4f466",
+        0
+    ],
+    [
+        "0000000000006eb74e5036cf42888759c4ebf91a5eb128463e60ae9ab02876a3",
+        0
+    ],
+    [
+        "000000000003aae0765dfee956b322477d786a2cde617ff073e0bc4eeaf7c252",
+        0
+    ],
+    [
+        "00000000000033421d804b4bc0f7dc61715d2fc0cc2a98904ff5e1f9ef909010",
+        0
+    ],
+    [
+        "0000000000002a24b916b5f03bd47250276ad32f08a1684334c7f181b0b7a055",
+        0
+    ],
+    [
+        "00000000000002a7399ec806255c4ae63d7583001bbde70e2038e9b90fb824f4",
+        0
+    ],
+    [
+        "00000000000000ec89aaa13c7222b3ec787a487cdc7a17c1ee87ce313e6ed4d3",
+        0
+    ],
+    [
+        "00000000000001564cf9db3397bd0983a68f450d5b7e59824339fe1d46ba1c75",
+        0
+    ],
+    [
+        "00000000000e932953388774b6b3492d8756f936d74fda1d33eace33538fb0bb",
+        0
+    ],
+    [
+        "0000000084c2d56f703e72f6ad637105409552792ee482bbc14376cfb29c30d9",
+        0
+    ],
+    [
+        "00000000392f30ba333fac2e4937e162105ba2b20fe953848b1a4c004f460223",
+        0
+    ],
+    [
+        "00000000000842b42c56e4dc573efd9b6b6864dba81730c4f95b837d52078ad5",
+        0
+    ],
+    [
+        "0000000003e4cca12f6109687fcccfc5c3827bf3bca2487096fec0293b4b351e",
+        0
+    ],
+    [
+        "00000000007b7eece3ebbf77ed583a711c8427284ea9b556ec67efd14e7f5d90",
+        0
+    ],
+    [
+        "000000000002c0e026657401be7998fce1618869ec073a49ac935a15d16c5741",
+        0
+    ],
+    [
+        "00000000000cf19ef67151f6d06b426371dfa63d9d2bbd6024cca520cf4d96b4",
+        0
+    ],
+    [
+        "0000000000019a6ef183423833a4347d77e8687b4fc83a85f4c98c579631acbe",
+        0
+    ],
+    [
+        "000000000000a292b9ff43becd4770243d2750e2b3c4e81a6ed79b8abd2f5052",
+        0
+    ],
+    [
+        "000000000000280db4a9a31097024bc81f0358ba624f1f8dd83a2362a156a817",
+        0
+    ],
+    [
+        "00000000000009b17b295d898cda8899ce547183fd63fa901b9f502aed00c45d",
+        0
+    ],
+    [
+        "0000000000000013f5c40f6b0e7e8fe854045135564a4df6ff4ca736861d7ea8",
+        0
+    ],
+    [
+        "000000000000c39ffca7d1daad0d4f8af9ee108443bb1b4352cd740fd8297aef",
+        0
+    ],
+    [
+        "000000000002f42ee90d7d459393eb90e2ea5a3ed292394ce1dc5f7a42d66ce0",
+        0
+    ],
+    [
+        "0000000000010d6bd31805e0a9b8629192c0ad704641d2b08c28865052bbf469",
+        0
+    ],
+    [
+        "0000000001015f5067612dc0d681d71b33d278c50ca88d7756322ab90f753290",
+        0
+    ],
+    [
+        "000000000003dadd324301ee6157c29e7aa9f120edefaf05369d849510e6d60c",
+        0
+    ],
+    [
+        "000000000000a62107ea11c5db9929d819181d8903624e9088b8700d1dc66ea7",
+        0
+    ],
+    [
+        "00000000000022b91e1b652f626cd3a81bfb2ff70717ace53c488dd45c75fcbb",
+        0
+    ],
+    [
+        "0000000000002845027a6a08c436c6e99aa8af0f7c744a722fd598ba0f66f4cb",
+        0
+    ],
+    [
+        "000000000000ae5347baecbcb3cd01265f0e52c8819f830dcfc6dafa1ec4327a",
+        0
+    ],
+    [
+        "0000000000008dd3169522647ae90ca0a3acc405f0e8c2b53dab013433708921",
+        0
+    ],
+    [
+        "00000000000023abea5dd709951fb1fa5c34a75670ddc7eea46d2d23c6033669",
+        0
+    ],
+    [
+        "00000000000006fe20edd4be3beabc4432fbe410ab53466660105ced53056190",
+        0
+    ],
+    [
+        "000000000000003f6d6889d2917ba88f6e286c156028baebf05be409e1b97ef8",
+        0
+    ],
+    [
+        "000000000000005d871f102aaa25e60855c96c1aa8404f004db1c8bbfab341e9",
+        0
+    ],
+    [
+        "0000000000000197fac06dd6c7f80c838b6a21f1ce72f10aa6ba0aff40c3cb92",
+        0
+    ],
+    [
+        "0000000000000289a999cf132efbee896d8c22e2f9d1036381b00d72c41660e3",
+        0
+    ],
+    [
+        "00000000e9f6bd4700dea0c0841272461e4e9d125b8fe2c35a2ca39f77269321",
+        0
+    ],
+    [
+        "00000000f91f03ac1d08214a3646c2bef1878961a8c40d867254d733fd9cb2a3",
+        0
+    ],
+    [
+        "000000003d42ef351c6a1fb5e2d43d1a28ca095052be35ad9bb901b097c667c8",
+        0
+    ],
+    [
+        "000000000014b426a9844698b6369c0e2befe4e369f1dd01c157dbdd472c9136",
+        0
+    ],
+    [
+        "000000000016dfa525db05b9db92a080e0da65a4a0b15e538649eb4c0c670cf4",
+        0
+    ],
+    [
+        "0000000000027a82eb5b1ab46a276a9aa19e3a1e52e2328c07a50db314664148",
+        0
+    ],
+    [
+        "000000000026945c53ba1f9b0c34f9e502f3aa64c9979ce583b93daf347d2292",
+        0
+    ],
+    [
+        "00000000000f64a42d38e16119aa724e6d859d8b7ed2964bd0929a226e57c838",
+        0
+    ],
+    [
+        "0000000000011bee42dca16315be14fd0be451e4385c787a66c7dc6c0a498ce2",
+        0
+    ],
+    [
+        "0000000000007fcace99545546c5ee4df862e21840543865ad0944ca7b82baf7",
+        0
+    ],
+    [
+        "0000000000003b3a9be8e418e11db77aa16dbf9f04a9b43b34466e7b41520fa2",
+        0
+    ],
+    [
+        "00000000000004ae741f8cd7f6f20231f8be6b89946e50339f0089a2e5c6d4d6",
+        0
+    ],
+    [
+        "0000000000000379b21385de297e65a62e4d15ee27fbf1e3b4fa7a46b4a274ba",
+        0
+    ],
+    [
+        "000000000001fd6b7db603c305be360c602800e5d9068bd65bae111b4561d5ab",
+        0
+    ],
+    [
+        "000000003925c7eb3144eb77e7891a607152b662b161cd4a052e2a5689c4b694",
+        0
+    ],
+    [
+        "000000000000a8476194924cd6612277821149e22f7326a054c09c7d55b8a9d5",
+        0
+    ],
+    [
+        "0000000009ddc12332eb5903b89ddfd116bfd9b300c4d70821e749a302fa438b",
+        0
+    ],
+    [
+        "0000000000028fe3bfc47a9ad8a71c90fa3edea0c1d04f823c5a9d8674b9d1c5",
+        0
+    ],
+    [
+        "000000000000075849c07342e632fa3f2b4e137de35703e91c62cb568a8583ea",
+        0
+    ],
+    [
+        "0000000000001100406d8447ce19989346956134e2dabb87f93ff1b32208dc21",
+        0
+    ],
+    [
+        "0000000000006a8a2fd9d16a22f28523940811b3c4f179f888249b6f5f19c708",
+        0
+    ],
+    [
+        "000000000001af7c8a48d294945d937c3f1ab297617bab1a0eb1d9a40e543139",
+        0
+    ],
+    [
+        "00000000000040eafb8f54cb988a19d0370379be0b2917787e640720677ba6de",
+        0
+    ],
+    [
+        "000000000000025f7bc6cb5759f267fd649620c69f6518213729bb6aeb4d98d3",
+        0
+    ],
+    [
+        "0000000000000217a8588f1af88d2f73a96a658f0aea62de5c53b5b348346456",
+        0
+    ],
+    [
+        "00000000000001b8aa8353bbafb6f47125f67a711c0a2a7a00bfebff5a8df093",
+        0
+    ],
+    [
+        "000000004ca77c8921259d7da52f341526df3f34edb62e3e2888b7ce42b8c29f",
+        0
+    ],
+    [
+        "000000005c8253a86af2492291e888d78d0a69a7a657a221e59b23eb6291fcff",
+        0
+    ],
+    [
+        "0000000000fba14ebb3757a9348a05b07ec207b25aaffeac4118237e665fc566",
+        0
+    ],
+    [
+        "0000000008f01a3c024cb6d1814e54659c72b17e34e2b60fd35af2184b6bd3ea",
+        0
+    ],
+    [
+        "0000000003da1325f0d607889753f3a7214c3e559b9834c6f0e37bd52e14eaec",
+        0
+    ],
+    [
+        "0000000000d303f0b50fc25ea141ad3c26d0dfe61fa4cfcc6875edbcef902163",
+        0
+    ],
+    [
+        "00000000002131de3bcff721c93c169e34450054c18fc02cd5a8e08c7c3fd567",
+        0
+    ],
+    [
+        "00000000000c69cdb751a4ef5f527ae244909ddfda10a4caed4d6f8dd44e51fe",
+        0
+    ],
+    [
+        "0000000000024819bfbc99fd2032441181dcb2456ada1d047c4b6b7829be62a0",
+        0
+    ],
+    [
+        "00000000000077021c5164bc1014b24abd321f160bb914a1257a86645f923385",
+        0
+    ],
+    [
+        "00000000000038e149b42e964bdeb10f01fbbfd38ce57ec25eb3fdfb712cf9b0",
+        0
+    ],
+    [
+        "000000000000047dd3d1ce9862add6979aa622a7cb2141b4c6ec569b172dd776",
+        0
+    ],
+    [
+        "0000000090c401521295d1040e0f9b6cb65da914085bb9346e60477837dab234",
+        0
+    ],
+    [
+        "00000000f36784781eaf4b0d3ef92525b6cf55e910c782bd4f355b71ee40dc36",
+        0
+    ],
+    [
+        "000000001d3848f040d48696a9e258798bea34969e810ad01e8092183f201dfd",
+        0
+    ],
+    [
+        "0000000007658642f1e8ac45feec2766358f425030b14ad824f3a6df30b9eb15",
+        0
+    ],
+    [
+        "00000000028e5b819d9e197b1d3f1246a2a6990d8e2360371dbf258c2c5861fb",
+        0
+    ],
+    [
+        "00000000002a8dbd19a807d955c7d01962fea32f5ae027345121176ac10c20f4",
+        0
+    ],
+    [
+        "0000000000144908febd5cbacd1d9b828817f0350211be3248a1ec2d3ac3e251",
+        0
+    ],
+    [
+        "00000000000a302f19d696c7be172c6ac92ec2adf956417bba482d3e5285e5d7",
+        0
+    ],
+    [
+        "000000000000a289eb62cae8c41644d7c9de31148f711744aa5409164b90d6e3",
+        0
+    ],
+    [
+        "000000000000036a6f6002c633b6be318745d2f2ff1520daa6a49db7649bca67",
+        0
+    ],
+    [
+        "0000000000000293db488f4a3c7289489664e6e7e1ec917dc58c83ec828a4730",
+        0
+    ],
+    [
+        "0000000000000e24d4ce3b9247d6316791438ab82ea755e788112bb9729730cf",
+        0
+    ],
+    [
+        "00000000000003a18b92493908ebe4ccecf24bfeda95bf3b8a026e3c01af116a",
+        0
+    ],
+    [
+        "0000000000000007a2b7ba9dd58c20651b477daf83df5a7ac24b856b22f1fb25",
+        0
+    ],
+    [
+        "000000000000000ce321e0271dd532a6ce58737151baa84a77a585df614c2ab6",
+        0
+    ],
+    [
+        "000000000000004ebec3379d6a8569295a2d0a0c0e0c815d2b01803315032185",
+        0
+    ],
+    [
+        "000000000000001bb9ed28d9b0a70fee0b6d42f91f3db53f2086eef4daabce30",
+        0
+    ],
+    [
+        "000000007c5711c573d147a6fae21faf529c039220c97dfe2ba96e732d88fa89",
+        0
+    ],
+    [
+        "000000008e5a5e820d1a10dbeecf6f6df3bf7ab56e46eec275d8ca1a52e86b68",
+        0
+    ],
+    [
+        "000000003fa06ace5db33de18cf03b0c56d4e62cdaf8ab533919953c22bffaf1",
+        0
+    ],
+    [
+        "000000000000e6442b0c74fa811319edf2edd5f8d9b2e3ee831b4bdee644fbd0",
+        0
+    ],
+    [
+        "00000000011d0c3f98e9c3db6b51468be632bdef0c47f5e45871b771e5b0bc57",
+        0
+    ],
+    [
+        "000000000000e3c0978d872ed3b3a43f6f319995459105159b5f4e92143d40d2",
+        0
+    ],
+    [
+        "000000000000cdf25c3e15601dcb798c6cf8d2dd89002a4e046b746be6b87fa0",
+        0
+    ],
+    [
+        "000000000000521507052d13f4fac6c01c0099466720bea95c2e9349aef7fa5f",
+        0
+    ],
+    [
+        "00000000000064823750f1a6b7cd1748dfcc73376086cfdba987d2a36fcddb71",
+        0
+    ],
+    [
+        "0000000000000b4a41be0612f47a58efb899dc1cc0965c1c1fac89e1ea69f587",
+        0
+    ],
+    [
+        "00000000000010aab857bf7d475d9a594dca8b1144597a9e69c70f20fdd20b4f",
+        0
+    ],
+    [
+        "0000000000000c264f193e8d5099f2c20c08fdf9e5ca9006fb53778c0d8eb869",
+        0
+    ],
+    [
+        "00000000000002adcce72a5cce517f1afc33c765927b77ccbce5cdc6f5f68e45",
+        0
+    ],
+    [
+        "00000000b179a6096a58938311b3b8cc4479ccdf3909667a58598acc4ebd0192",
+        0
+    ],
+    [
+        "000000004e86c06d23b8a4c20e6cb5a4c51cad24fca30e41695f8ad00852a88e",
+        0
+    ],
+    [
+        "000000000bafa134d62d9df490ffdbc1f2b86b4373b86c079c5b730034aad214",
+        0
+    ],
+    [
+        "00000000033e9b623ca1d89418114f63af55e042dafbfe97952e7a5fe7a3ebf4",
+        0
+    ],
+    [
+        "000000000119025b6c9bbc3390708b1a77e85eda69fcb79666418ac2cb874a17",
+        0
+    ],
+    [
+        "000000000000feafbf3a525a1dd7950fa53f7df1b0210e79337ce588d35a8b9a",
+        0
+    ],
+    [
+        "0000000000007044088a1cc9ddc0c3779c0e156dee10fa15a760897ed4249f8f",
+        0
+    ],
+    [
+        "000000000001a10e8b1ad577278f946252298b49b74ac9db70ea80c0a9c12db3",
+        0
+    ],
+    [
+        "000000000001281354a7d86b3c750681283276c0bdde2b18c38d8354138ca4e1",
+        0
+    ],
+    [
+        "0000000000000398b17fcd5d4d59ccb31d642f7b60c2a4d4d2aa7239ebc0efa9",
+        0
+    ],
+    [
+        "00000000000021a571a2c475115fe723b593633efb85bf0ec0f7d67b780e70c3",
+        0
+    ],
+    [
+        "00000000000002d1506c82becd7b480c85402d27f23a1248cfa128b7a8c009a6",
+        0
+    ],
+    [
+        "00000000000001978f804f5cf8e4a0dc0c454fce0f0e2614510b8eae6e504b2e",
+        0
+    ],
+    [
+        "00000000000001c4558889a43ac35208f502bccd9d38c741571723e9d79bcc26",
+        0
+    ],
+    [
+        "000000000000005c782bbbc75358216e1ffc37973cd43a474b87dfbac4c61fab",
+        0
+    ],
+    [
+        "0000000053bffe3e3db3672c5f050fa54239f93833ec5c38af92e83dec71a9fc",
+        0
+    ],
+    [
+        "000000000001362fd5182f1cbfc1981937cd67ba54bc7b6d7f0a68f94e369f0a",
+        0
+    ],
+    [
+        "000000000386ae84caa25e9dfa7816594b7c30a079e340bfcd951be2b5c092b2",
+        0
+    ],
+    [
+        "0000000003cc09a351d647c0e12063d45b20e6f99c27c18ea62342b9d246581d",
+        0
+    ],
+    [
+        "0000000002527c4756350bafee88786cd7ea27bc802f482c4e50cafc547ff9f7",
+        0
+    ],
+    [
+        "00000000003d7288f44aa0b725af7816d2d333e118de12c390423d641139d5d5",
+        0
+    ],
+    [
+        "000000000008c0a0fadcfbe27a880ce9c387425d3a2c6b06c1a599e4ce51ec92",
+        0
+    ],
+    [
+        "00000000000158ab2486a8f1251c5c94502763ced9eb85847bb9d2eb476b515a",
+        0
+    ],
+    [
+        "000000000000c817e5775378accf08412657e2557d2895df0fbb8475b5e190ba",
+        0
+    ],
+    [
+        "00000000000078d59d08215b3aecdf0e0665d3a16ae1716e408df790a3566e72",
+        0
+    ],
+    [
+        "0000000000002208404b39b95cc20845de19b47e05e8146146056d3d9bb382ae",
+        0
+    ],
+    [
+        "0000000000000543e9315ca8b3b72bd3590f24535e4ddc6ccb1050b607777530",
+        0
+    ],
+    [
+        "00000000000000abb8d3ffd3cc347cee5c092dde5355a7dc5d288036a28760fb",
+        0
+    ],
+    [
+        "000000000000008bfbcce7d768df6f4610205dcb40173e8c4c417a2325487f34",
+        0
+    ],
+    [
+        "00000000209e49391ad09577f87d1e0ffda27d2e749fd305c51692112627c99d",
+        0
+    ],
+    [
+        "000000000005561eb4b2e0cb8107c81617284e7bcd7d390d16a3cd5925cf42a9",
+        0
+    ],
+    [
+        "00000000006b24215c790a371bc18c53c83ff35e2c82d459bb6240cd9615dde5",
+        0
+    ],
+    [
+        "0000000000af315d6fbde8488d68dbd055a56d79555ed32c3ad4d70286b4df2a",
+        0
+    ],
+    [
+        "00000000019e49bc89fcabc4050521fb8835f926a62cc10b68e9618ffc117162",
+        0
+    ],
+    [
+        "00000000009c0dcde4e694463245e8e5e45d2897e7fa67772ce0ef37094f3afd",
+        0
+    ],
+    [
+        "000000000005efbda8c010f29a5b81606d186459047ce4b7eacde8d9659dce97",
+        0
+    ],
+    [
+        "0000000000051c1655579a441a7f4d543c323d482405cf1d1250c3ccb665d426",
+        0
+    ],
+    [
+        "0000000000007f13adadd1fc6462fbc5231425b81826af4e5f0cbb0de54a5b3a",
+        0
+    ],
+    [
+        "00000000000011e00df09353fcb53766447279b96228da0525d769f33026bebb",
+        0
+    ],
+    [
+        "0000000000002b91e6bb56015e0e60dc650a63666aa3943058e9641d4d679fa3",
+        0
+    ],
+    [
+        "00000000000008e4d5fbcf207583267efff33e6c8d0a5fbdaa5704aeb674fe29",
+        0
+    ],
+    [
+        "000000000000018aeeabcb422b5b0a46cf3a5f2458125c043c5781ffafeffbf9",
+        0
+    ],
+    [
+        "000000000000004ca501cc9138ef5fef4b7b235682b81ab9719b3cf215e94f73",
+        0
+    ],
+    [
+        "000000000000002b5bb1c4c43059575556a0ed10099ce5095f805d3d9ae10cab",
+        0
     ]
 ]

electrum/coinchooser.py

@@ -44,12 +44,12 @@ class PRNG:
         self.sha = sha256(seed)
         self.pool = bytearray()
 
-    def get_bytes(self, n):
+    def get_bytes(self, n: int) -> bytes:
         while len(self.pool) < n:
             self.pool.extend(self.sha)
             self.sha = sha256(self.sha)
         result, self.pool = self.pool[:n], self.pool[n:]
-        return result
+        return bytes(result)
 
     def randint(self, start, end):
         # Returns random integer in [start, end)
@@ -103,10 +103,9 @@ def strip_unneeded(bkts: List[Bucket], sufficient_funds) -> List[Bucket]:
 
 class CoinChooserBase(Logger):
 
-    enable_output_value_rounding = False
-
-    def __init__(self):
+    def __init__(self, *, enable_output_value_rounding: bool):
         Logger.__init__(self)
+        self.enable_output_value_rounding = enable_output_value_rounding
 
     def keys(self, coins: Sequence[PartialTxInput]) -> Sequence[str]:
         raise NotImplementedError
@@ -121,7 +120,7 @@ class CoinChooserBase(Logger):
         constant_fee = fee_estimator_vb(2000) == fee_estimator_vb(200)
 
         def make_Bucket(desc: str, coins: List[PartialTxInput]):
-            witness = any(Transaction.is_segwit_input(coin, guess_for_address=True) for coin in coins)
+            witness = any(coin.is_segwit(guess_for_address=True) for coin in coins)
             # note that we're guessing whether the tx uses segwit based
             # on this single bucket
             weight = sum(Transaction.estimated_input_weight(coin, witness)
@@ -485,6 +484,12 @@ def get_name(config):
 
 def get_coin_chooser(config):
     klass = COIN_CHOOSERS[get_name(config)]
-    coinchooser = klass()
-    coinchooser.enable_output_value_rounding = config.get('coin_chooser_output_rounding', False)
+    # note: we enable enable_output_value_rounding by default as
+    #       - for sacrificing a few satoshis
+    #       + it gives better privacy for the user re change output
+    #       + it also helps the network as a whole as fees will become noisier
+    #         (trying to counter the heuristic that "whole integer sat/byte feerates" are common)
+    coinchooser = klass(
+        enable_output_value_rounding=config.get('coin_chooser_output_rounding', True),
+    )
     return coinchooser

electrum/commands.py

@@ -39,25 +39,28 @@ from decimal import Decimal
 from typing import Optional, TYPE_CHECKING, Dict, List
 
 from .import util, ecc
-from .util import bfh, bh2u, format_satoshis, json_decode, json_encode, is_hash256_str, is_hex_str, to_bytes, timestamp_to_datetime
-from .util import standardize_path
+from .util import (bfh, bh2u, format_satoshis, json_decode, json_normalize,
+                   is_hash256_str, is_hex_str, to_bytes)
 from . import bitcoin
 from .bitcoin import is_address,  hash_160, COIN
 from .bip32 import BIP32Node
 from .i18n import _
 from .transaction import (Transaction, multisig_script, TxOutput, PartialTransaction, PartialTxOutput,
                           tx_from_any, PartialTxInput, TxOutpoint)
-from .paymentrequest import PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED
+from .invoices import PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED
 from .synchronizer import Notifier
-from .wallet import Abstract_Wallet, create_new_wallet, restore_wallet_from_text
+from .wallet import Abstract_Wallet, create_new_wallet, restore_wallet_from_text, Deterministic_Wallet
 from .address_synchronizer import TX_HEIGHT_LOCAL
 from .mnemonic import Mnemonic
 from .lnutil import SENT, RECEIVED
+from .lnutil import LnFeatures
 from .lnutil import ln_dummy_address
 from .lnpeer import channel_id_from_funding_tx
 from .plugin import run_hook
 from .version import ELECTRUM_VERSION
 from .simple_config import SimpleConfig
+from .invoices import LNInvoice
+from . import submarine_swaps
 
 
 if TYPE_CHECKING:
@@ -68,17 +71,16 @@ if TYPE_CHECKING:
 known_commands = {}  # type: Dict[str, Command]
 
 
+class NotSynchronizedException(Exception):
+    pass
+
+
 def satoshis(amount):
     # satoshi conversion must not be performed by the parser
     return int(COIN*Decimal(amount)) if amount not in ['!', None] else amount
 
-
-def json_normalize(x):
-    # note: The return value of commands, when going through the JSON-RPC interface,
-    #       is json-encoded. The encoder used there cannot handle some types, e.g. electrum.util.Satoshis.
-    # note: We should not simply do "json_encode(x)" here, as then later x would get doubly json-encoded.
-    # see #5868
-    return json_decode(json_encode(x))
+def format_satoshis(x):
+    return str(Decimal(x)/COIN) if x is not None else None
 
 
 class Command:
@@ -100,6 +102,16 @@ class Command:
             self.options = []
             self.defaults = []
 
+        # sanity checks
+        if self.requires_password:
+            assert self.requires_wallet
+        for varname in ('wallet_path', 'wallet'):
+            if varname in varnames:
+                assert varname in self.options
+        assert not ('wallet_path' in varnames and 'wallet' in varnames)
+        if self.requires_wallet:
+            assert 'wallet' in varnames
+
 
 def command(s):
     def decorator(func):
@@ -113,18 +125,20 @@ def command(s):
             password = kwargs.get('password')
             daemon = cmd_runner.daemon
             if daemon:
-                if (cmd.requires_wallet or 'wallet_path' in cmd.options) and kwargs.get('wallet_path') is None:
-                    # using JSON-RPC, sometimes the "wallet" kwarg needs to be used to specify a wallet
-                    kwargs['wallet_path'] = kwargs.pop('wallet', None) or daemon.config.get_wallet_path()
-                if cmd.requires_wallet:
-                    wallet_path = kwargs.pop('wallet_path')
-                    wallet = daemon.get_wallet(wallet_path)
-                    if wallet is None:
-                        raise Exception('wallet not loaded')
-                    kwargs['wallet'] = wallet
-            else:
-                # we are offline. the wallet must have been passed if required
-                wallet = kwargs.get('wallet')
+                if 'wallet_path' in cmd.options and kwargs.get('wallet_path') is None:
+                    kwargs['wallet_path'] = daemon.config.get_wallet_path()
+                if cmd.requires_wallet and kwargs.get('wallet') is None:
+                    kwargs['wallet'] = daemon.config.get_wallet_path()
+                if 'wallet' in cmd.options:
+                    wallet_path = kwargs.get('wallet', None)
+                    if isinstance(wallet_path, str):
+                        wallet = daemon.get_wallet(wallet_path)
+                        if wallet is None:
+                            raise Exception('wallet not loaded')
+                        kwargs['wallet'] = wallet
+            wallet = kwargs.get('wallet')  # type: Optional[Abstract_Wallet]
+            if cmd.requires_wallet and not wallet:
+                raise Exception('wallet not loaded')
             if cmd.requires_password and password is None and wallet.has_password():
                 raise Exception('Password required')
             return await func(*args, **kwargs)
@@ -181,7 +195,7 @@ class Commands:
         net_params = self.network.get_parameters()
         response = {
             'path': self.network.config.path,
-            'server': net_params.host,
+            'server': net_params.server.host,
             'blockchain_height': self.network.get_local_height(),
             'server_height': self.network.get_server_height(),
             'spv_nodes': len(self.network.get_interfaces()),
@@ -279,6 +293,7 @@ class Commands:
     def _setconfig_normalize_value(cls, key, value):
         if key not in ('rpcuser', 'rpcpassword'):
             value = json_decode(value)
+            # call literal_eval for backward compatibility (see #4225)
             try:
                 value = ast.literal_eval(value)
             except:
@@ -289,14 +304,24 @@ class Commands:
     async def setconfig(self, key, value):
         """Set a configuration variable. 'value' may be a string or a Python expression."""
         value = self._setconfig_normalize_value(key, value)
+        if self.daemon and key == 'rpcuser':
+            self.daemon.commands_server.rpc_user = value
+        if self.daemon and key == 'rpcpassword':
+            self.daemon.commands_server.rpc_password = value
         self.config.set_key(key, value)
         return True
 
     @command('')
-    async def make_seed(self, nbits=132, language=None, seed_type=None):
+    async def get_ssl_domain(self):
+        """Check and return the SSL domain set in ssl_keyfile and ssl_certfile
+        """
+        return self.config.get_ssl_domain()
+
+    @command('')
+    async def make_seed(self, nbits=None, language=None, seed_type=None):
         """Create a seed"""
         from .mnemonic import Mnemonic
-        s = Mnemonic(language).make_seed(seed_type, num_bits=nbits)
+        s = Mnemonic(language).make_seed(seed_type=seed_type, num_bits=nbits)
         return s
 
     @command('n')
@@ -345,6 +370,9 @@ class Commands:
                 raise Exception("missing prevout for txin")
             txin = PartialTxInput(prevout=prevout)
             txin._trusted_value_sats = int(txin_dict['value'])
+            nsequence = txin_dict.get('nsequence', None)
+            if nsequence is not None:
+                txin.nsequence = nsequence
             sec = txin_dict.get('privkey')
             if sec:
                 txin_type, privkey, compressed = bitcoin.deserialize_privkey(sec)
@@ -364,7 +392,7 @@ class Commands:
     @command('wp')
     async def signtransaction(self, tx, privkey=None, password=None, wallet: Abstract_Wallet = None):
         """Sign a transaction. The wallet keys will be used unless a private key is provided."""
-        tx = PartialTransaction(tx)
+        tx = tx_from_any(tx)
         if privkey:
             txin_type, privkey2, compressed = bitcoin.deserialize_privkey(privkey)
             pubkey = ecc.ECPrivkey(privkey2).get_public_key_bytes(compressed=compressed).hex()
@@ -414,6 +442,13 @@ class Commands:
         domain = address
         return [wallet.export_private_key(address, password) for address in domain]
 
+    @command('wp')
+    async def getprivatekeyforpath(self, path, password=None, wallet: Abstract_Wallet = None):
+        """Get private key corresponding to derivation path (address index).
+        'path' can be either a str such as "m/0/50", or a list of ints such as [0, 50].
+        """
+        return wallet.export_private_key_for_path(path, password)
+
     @command('w')
     async def ismine(self, address, wallet: Abstract_Wallet = None):
         """Check if address is in wallet. Return true if and only address is in wallet"""
@@ -467,7 +502,7 @@ class Commands:
 
     @command('n')
     async def getservers(self):
-        """Return the list of available servers"""
+        """Return the list of known servers (candidates for connecting)."""
         return self.network.get_servers()
 
     @command('')
@@ -531,12 +566,14 @@ class Commands:
         privkeys = privkey.split()
         self.nocheck = nocheck
         #dest = self._resolver(destination)
-        tx = sweep(privkeys,
-                   network=self.network,
-                   config=self.config,
-                   to_address=destination,
-                   fee=tx_fee,
-                   imax=imax)
+        tx = await sweep(
+            privkeys,
+            network=self.network,
+            config=self.config,
+            to_address=destination,
+            fee=tx_fee,
+            imax=imax,
+        )
         return tx.serialize() if tx else None
 
     @command('wp')
@@ -553,82 +590,64 @@ class Commands:
         message = util.to_bytes(message)
         return ecc.verify_message_with_address(address, sig, message)
 
-    def _mktx(self, wallet: Abstract_Wallet, outputs, *, fee=None, feerate=None, change_addr=None, domain_addr=None, domain_coins=None,
-              nocheck=False, unsigned=False, rbf=None, password=None, locktime=None):
-        if fee is not None and feerate is not None:
-            raise Exception("Cannot specify both 'fee' and 'feerate' at the same time!")
+    @command('wp')
+    async def payto(self, destination, amount, fee=None, feerate=None, from_addr=None, from_coins=None, change_addr=None,
+                    nocheck=False, unsigned=False, rbf=None, password=None, locktime=None, addtransaction=False, wallet: Abstract_Wallet = None):
+        """Create a transaction. """
         self.nocheck = nocheck
+        tx_fee = satoshis(fee)
+        domain_addr = from_addr.split(',') if from_addr else None
+        domain_coins = from_coins.split(',') if from_coins else None
+        change_addr = self._resolver(change_addr, wallet)
+        domain_addr = None if domain_addr is None else map(self._resolver, domain_addr, repeat(wallet))
+        amount_sat = satoshis(amount)
+        outputs = [PartialTxOutput.from_address_and_value(destination, amount_sat)]
+        tx = wallet.create_transaction(
+            outputs,
+            fee=tx_fee,
+            feerate=feerate,
+            change_addr=change_addr,
+            domain_addr=domain_addr,
+            domain_coins=domain_coins,
+            unsigned=unsigned,
+            rbf=rbf,
+            password=password,
+            locktime=locktime)
+        result = tx.serialize()
+        if addtransaction:
+            await self.addtransaction(result, wallet=wallet)
+        return result
+
+    @command('wp')
+    async def paytomany(self, outputs, fee=None, feerate=None, from_addr=None, from_coins=None, change_addr=None,
+                        nocheck=False, unsigned=False, rbf=None, password=None, locktime=None, addtransaction=False, wallet: Abstract_Wallet = None):
+        """Create a multi-output transaction. """
+        self.nocheck = nocheck
+        tx_fee = satoshis(fee)
+        domain_addr = from_addr.split(',') if from_addr else None
+        domain_coins = from_coins.split(',') if from_coins else None
         change_addr = self._resolver(change_addr, wallet)
         domain_addr = None if domain_addr is None else map(self._resolver, domain_addr, repeat(wallet))
         final_outputs = []
         for address, amount in outputs:
             address = self._resolver(address, wallet)
-            amount = satoshis(amount)
-            final_outputs.append(PartialTxOutput.from_address_and_value(address, amount))
-
-        coins = wallet.get_spendable_coins(domain_addr)
-        if domain_coins is not None:
-            coins = [coin for coin in coins if (coin.prevout.to_str() in domain_coins)]
-        if feerate is not None:
-            fee_per_kb = 1000 * Decimal(feerate)
-            fee_estimator = partial(SimpleConfig.estimate_fee_for_feerate, fee_per_kb)
-        else:
-            fee_estimator = fee
-        tx = wallet.make_unsigned_transaction(coins=coins,
-                                              outputs=final_outputs,
-                                              fee=fee_estimator,
-                                              change_addr=change_addr)
-        if locktime is not None:
-            tx.locktime = locktime
-        if rbf is None:
-            rbf = self.config.get('use_rbf', True)
-        if rbf:
-            tx.set_rbf(True)
-        if not unsigned:
-            wallet.sign_transaction(tx, password)
-        return tx
-
-    @command('wp')
-    async def payto(self, destination, amount, fee=None, feerate=None, from_addr=None, from_coins=None, change_addr=None,
-                    nocheck=False, unsigned=False, rbf=None, password=None, locktime=None, wallet: Abstract_Wallet = None):
-        """Create a transaction. """
-        tx_fee = satoshis(fee)
-        domain_addr = from_addr.split(',') if from_addr else None
-        domain_coins = from_coins.split(',') if from_coins else None
-        tx = self._mktx(wallet,
-                        [(destination, amount)],
-                        fee=tx_fee,
-                        feerate=feerate,
-                        change_addr=change_addr,
-                        domain_addr=domain_addr,
-                        domain_coins=domain_coins,
-                        nocheck=nocheck,
-                        unsigned=unsigned,
-                        rbf=rbf,
-                        password=password,
-                        locktime=locktime)
-        return tx.serialize()
-
-    @command('wp')
-    async def paytomany(self, outputs, fee=None, feerate=None, from_addr=None, from_coins=None, change_addr=None,
-                        nocheck=False, unsigned=False, rbf=None, password=None, locktime=None, wallet: Abstract_Wallet = None):
-        """Create a multi-output transaction. """
-        tx_fee = satoshis(fee)
-        domain_addr = from_addr.split(',') if from_addr else None
-        domain_coins = from_coins.split(',') if from_coins else None
-        tx = self._mktx(wallet,
-                        outputs,
-                        fee=tx_fee,
-                        feerate=feerate,
-                        change_addr=change_addr,
-                        domain_addr=domain_addr,
-                        domain_coins=domain_coins,
-                        nocheck=nocheck,
-                        unsigned=unsigned,
-                        rbf=rbf,
-                        password=password,
-                        locktime=locktime)
-        return tx.serialize()
+            amount_sat = satoshis(amount)
+            final_outputs.append(PartialTxOutput.from_address_and_value(address, amount_sat))
+        tx = wallet.create_transaction(
+            final_outputs,
+            fee=tx_fee,
+            feerate=feerate,
+            change_addr=change_addr,
+            domain_addr=domain_addr,
+            domain_coins=domain_coins,
+            unsigned=unsigned,
+            rbf=rbf,
+            password=password,
+            locktime=locktime)
+        result = tx.serialize()
+        if addtransaction:
+            await self.addtransaction(result, wallet=wallet)
+        return result
 
     @command('w')
     async def onchain_history(self, year=None, show_addresses=False, show_fiat=False, wallet: Abstract_Wallet = None):
@@ -648,17 +667,6 @@ class Commands:
             kwargs['fx'] = fx
         return json_normalize(wallet.get_detailed_history(**kwargs))
 
-    @command('w')
-    async def init_lightning(self, wallet: Abstract_Wallet = None):
-        """Enable lightning payments"""
-        wallet.init_lightning()
-        return "Lightning keys have been created."
-
-    @command('w')
-    async def remove_lightning(self, wallet: Abstract_Wallet = None):
-        """Disable lightning payments"""
-        wallet.remove_lightning()
-
     @command('w')
     async def lightning_history(self, show_fiat=False, wallet: Abstract_Wallet = None):
         """ lightning history """
@@ -711,7 +719,7 @@ class Commands:
             if balance:
                 item += (format_satoshis(sum(wallet.get_addr_balance(addr))),)
             if labels:
-                item += (repr(wallet.labels.get(addr, '')),)
+                item += (repr(wallet.get_label(addr)),)
             out.append(item)
         return out
 
@@ -754,19 +762,13 @@ class Commands:
         decrypted = wallet.decrypt_message(pubkey, encrypted, password)
         return decrypted.decode('utf-8')
 
-    def _format_request(self, out):
-        from .util import get_request_status
-        out['amount_BTC'] = format_satoshis(out.get('amount'))
-        out['status_str'] = get_request_status(out)
-        return out
-
     @command('w')
     async def getrequest(self, key, wallet: Abstract_Wallet = None):
         """Return a payment request"""
         r = wallet.get_request(key)
         if not r:
             raise Exception("Request not found")
-        return self._format_request(r)
+        return wallet.export_request(r)
 
     #@command('w')
     #async def ackrequest(self, serialized):
@@ -776,7 +778,6 @@ class Commands:
     @command('w')
     async def list_requests(self, pending=False, expired=False, paid=False, wallet: Abstract_Wallet = None):
         """List the payment requests you made."""
-        out = wallet.get_sorted_requests()
         if pending:
             f = PR_UNPAID
         elif expired:
@@ -785,15 +786,40 @@ class Commands:
             f = PR_PAID
         else:
             f = None
+        out = wallet.get_sorted_requests()
         if f is not None:
-            out = list(filter(lambda x: x.get('status')==f, out))
-        return list(map(self._format_request, out))
+            out = list(filter(lambda x: x.status==f, out))
+        return [wallet.export_request(x) for x in out]
 
     @command('w')
     async def createnewaddress(self, wallet: Abstract_Wallet = None):
         """Create a new receiving address, beyond the gap limit of the wallet"""
         return wallet.create_new_address(False)
 
+    @command('w')
+    async def changegaplimit(self, new_limit, iknowwhatimdoing=False, wallet: Abstract_Wallet = None):
+        """Change the gap limit of the wallet."""
+        if not iknowwhatimdoing:
+            raise Exception("WARNING: Are you SURE you want to change the gap limit?\n"
+                            "It makes recovering your wallet from seed difficult!\n"
+                            "Please do your research and make sure you understand the implications.\n"
+                            "Typically only merchants and power users might want to do this.\n"
+                            "To proceed, try again, with the --iknowwhatimdoing option.")
+        if not isinstance(wallet, Deterministic_Wallet):
+            raise Exception("This wallet is not deterministic.")
+        return wallet.change_gap_limit(new_limit)
+
+    @command('wn')
+    async def getminacceptablegap(self, wallet: Abstract_Wallet = None):
+        """Returns the minimum value for gap limit that would be sufficient to discover all
+        known addresses in the wallet.
+        """
+        if not isinstance(wallet, Deterministic_Wallet):
+            raise Exception("This wallet is not deterministic.")
+        if not wallet.is_up_to_date():
+            raise NotSynchronizedException("Wallet not fully synchronized.")
+        return wallet.min_acceptable_gap()
+
     @command('w')
     async def getunusedaddress(self, wallet: Abstract_Wallet = None):
         """Returns the first unused address of the wallet, or None if all addresses are used.
@@ -815,14 +841,15 @@ class Commands:
         expiration = int(expiration) if expiration else None
         req = wallet.make_payment_request(addr, amount, memo, expiration)
         wallet.add_payment_request(req)
-        out = wallet.get_request(addr)
-        return self._format_request(out)
+        wallet.save_db()
+        return wallet.export_request(req)
 
     @command('wn')
     async def add_lightning_request(self, amount, memo='', expiration=3600, wallet: Abstract_Wallet = None):
         amount_sat = int(satoshis(amount))
         key = await wallet.lnworker._add_request_coro(amount_sat, memo, expiration)
-        return wallet.get_request(key)['invoice']
+        wallet.save_db()
+        return wallet.get_formatted_request(key)
 
     @command('w')
     async def addtransaction(self, tx, wallet: Abstract_Wallet = None):
@@ -845,13 +872,15 @@ class Commands:
     @command('w')
     async def rmrequest(self, address, wallet: Abstract_Wallet = None):
         """Remove a payment request"""
-        return wallet.remove_payment_request(address)
+        result = wallet.remove_payment_request(address)
+        wallet.save_db()
+        return result
 
     @command('w')
     async def clear_requests(self, wallet: Abstract_Wallet = None):
         """Remove all payment requests"""
-        for k in list(wallet.receive_requests.keys()):
-            wallet.remove_payment_request(k)
+        wallet.clear_requests()
+        return True
 
     @command('w')
     async def clear_invoices(self, wallet: Abstract_Wallet = None):
@@ -860,11 +889,16 @@ class Commands:
         return True
 
     @command('n')
-    async def notify(self, address: str, URL: str):
-        """Watch an address. Every time the address changes, a http POST is sent to the URL."""
+    async def notify(self, address: str, URL: Optional[str]):
+        """Watch an address. Every time the address changes, a http POST is sent to the URL.
+        Call with an empty URL to stop watching an address.
+        """
         if not hasattr(self, "_notifier"):
             self._notifier = Notifier(self.network)
-        await self._notifier.start_watching_queue.put((address, URL))
+        if URL:
+            await self._notifier.start_watching_addr(address, URL)
+        else:
+            await self._notifier.stop_watching_addr(address)
         return True
 
     @command('wn')
@@ -928,10 +962,22 @@ class Commands:
 
     # lightning network commands
     @command('wn')
-    async def add_peer(self, connection_string, timeout=20, wallet: Abstract_Wallet = None):
-        await wallet.lnworker.add_peer(connection_string)
+    async def add_peer(self, connection_string, timeout=20, gossip=False, wallet: Abstract_Wallet = None):
+        lnworker = self.network.lngossip if gossip else wallet.lnworker
+        await lnworker.add_peer(connection_string)
         return True
 
+    @command('wn')
+    async def list_peers(self, gossip=False, wallet: Abstract_Wallet = None):
+        lnworker = self.network.lngossip if gossip else wallet.lnworker
+        return [{
+            'node_id':p.pubkey.hex(),
+            'address':p.transport.name(),
+            'initialized':p.is_initialized(),
+            'features': str(LnFeatures(p.features)),
+            'channels': [c.funding_outpoint.to_str() for c in p.channels.values()],
+        } for p in lnworker.peers.values()]
+
     @command('wpn')
     async def open_channel(self, connection_string, amount, push_amount=0, password=None, wallet: Abstract_Wallet = None):
         funding_sat = satoshis(amount)
@@ -945,16 +991,23 @@ class Commands:
                                                                          password=password)
         return chan.funding_outpoint.to_str()
 
+    @command('')
+    async def decode_invoice(self, invoice: str):
+        invoice = LNInvoice.from_bech32(invoice)
+        return invoice.to_debug_json()
+
     @command('wn')
-    async def lnpay(self, invoice, attempts=1, timeout=10, wallet: Abstract_Wallet = None):
+    async def lnpay(self, invoice, attempts=1, timeout=30, wallet: Abstract_Wallet = None):
         lnworker = wallet.lnworker
-        lnaddr = lnworker._check_invoice(invoice, None)
+        lnaddr = lnworker._check_invoice(invoice)
         payment_hash = lnaddr.paymenthash
-        success = await lnworker._pay(invoice, attempts=attempts)
+        wallet.save_invoice(LNInvoice.from_bech32(invoice))
+        success, log = await lnworker._pay(invoice, attempts=attempts)
         return {
             'payment_hash': payment_hash.hex(),
             'success': success,
             'preimage': lnworker.get_preimage(payment_hash).hex() if success else None,
+            'log': [x.formatted_tuple() for x in log]
         }
 
     @command('w')
@@ -966,25 +1019,27 @@ class Commands:
     async def list_channels(self, wallet: Abstract_Wallet = None):
         # we output the funding_outpoint instead of the channel_id because lnd uses channel_point (funding outpoint) to identify channels
         from .lnutil import LOCAL, REMOTE, format_short_channel_id
-        encoder = util.MyEncoder()
         l = list(wallet.lnworker.channels.items())
         return [
             {
-                'local_htlcs': json.loads(encoder.encode(chan.hm.log[LOCAL])),
-                'remote_htlcs': json.loads(encoder.encode(chan.hm.log[REMOTE])),
-                'channel_id': format_short_channel_id(chan.short_channel_id) if chan.short_channel_id else None,
-                'full_channel_id': bh2u(chan.channel_id),
+                'short_channel_id': format_short_channel_id(chan.short_channel_id) if chan.short_channel_id else None,
+                'channel_id': bh2u(chan.channel_id),
                 'channel_point': chan.funding_outpoint.to_str(),
                 'state': chan.get_state().name,
+                'peer_state': chan.peer_state.name,
                 'remote_pubkey': bh2u(chan.node_id),
                 'local_balance': chan.balance(LOCAL)//1000,
                 'remote_balance': chan.balance(REMOTE)//1000,
+                'local_reserve': chan.config[REMOTE].reserve_sat, # their config has our reserve
+                'remote_reserve': chan.config[LOCAL].reserve_sat,
+                'local_unsettled_sent': chan.balance_tied_up_in_htlcs_by_direction(LOCAL, direction=SENT) // 1000,
+                'remote_unsettled_sent': chan.balance_tied_up_in_htlcs_by_direction(REMOTE, direction=SENT) // 1000,
             } for channel_id, chan in l
         ]
 
     @command('wn')
     async def dumpgraph(self, wallet: Abstract_Wallet = None):
-        return list(map(bh2u, wallet.lnworker.channel_db.nodes.keys()))
+        return wallet.lnworker.channel_db.to_dict()
 
     @command('n')
     async def inject_fees(self, fees):
@@ -992,13 +1047,19 @@ class Commands:
         self.network.config.fee_estimates = ast.literal_eval(fees)
         self.network.notify('fee')
 
+    @command('wn')
+    async def enable_htlc_settle(self, b: bool, wallet: Abstract_Wallet = None):
+        e = wallet.lnworker.enable_htlc_settle
+        e.set() if b else e.clear()
+
     @command('n')
     async def clear_ln_blacklist(self):
         self.network.path_finder.blacklist.clear()
 
     @command('w')
     async def list_invoices(self, wallet: Abstract_Wallet = None):
-        return wallet.get_invoices()
+        l = wallet.get_invoices()
+        return [wallet.export_invoice(x) for x in l]
 
     @command('wn')
     async def close_channel(self, channel_point, force=False, wallet: Abstract_Wallet = None):
@@ -1007,9 +1068,22 @@ class Commands:
         coro = wallet.lnworker.force_close_channel(chan_id) if force else wallet.lnworker.close_channel(chan_id)
         return await coro
 
+    @command('w')
+    async def export_channel_backup(self, channel_point, wallet: Abstract_Wallet = None):
+        txid, index = channel_point.split(':')
+        chan_id, _ = channel_id_from_funding_tx(txid, int(index))
+        return wallet.lnworker.export_channel_backup(chan_id)
+
+    @command('w')
+    async def import_channel_backup(self, encrypted, wallet: Abstract_Wallet = None):
+        return wallet.lnbackups.import_channel_backup(encrypted)
+
     @command('wn')
-    async def get_channel_ctx(self, channel_point, wallet: Abstract_Wallet = None):
+    async def get_channel_ctx(self, channel_point, iknowwhatimdoing=False, wallet: Abstract_Wallet = None):
         """ return the current commitment transaction of a channel """
+        if not iknowwhatimdoing:
+            raise Exception("WARNING: this command is potentially unsafe.\n"
+                            "To proceed, try again, with the --iknowwhatimdoing option.")
         txid, index = channel_point.split(':')
         chan_id, _ = channel_id_from_funding_tx(txid, int(index))
         chan = wallet.lnworker.channels[chan_id]
@@ -1021,6 +1095,58 @@ class Commands:
         """ return the local watchtower's ctn of channel. used in regtests """
         return await self.network.local_watchtower.sweepstore.get_ctn(channel_point, None)
 
+    @command('wnp')
+    async def normal_swap(self, onchain_amount, lightning_amount, password=None, wallet: Abstract_Wallet = None):
+        """
+        Normal submarine swap: send on-chain BTC, receive on Lightning
+        Note that your funds will be locked for 24h if you do not have enough incoming capacity.
+        """
+        sm = wallet.lnworker.swap_manager
+        if lightning_amount == 'dryrun':
+            await sm.get_pairs()
+            onchain_amount_sat = satoshis(onchain_amount)
+            lightning_amount_sat = sm.get_recv_amount(onchain_amount_sat, is_reverse=False)
+            txid = None
+        elif onchain_amount == 'dryrun':
+            await sm.get_pairs()
+            lightning_amount_sat = satoshis(lightning_amount)
+            onchain_amount_sat = sm.get_send_amount(lightning_amount_sat, is_reverse=False)
+            txid = None
+        else:
+            lightning_amount_sat = satoshis(lightning_amount)
+            onchain_amount_sat = satoshis(onchain_amount)
+            txid = await wallet.lnworker.swap_manager.normal_swap(lightning_amount_sat, onchain_amount_sat, password)
+        return {
+            'txid': txid,
+            'lightning_amount': format_satoshis(lightning_amount_sat),
+            'onchain_amount': format_satoshis(onchain_amount_sat),
+        }
+
+    @command('wn')
+    async def reverse_swap(self, lightning_amount, onchain_amount, wallet: Abstract_Wallet = None):
+        """Reverse submarine swap: send on Lightning, receive on-chain
+        """
+        sm = wallet.lnworker.swap_manager
+        if onchain_amount == 'dryrun':
+            await sm.get_pairs()
+            lightning_amount_sat = satoshis(lightning_amount)
+            onchain_amount_sat = sm.get_recv_amount(lightning_amount_sat, is_reverse=True)
+            success = None
+        elif lightning_amount == 'dryrun':
+            await sm.get_pairs()
+            onchain_amount_sat = satoshis(onchain_amount)
+            lightning_amount_sat = sm.get_send_amount(onchain_amount_sat, is_reverse=True)
+            success = None
+        else:
+            lightning_amount_sat = satoshis(lightning_amount)
+            onchain_amount_sat = satoshis(onchain_amount)
+            success = await wallet.lnworker.swap_manager.reverse_swap(lightning_amount_sat, onchain_amount_sat)
+        return {
+            'success': success,
+            'lightning_amount': format_satoshis(lightning_amount_sat),
+            'onchain_amount': format_satoshis(onchain_amount_sat),
+        }
+
 
 def eval_bool(x: str) -> bool:
     if x == 'false': return False
@@ -1047,6 +1173,8 @@ param_descriptions = {
     'requested_amount': 'Requested amount (in BTC).',
     'outputs': 'list of ["address", amount]',
     'redeem_script': 'redeem script (hexadecimal)',
+    'lightning_amount': "Amount sent or received in a submarine swap. Set it to 'dryrun' to receive a value",
+    'onchain_amount': "Amount sent or received in a submarine swap. Set it to 'dryrun' to receive a value",
 }
 
 command_options = {
@@ -1075,6 +1203,7 @@ command_options = {
     'unsigned':    ("-u", "Do not sign transaction"),
     'rbf':         (None, "Whether to signal opt-in Replace-By-Fee in the transaction (true/false)"),
     'locktime':    (None, "Set locktime block number"),
+    'addtransaction': (None,'Whether transaction is to be used for broadcasting afterwards. Adds transaction to the wallet'),
     'domain':      ("-D", "List of addresses"),
     'memo':        ("-m", "Description of the request"),
     'expiration':  (None, "Time in seconds"),
@@ -1093,6 +1222,8 @@ command_options = {
     'fee_level':   (None, "Float between 0.0 and 1.0, representing fee slider position"),
     'from_height': (None, "Only show transactions that confirmed after given block height"),
     'to_height':   (None, "Only show transactions that confirmed before given block height"),
+    'iknowwhatimdoing': (None, "Acknowledge that I understand the full implications of what I am about to do"),
+    'gossip':      (None, "Apply command to gossip node instead of wallet"),
 }
 
 
@@ -1114,11 +1245,13 @@ arg_types = {
     'fee': lambda x: str(Decimal(x)) if x is not None else None,
     'amount': lambda x: str(Decimal(x)) if x != '!' else '!',
     'locktime': int,
+    'addtransaction': eval_bool,
     'fee_method': str,
     'fee_level': json_loads,
     'encrypt_file': eval_bool,
     'rbf': eval_bool,
     'timeout': float,
+    'attempts': int,
 }
 
 config_variables = {
@@ -1186,11 +1319,13 @@ argparse._SubParsersAction.__call__ = subparser_call
 
 
 def add_network_options(parser):
+    parser.add_argument("-f", "--serverfingerprint", dest="serverfingerprint", default=None, help="only allow connecting to servers with a matching SSL certificate SHA256 fingerprint." + " " +
+                                                                                                  "To calculate this yourself: '$ openssl x509 -noout -fingerprint -sha256 -inform pem -in mycertfile.crt'. Enter as 64 hex chars.")
     parser.add_argument("-1", "--oneserver", action="store_true", dest="oneserver", default=None, help="connect to one server only")
     parser.add_argument("-s", "--server", dest="server", default=None, help="set server host:port:protocol, where protocol is either t (tcp) or s (ssl)")
-    parser.add_argument("-p", "--proxy", dest="proxy", default=None, help="set proxy [type:]host[:port], where type is socks4,socks5 or http")
+    parser.add_argument("-p", "--proxy", dest="proxy", default=None, help="set proxy [type:]host[:port] (or 'none' to disable proxy), where type is socks4,socks5 or http")
     parser.add_argument("--noonion", action="store_true", dest="noonion", default=None, help="do not try to connect to onion servers")
-    parser.add_argument("--skipmerklecheck", action="store_true", dest="skipmerklecheck", default=False, help="Tolerate invalid merkle proofs from server")
+    parser.add_argument("--skipmerklecheck", action="store_true", dest="skipmerklecheck", default=None, help="Tolerate invalid merkle proofs from server")
 
 def add_global_options(parser):
     group = parser.add_argument_group('global options')
@@ -1205,12 +1340,14 @@ def add_global_options(parser):
 
 def add_wallet_option(parser):
     parser.add_argument("-w", "--wallet", dest="wallet_path", help="wallet path")
+    parser.add_argument("--forgetconfig", action="store_true", dest="forget_config", default=False, help="Forget config on exit")
 
 def get_parser():
     # create main parser
     parser = argparse.ArgumentParser(
         epilog="Run 'electrum help <command>' to see the help for a command")
     add_global_options(parser)
+    add_wallet_option(parser)
     subparsers = parser.add_subparsers(dest='cmd', metavar='<command>')
     # gui
     parser_gui = subparsers.add_parser('gui', description="Run Electrum's Graphical User Interface.", help="Run GUI (default)")

electrum/constants.py

@@ -42,6 +42,7 @@ def read_json(filename, default):
 
 GIT_REPO_URL = "https://github.com/spesmilo/electrum"
 GIT_REPO_ISSUES_URL = "https://github.com/spesmilo/electrum/issues"
+BIP39_WALLET_FORMATS = read_json('bip39_wallet_formats.json', [])
 
 
 class AbstractNet:
@@ -124,9 +125,9 @@ class BitcoinTestnet(AbstractNet):
     XPUB_HEADERS_INV = inv_dict(XPUB_HEADERS)
     BIP44_COIN_TYPE = 1
     LN_REALM_BYTE = 1
-    LN_DNS_SEEDS = [
-        'test.nodes.lightning.directory.',
-        'lseed.bitcoinstats.com.',
+    LN_DNS_SEEDS = [  # TODO investigate this again
+        #'test.nodes.lightning.directory.',  # times out.
+        #'lseed.bitcoinstats.com.',  # ignores REALM byte and returns mainnet peers...
     ]
 
 

electrum/contacts.py

@@ -27,7 +27,7 @@ from dns.exception import DNSException
 
 from . import bitcoin
 from . import dnssec
-from .util import export_meta, import_meta, to_string
+from .util import read_json_file, write_json_file, to_string
 from .logging import Logger
 
 
@@ -52,14 +52,13 @@ class Contacts(dict, Logger):
         self.db.put('contacts', dict(self))
 
     def import_file(self, path):
-        import_meta(path, self._validate, self.load_meta)
-
-    def load_meta(self, data):
+        data = read_json_file(path)
+        data = self._validate(data)
         self.update(data)
         self.save()
 
-    def export_file(self, filename):
-        export_meta(self, filename)
+    def export_file(self, path):
+        write_json_file(path, self)
 
     def __setitem__(self, key, value):
         dict.__setitem__(self, key, value)

electrum/crypto.py

@@ -25,20 +25,63 @@
 
 import base64
 import os
+import sys
 import hashlib
 import hmac
 from typing import Union
 
-import pyaes
-
-from .util import assert_bytes, InvalidPassword, to_bytes, to_string, WalletFileException
+from .util import assert_bytes, InvalidPassword, to_bytes, to_string, WalletFileException, versiontuple
 from .i18n import _
+from .logging import get_logger
 
 
+_logger = get_logger(__name__)
+
+
+HAS_PYAES = False
 try:
-    from Cryptodome.Cipher import AES
+    import pyaes
 except:
-    AES = None
+    pass
+else:
+    HAS_PYAES = True
+
+HAS_CRYPTODOME = False
+MIN_CRYPTODOME_VERSION = "3.7"
+try:
+    import Cryptodome
+    if versiontuple(Cryptodome.__version__) < versiontuple(MIN_CRYPTODOME_VERSION):
+        _logger.warning(f"found module 'Cryptodome' but it is too old: {Cryptodome.__version__}<{MIN_CRYPTODOME_VERSION}")
+        raise Exception()
+    from Cryptodome.Cipher import ChaCha20_Poly1305 as CD_ChaCha20_Poly1305
+    from Cryptodome.Cipher import ChaCha20 as CD_ChaCha20
+    from Cryptodome.Cipher import AES as CD_AES
+except:
+    pass
+else:
+    HAS_CRYPTODOME = True
+
+HAS_CRYPTOGRAPHY = False
+MIN_CRYPTOGRAPHY_VERSION = "2.1"
+try:
+    import cryptography
+    if versiontuple(cryptography.__version__) < versiontuple(MIN_CRYPTOGRAPHY_VERSION):
+        _logger.warning(f"found module 'cryptography' but it is too old: {cryptography.__version__}<{MIN_CRYPTOGRAPHY_VERSION}")
+        raise Exception()
+    from cryptography import exceptions
+    from cryptography.hazmat.primitives.ciphers import Cipher as CG_Cipher
+    from cryptography.hazmat.primitives.ciphers import algorithms as CG_algorithms
+    from cryptography.hazmat.primitives.ciphers import modes as CG_modes
+    from cryptography.hazmat.backends import default_backend as CG_default_backend
+    import cryptography.hazmat.primitives.ciphers.aead as CG_aead
+except:
+    pass
+else:
+    HAS_CRYPTOGRAPHY = True
+
+
+if not (HAS_CRYPTODOME or HAS_CRYPTOGRAPHY):
+    sys.exit(f"Error: at least one of ('pycryptodomex', 'cryptography') needs to be installed.")
 
 
 class InvalidPadding(Exception):
@@ -67,24 +110,36 @@ def strip_PKCS7_padding(data: bytes) -> bytes:
 def aes_encrypt_with_iv(key: bytes, iv: bytes, data: bytes) -> bytes:
     assert_bytes(key, iv, data)
     data = append_PKCS7_padding(data)
-    if AES:
-        e = AES.new(key, AES.MODE_CBC, iv).encrypt(data)
-    else:
+    if HAS_CRYPTODOME:
+        e = CD_AES.new(key, CD_AES.MODE_CBC, iv).encrypt(data)
+    elif HAS_CRYPTOGRAPHY:
+        cipher = CG_Cipher(CG_algorithms.AES(key), CG_modes.CBC(iv), backend=CG_default_backend())
+        encryptor = cipher.encryptor()
+        e = encryptor.update(data) + encryptor.finalize()
+    elif HAS_PYAES:
         aes_cbc = pyaes.AESModeOfOperationCBC(key, iv=iv)
         aes = pyaes.Encrypter(aes_cbc, padding=pyaes.PADDING_NONE)
         e = aes.feed(data) + aes.feed()  # empty aes.feed() flushes buffer
+    else:
+        raise Exception("no AES backend found")
     return e
 
 
 def aes_decrypt_with_iv(key: bytes, iv: bytes, data: bytes) -> bytes:
     assert_bytes(key, iv, data)
-    if AES:
-        cipher = AES.new(key, AES.MODE_CBC, iv)
+    if HAS_CRYPTODOME:
+        cipher = CD_AES.new(key, CD_AES.MODE_CBC, iv)
         data = cipher.decrypt(data)
-    else:
+    elif HAS_CRYPTOGRAPHY:
+        cipher = CG_Cipher(CG_algorithms.AES(key), CG_modes.CBC(iv), backend=CG_default_backend())
+        decryptor = cipher.decryptor()
+        data = decryptor.update(data) + decryptor.finalize()
+    elif HAS_PYAES:
         aes_cbc = pyaes.AESModeOfOperationCBC(key, iv=iv)
         aes = pyaes.Decrypter(aes_cbc, padding=pyaes.PADDING_NONE)
         data = aes.feed(data) + aes.feed()  # empty aes.feed() flushes buffer
+    else:
+        raise Exception("no AES backend found")
     try:
         return strip_PKCS7_padding(data)
     except InvalidPadding:
@@ -157,35 +212,89 @@ def _hash_password(password: Union[bytes, str], *, version: int) -> bytes:
         raise UnexpectedPasswordHashVersion(version)
 
 
-def pw_encode(data: str, password: Union[bytes, str, None], *, version: int) -> str:
-    if not password:
-        return data
+def _pw_encode_raw(data: bytes, password: Union[bytes, str], *, version: int) -> bytes:
     if version not in KNOWN_PW_HASH_VERSIONS:
         raise UnexpectedPasswordHashVersion(version)
     # derive key from password
     secret = _hash_password(password, version=version)
     # encrypt given data
-    ciphertext = EncodeAES_bytes(secret, to_bytes(data, "utf8"))
-    ciphertext_b64 = base64.b64encode(ciphertext)
-    return ciphertext_b64.decode('utf8')
+    ciphertext = EncodeAES_bytes(secret, data)
+    return ciphertext
 
 
-def pw_decode(data: str, password: Union[bytes, str, None], *, version: int) -> str:
-    if password is None:
-        return data
+def _pw_decode_raw(data_bytes: bytes, password: Union[bytes, str], *, version: int) -> bytes:
     if version not in KNOWN_PW_HASH_VERSIONS:
         raise UnexpectedPasswordHashVersion(version)
-    data_bytes = bytes(base64.b64decode(data))
     # derive key from password
     secret = _hash_password(password, version=version)
     # decrypt given data
     try:
-        d = to_string(DecodeAES_bytes(secret, data_bytes), "utf8")
+        d = DecodeAES_bytes(secret, data_bytes)
     except Exception as e:
         raise InvalidPassword() from e
     return d
 
 
+def pw_encode_bytes(data: bytes, password: Union[bytes, str], *, version: int) -> str:
+    """plaintext bytes -> base64 ciphertext"""
+    ciphertext = _pw_encode_raw(data, password, version=version)
+    ciphertext_b64 = base64.b64encode(ciphertext)
+    return ciphertext_b64.decode('utf8')
+
+
+def pw_decode_bytes(data: str, password: Union[bytes, str], *, version:int) -> bytes:
+    """base64 ciphertext -> plaintext bytes"""
+    if version not in KNOWN_PW_HASH_VERSIONS:
+        raise UnexpectedPasswordHashVersion(version)
+    data_bytes = bytes(base64.b64decode(data))
+    return _pw_decode_raw(data_bytes, password, version=version)
+
+
+def pw_encode_with_version_and_mac(data: bytes, password: Union[bytes, str]) -> str:
+    """plaintext bytes -> base64 ciphertext"""
+    # https://crypto.stackexchange.com/questions/202/should-we-mac-then-encrypt-or-encrypt-then-mac
+    # Encrypt-and-MAC. The MAC will be used to detect invalid passwords
+    version = PW_HASH_VERSION_LATEST
+    mac = sha256(data)[0:4]
+    ciphertext = _pw_encode_raw(data, password, version=version)
+    ciphertext_b64 = base64.b64encode(bytes([version]) + ciphertext + mac)
+    return ciphertext_b64.decode('utf8')
+
+
+def pw_decode_with_version_and_mac(data: str, password: Union[bytes, str]) -> bytes:
+    """base64 ciphertext -> plaintext bytes"""
+    data_bytes = bytes(base64.b64decode(data))
+    version = int(data_bytes[0])
+    encrypted = data_bytes[1:-4]
+    mac = data_bytes[-4:]
+    if version not in KNOWN_PW_HASH_VERSIONS:
+        raise UnexpectedPasswordHashVersion(version)
+    decrypted = _pw_decode_raw(encrypted, password, version=version)
+    if sha256(decrypted)[0:4] != mac:
+        raise InvalidPassword()
+    return decrypted
+
+
+def pw_encode(data: str, password: Union[bytes, str, None], *, version: int) -> str:
+    """plaintext str -> base64 ciphertext"""
+    if not password:
+        return data
+    plaintext_bytes = to_bytes(data, "utf8")
+    return pw_encode_bytes(plaintext_bytes, password, version=version)
+
+
+def pw_decode(data: str, password: Union[bytes, str, None], *, version: int) -> str:
+    """base64 ciphertext -> plaintext str"""
+    if password is None:
+        return data
+    plaintext_bytes = pw_decode_bytes(data, password, version=version)
+    try:
+        plaintext_str = to_string(plaintext_bytes, "utf8")
+    except UnicodeDecodeError as e:
+        raise InvalidPassword() from e
+    return plaintext_str
+
+
 def sha256(x: Union[bytes, str]) -> bytes:
     x = to_bytes(x, 'utf8')
     return bytes(hashlib.sha256(x).digest())
@@ -216,3 +325,69 @@ def hmac_oneshot(key: bytes, msg: bytes, digest) -> bytes:
         return hmac.digest(key, msg, digest)
     else:
         return hmac.new(key, msg, digest).digest()
+
+
+def chacha20_poly1305_encrypt(
+        *,
+        key: bytes,
+        nonce: bytes,
+        associated_data: bytes = None,
+        data: bytes
+) -> bytes:
+    assert isinstance(key, (bytes, bytearray))
+    assert isinstance(nonce, (bytes, bytearray))
+    assert isinstance(associated_data, (bytes, bytearray, type(None)))
+    assert isinstance(data, (bytes, bytearray))
+    if HAS_CRYPTODOME:
+        cipher = CD_ChaCha20_Poly1305.new(key=key, nonce=nonce)
+        if associated_data is not None:
+            cipher.update(associated_data)
+        ciphertext, mac = cipher.encrypt_and_digest(plaintext=data)
+        return ciphertext + mac
+    if HAS_CRYPTOGRAPHY:
+        a = CG_aead.ChaCha20Poly1305(key)
+        return a.encrypt(nonce, data, associated_data)
+    raise Exception("no chacha20 backend found")
+
+
+def chacha20_poly1305_decrypt(
+        *,
+        key: bytes,
+        nonce: bytes,
+        associated_data: bytes = None,
+        data: bytes
+) -> bytes:
+    assert isinstance(key, (bytes, bytearray))
+    assert isinstance(nonce, (bytes, bytearray))
+    assert isinstance(associated_data, (bytes, bytearray, type(None)))
+    assert isinstance(data, (bytes, bytearray))
+    if HAS_CRYPTODOME:
+        cipher = CD_ChaCha20_Poly1305.new(key=key, nonce=nonce)
+        if associated_data is not None:
+            cipher.update(associated_data)
+        # raises ValueError if not valid (e.g. incorrect MAC)
+        return cipher.decrypt_and_verify(ciphertext=data[:-16], received_mac_tag=data[-16:])
+    if HAS_CRYPTOGRAPHY:
+        a = CG_aead.ChaCha20Poly1305(key)
+        try:
+            return a.decrypt(nonce, data, associated_data)
+        except cryptography.exceptions.InvalidTag as e:
+            raise ValueError("invalid tag") from e
+    raise Exception("no chacha20 backend found")
+
+
+def chacha20_encrypt(*, key: bytes, nonce: bytes, data: bytes) -> bytes:
+    assert isinstance(key, (bytes, bytearray))
+    assert isinstance(nonce, (bytes, bytearray))
+    assert isinstance(data, (bytes, bytearray))
+    assert len(nonce) == 8, f"unexpected nonce size: {len(nonce)} (expected: 8)"
+    if HAS_CRYPTODOME:
+        cipher = CD_ChaCha20.new(key=key, nonce=nonce)
+        return cipher.encrypt(data)
+    if HAS_CRYPTOGRAPHY:
+        nonce = bytes(8) + nonce  # cryptography wants 16 byte nonces
+        algo = CG_algorithms.ChaCha20(key=key, nonce=nonce)
+        cipher = CG_Cipher(algo, mode=None, backend=CG_default_backend())
+        encryptor = cipher.encryptor()
+        return encryptor.update(data)
+    raise Exception("no chacha20 backend found")

electrum/currencies.json

@@ -793,6 +793,9 @@
         "ZRX",
         "ZWL"
     ],
+    "CointraderMonitor": [
+        "BRL"
+    ],
     "Kraken": [
         "CAD",
         "EUR",
@@ -882,14 +885,20 @@
     "MercadoBitcoin": [
         "BRL"
     ],
-    "NegocieCoins": [
-        "BRL"
-    ],
     "TheRockTrading": [
         "EUR"
     ],
     "Zaif": [
         "JPY"
     ],
-    "itBit": []
+    "itBit": [],
+    "Bitragem": [
+        "BRL"
+    ],
+    "Biscoint": [
+        "BRL"
+    ],
+    "Walltime": [
+        "BRL"
+    ]
 }

electrum/daemon.py

@@ -29,21 +29,21 @@ import time
 import traceback
 import sys
 import threading
-from typing import Dict, Optional, Tuple, Iterable
+from typing import Dict, Optional, Tuple, Iterable, Callable, Union, Sequence, Mapping
 from base64 import b64decode, b64encode
 from collections import defaultdict
+import concurrent
+from concurrent import futures
+import json
 
 import aiohttp
 from aiohttp import web, client_exceptions
-import jsonrpcclient
-import jsonrpcserver
-from jsonrpcserver import response
-from jsonrpcclient.clients.aiohttp_client import AiohttpClient
 from aiorpcx import TaskGroup
 
+from . import util
 from .network import Network
 from .util import (json_decode, to_bytes, to_string, profiler, standardize_path, constant_time_compare)
-from .util import PR_PAID, PR_EXPIRED, get_request_status
+from .invoices import PR_PAID, PR_EXPIRED
 from .util import log_exceptions, ignore_exceptions, randrange
 from .wallet import Wallet, Abstract_Wallet
 from .storage import WalletStorage
@@ -104,10 +104,8 @@ def request(config: SimpleConfig, endpoint, args=(), timeout=60):
         loop = asyncio.get_event_loop()
         async def request_coroutine():
             async with aiohttp.ClientSession(auth=auth) as session:
-                server = AiohttpClient(session, server_url)
-                f = getattr(server, endpoint)
-                response = await f(*args)
-                return response.data.result
+                c = util.JsonRPCClient(session, server_url)
+                return await c.request(endpoint, *args)
         try:
             fut = asyncio.run_coroutine_threadsafe(request_coroutine(), loop)
             return fut.result(timeout=timeout)
@@ -122,6 +120,10 @@ def request(config: SimpleConfig, endpoint, args=(), timeout=60):
 def get_rpc_credentials(config: SimpleConfig) -> Tuple[str, str]:
     rpc_user = config.get('rpcuser', None)
     rpc_password = config.get('rpcpassword', None)
+    if rpc_user == '':
+        rpc_user = None
+    if rpc_password == '':
+        rpc_password = None
     if rpc_user is None or rpc_password is None:
         rpc_user = 'user'
         bits = 128
@@ -132,132 +134,9 @@ def get_rpc_credentials(config: SimpleConfig) -> Tuple[str, str]:
         rpc_password = to_string(pw_b64, 'ascii')
         config.set_key('rpcuser', rpc_user)
         config.set_key('rpcpassword', rpc_password, save=True)
-    elif rpc_password == '':
-        _logger.warning('RPC authentication is disabled.')
     return rpc_user, rpc_password
 
 
-class WatchTowerServer(Logger):
-
-    def __init__(self, network):
-        Logger.__init__(self)
-        self.config = network.config
-        self.network = network
-        self.lnwatcher = network.local_watchtower
-        self.app = web.Application()
-        self.app.router.add_post("/", self.handle)
-        self.methods = jsonrpcserver.methods.Methods()
-        self.methods.add(self.get_ctn)
-        self.methods.add(self.add_sweep_tx)
-
-    async def handle(self, request):
-        request = await request.text()
-        self.logger.info(f'{request}')
-        response = await jsonrpcserver.async_dispatch(request, methods=self.methods)
-        if response.wanted:
-            return web.json_response(response.deserialized(), status=response.http_status)
-        else:
-            return web.Response()
-
-    async def run(self):
-        host = self.config.get('watchtower_host')
-        port = self.config.get('watchtower_port', 12345)
-        self.runner = web.AppRunner(self.app)
-        await self.runner.setup()
-        site = web.TCPSite(self.runner, host, port, ssl_context=self.config.get_ssl_context())
-        await site.start()
-
-    async def get_ctn(self, *args):
-        return await self.lnwatcher.sweepstore.get_ctn(*args)
-
-    async def add_sweep_tx(self, *args):
-        return await self.lnwatcher.sweepstore.add_sweep_tx(*args)
-
-
-class PayServer(Logger):
-
-    def __init__(self, daemon: 'Daemon'):
-        Logger.__init__(self)
-        self.daemon = daemon
-        self.config = daemon.config
-        self.pending = defaultdict(asyncio.Event)
-        self.daemon.network.register_callback(self.on_payment, ['payment_received'])
-
-    async def on_payment(self, evt, wallet, key, status):
-        if status == PR_PAID:
-            await self.pending[key].set()
-
-    @ignore_exceptions
-    @log_exceptions
-    async def run(self):
-        host = self.config.get('payserver_host', 'localhost')
-        port = self.config.get('payserver_port')
-        root = self.config.get('payserver_root', '/r')
-        app = web.Application()
-        app.add_routes([web.post('/api/create_invoice', self.create_request)])
-        app.add_routes([web.get('/api/get_invoice', self.get_request)])
-        app.add_routes([web.get('/api/get_status', self.get_status)])
-        app.add_routes([web.get('/bip70/{key}.bip70', self.get_bip70_request)])
-        app.add_routes([web.static(root, os.path.join(os.path.dirname(__file__), 'www'))])
-        runner = web.AppRunner(app)
-        await runner.setup()
-        site = web.TCPSite(runner, port=port, host=host, ssl_context=self.config.get_ssl_context())
-        await site.start()
-
-    async def create_request(self, request):
-        params = await request.post()
-        wallet = self.daemon.wallet
-        if 'amount_sat' not in params or not params['amount_sat'].isdigit():
-            raise web.HTTPUnsupportedMediaType()
-        amount = int(params['amount_sat'])
-        message = params['message'] or "donation"
-        payment_hash = await wallet.lnworker._add_invoice_coro(amount, message, 3600)
-        key = payment_hash.hex()
-        raise web.HTTPFound(self.root + '/pay?id=' + key)
-
-    async def get_request(self, r):
-        key = r.query_string
-        request = self.daemon.wallet.get_request(key)
-        return web.json_response(request)
-
-    async def get_bip70_request(self, r):
-        from .paymentrequest import make_request
-        key = r.match_info['key']
-        request = self.daemon.wallet.get_request(key)
-        if not request:
-            return web.HTTPNotFound()
-        pr = make_request(self.config, request)
-        return web.Response(body=pr.SerializeToString(), content_type='application/bitcoin-paymentrequest')
-
-    async def get_status(self, request):
-        ws = web.WebSocketResponse()
-        await ws.prepare(request)
-        key = request.query_string
-        info = self.daemon.wallet.get_request(key)
-        if not info:
-            await ws.send_str('unknown invoice')
-            await ws.close()
-            return ws
-        if info.get('status') == PR_PAID:
-            await ws.send_str(f'paid')
-            await ws.close()
-            return ws
-        if info.get('status') == PR_EXPIRED:
-            await ws.send_str(f'expired')
-            await ws.close()
-            return ws
-        while True:
-            try:
-                await asyncio.wait_for(self.pending[key].wait(), 1)
-                break
-            except asyncio.TimeoutError:
-                # send data on the websocket, to keep it alive
-                await ws.send_str('waiting')
-        await ws.send_str('paid')
-        await ws.close()
-        return ws
-
-
 class AuthenticationError(Exception):
     pass
 
@@ -267,59 +146,18 @@ class AuthenticationInvalidOrMissing(AuthenticationError):
 class AuthenticationCredentialsInvalid(AuthenticationError):
     pass
 
-class Daemon(Logger):
+class AuthenticatedServer(Logger):
 
-    @profiler
-    def __init__(self, config: SimpleConfig, fd=None, *, listen_jsonrpc=True):
+    def __init__(self, rpc_user, rpc_password):
         Logger.__init__(self)
+        self.rpc_user = rpc_user
+        self.rpc_password = rpc_password
         self.auth_lock = asyncio.Lock()
-        self.running = False
-        self.running_lock = threading.Lock()
-        self.config = config
-        if fd is None and listen_jsonrpc:
-            fd = get_file_descriptor(config)
-            if fd is None:
-                raise Exception('failed to lock daemon; already running?')
-        self.asyncio_loop = asyncio.get_event_loop()
-        self.network = None
-        if not config.get('offline'):
-            self.network = Network(config, daemon=self)
-        self.fx = FxThread(config, self.network)
-        self.gui_object = None
-        # path -> wallet;   make sure path is standardized.
-        self._wallets = {}  # type: Dict[str, Abstract_Wallet]
-        daemon_jobs = []
-        # Setup JSONRPC server
-        if listen_jsonrpc:
-            daemon_jobs.append(self.start_jsonrpc(config, fd))
-        # request server
-        self.pay_server = None
-        if not config.get('offline') and self.config.get('run_payserver'):
-            self.pay_server = PayServer(self)
-            daemon_jobs.append(self.pay_server.run())
-        # server-side watchtower
-        self.watchtower = None
-        if not config.get('offline') and self.config.get('run_watchtower'):
-            self.watchtower = WatchTowerServer(self.network)
-            daemon_jobs.append(self.watchtower.run)
-        if self.network:
-            self.network.start(jobs=[self.fx.run])
+        self._methods = {}  # type: Dict[str, Callable]
 
-        self.taskgroup = TaskGroup()
-        asyncio.run_coroutine_threadsafe(self._run(jobs=daemon_jobs), self.asyncio_loop)
-
-    @log_exceptions
-    async def _run(self, jobs: Iterable = None):
-        if jobs is None:
-            jobs = []
-        try:
-            async with self.taskgroup as group:
-                [await group.spawn(job) for job in jobs]
-                await group.spawn(asyncio.Event().wait)  # run forever (until cancel)
-        except BaseException as e:
-            self.logger.exception('daemon.taskgroup died.')
-        finally:
-            self.logger.info("stopping daemon.taskgroup")
+    def register_method(self, f):
+        assert f.__name__ not in self._methods, f"name collision for {f.__name__}"
+        self._methods[f.__name__] = f
 
     async def authenticate(self, headers):
         if self.rpc_password == '':
@@ -348,46 +186,72 @@ class Daemon(Logger):
                                     text='Unauthorized', status=401)
             except AuthenticationCredentialsInvalid:
                 return web.Response(text='Forbidden', status=403)
-        request = await request.text()
-        response = await jsonrpcserver.async_dispatch(request, methods=self.methods)
-        if isinstance(response, jsonrpcserver.response.ExceptionResponse):
-            self.logger.error(f"error handling request: {request}", exc_info=response.exc)
-            # this exposes the error message to the client
-            response.message = str(response.exc)
-        if response.wanted:
-            return web.json_response(response.deserialized(), status=response.http_status)
-        else:
-            return web.Response()
+        try:
+            request = await request.text()
+            request = json.loads(request)
+            method = request['method']
+            _id = request['id']
+            params = request.get('params', [])  # type: Union[Sequence, Mapping]
+            if method not in self._methods:
+                raise Exception(f"attempting to use unregistered method: {method}")
+            f = self._methods[method]
+        except Exception as e:
+            self.logger.exception("invalid request")
+            return web.Response(text='Invalid Request', status=500)
+        response = {
+            'id': _id,
+            'jsonrpc': '2.0',
+        }
+        try:
+            if isinstance(params, dict):
+                response['result'] = await f(**params)
+            else:
+                response['result'] = await f(*params)
+        except BaseException as e:
+            self.logger.exception("internal error while executing RPC")
+            response['error'] = {
+                'code': 1,
+                'message': str(e),
+            }
+        return web.json_response(response)
 
-    async def start_jsonrpc(self, config: SimpleConfig, fd):
+
+class CommandsServer(AuthenticatedServer):
+
+    def __init__(self, daemon, fd):
+        rpc_user, rpc_password = get_rpc_credentials(daemon.config)
+        AuthenticatedServer.__init__(self, rpc_user, rpc_password)
+        self.daemon = daemon
+        self.fd = fd
+        self.config = daemon.config
+        self.host = self.config.get('rpchost', '127.0.0.1')
+        self.port = self.config.get('rpcport', 0)
         self.app = web.Application()
         self.app.router.add_post("/", self.handle)
-        self.rpc_user, self.rpc_password = get_rpc_credentials(config)
-        self.methods = jsonrpcserver.methods.Methods()
-        self.methods.add(self.ping)
-        self.methods.add(self.gui)
-        self.cmd_runner = Commands(config=self.config, network=self.network, daemon=self)
+        self.register_method(self.ping)
+        self.register_method(self.gui)
+        self.cmd_runner = Commands(config=self.config, network=self.daemon.network, daemon=self.daemon)
         for cmdname in known_commands:
-            self.methods.add(getattr(self.cmd_runner, cmdname))
-        self.methods.add(self.run_cmdline)
-        self.host = config.get('rpchost', '127.0.0.1')
-        self.port = config.get('rpcport', 0)
+            self.register_method(getattr(self.cmd_runner, cmdname))
+        self.register_method(self.run_cmdline)
+
+    async def run(self):
         self.runner = web.AppRunner(self.app)
         await self.runner.setup()
         site = web.TCPSite(self.runner, self.host, self.port)
         await site.start()
         socket = site._server.sockets[0]
-        os.write(fd, bytes(repr((socket.getsockname(), time.time())), 'utf8'))
-        os.close(fd)
+        os.write(self.fd, bytes(repr((socket.getsockname(), time.time())), 'utf8'))
+        os.close(self.fd)
 
     async def ping(self):
         return True
 
     async def gui(self, config_options):
-        if self.gui_object:
-            if hasattr(self.gui_object, 'new_window'):
+        if self.daemon.gui_object:
+            if hasattr(self.daemon.gui_object, 'new_window'):
                 path = self.config.get_wallet_path(use_gui_last_wallet=True)
-                self.gui_object.new_window(path, config_options.get('url'))
+                self.daemon.gui_object.new_window(path, config_options.get('url'))
                 response = "ok"
             else:
                 response = "error: current GUI does not support multiple windows"
@@ -395,6 +259,218 @@ class Daemon(Logger):
             response = "Error: Electrum is running in daemon mode. Please stop the daemon first."
         return response
 
+    async def run_cmdline(self, config_options):
+        cmdname = config_options['cmd']
+        cmd = known_commands[cmdname]
+        # arguments passed to function
+        args = [config_options.get(x) for x in cmd.params]
+        # decode json arguments
+        args = [json_decode(i) for i in args]
+        # options
+        kwargs = {}
+        for x in cmd.options:
+            kwargs[x] = config_options.get(x)
+        if 'wallet_path' in cmd.options:
+            kwargs['wallet_path'] = config_options.get('wallet_path')
+        elif 'wallet' in cmd.options:
+            kwargs['wallet'] = config_options.get('wallet_path')
+        func = getattr(self.cmd_runner, cmd.name)
+        # fixme: not sure how to retrieve message in jsonrpcclient
+        try:
+            result = await func(*args, **kwargs)
+        except Exception as e:
+            result = {'error':str(e)}
+        return result
+
+
+class WatchTowerServer(AuthenticatedServer):
+
+    def __init__(self, network, netaddress):
+        self.addr = netaddress
+        self.config = network.config
+        self.network = network
+        watchtower_user = self.config.get('watchtower_user', '')
+        watchtower_password = self.config.get('watchtower_password', '')
+        AuthenticatedServer.__init__(self, watchtower_user, watchtower_password)
+        self.lnwatcher = network.local_watchtower
+        self.app = web.Application()
+        self.app.router.add_post("/", self.handle)
+        self.register_method(self.get_ctn)
+        self.register_method(self.add_sweep_tx)
+
+    async def run(self):
+        self.runner = web.AppRunner(self.app)
+        await self.runner.setup()
+        site = web.TCPSite(self.runner, host=str(self.addr.host), port=self.addr.port, ssl_context=self.config.get_ssl_context())
+        await site.start()
+
+    async def get_ctn(self, *args):
+        return await self.lnwatcher.sweepstore.get_ctn(*args)
+
+    async def add_sweep_tx(self, *args):
+        return await self.lnwatcher.sweepstore.add_sweep_tx(*args)
+
+
+class PayServer(Logger):
+
+    def __init__(self, daemon: 'Daemon', netaddress):
+        Logger.__init__(self)
+        self.addr = netaddress
+        self.daemon = daemon
+        self.config = daemon.config
+        self.pending = defaultdict(asyncio.Event)
+        util.register_callback(self.on_payment, ['request_status'])
+
+    @property
+    def wallet(self):
+        # FIXME specify wallet somehow?
+        return list(self.daemon.get_wallets().values())[0]
+
+    async def on_payment(self, evt, wallet, key, status):
+        if status == PR_PAID:
+            self.pending[key].set()
+
+    @ignore_exceptions
+    @log_exceptions
+    async def run(self):
+        root = self.config.get('payserver_root', '/r')
+        app = web.Application()
+        app.add_routes([web.get('/api/get_invoice', self.get_request)])
+        app.add_routes([web.get('/api/get_status', self.get_status)])
+        app.add_routes([web.get('/bip70/{key}.bip70', self.get_bip70_request)])
+        app.add_routes([web.static(root, os.path.join(os.path.dirname(__file__), 'www'))])
+        if self.config.get('payserver_allow_create_invoice'):
+            app.add_routes([web.post('/api/create_invoice', self.create_request)])
+        runner = web.AppRunner(app)
+        await runner.setup()
+        site = web.TCPSite(runner, host=str(self.addr.host), port=self.addr.port, ssl_context=self.config.get_ssl_context())
+        await site.start()
+
+    async def create_request(self, request):
+        params = await request.post()
+        wallet = self.wallet
+        if 'amount_sat' not in params or not params['amount_sat'].isdigit():
+            raise web.HTTPUnsupportedMediaType()
+        amount = int(params['amount_sat'])
+        message = params['message'] or "donation"
+        payment_hash = wallet.lnworker.add_request(
+            amount_sat=amount,
+            message=message,
+            expiry=3600)
+        key = payment_hash.hex()
+        raise web.HTTPFound(self.root + '/pay?id=' + key)
+
+    async def get_request(self, r):
+        key = r.query_string
+        request = self.wallet.get_formatted_request(key)
+        return web.json_response(request)
+
+    async def get_bip70_request(self, r):
+        from .paymentrequest import make_request
+        key = r.match_info['key']
+        request = self.wallet.get_request(key)
+        if not request:
+            return web.HTTPNotFound()
+        pr = make_request(self.config, request)
+        return web.Response(body=pr.SerializeToString(), content_type='application/bitcoin-paymentrequest')
+
+    async def get_status(self, request):
+        ws = web.WebSocketResponse()
+        await ws.prepare(request)
+        key = request.query_string
+        info = self.wallet.get_formatted_request(key)
+        if not info:
+            await ws.send_str('unknown invoice')
+            await ws.close()
+            return ws
+        if info.get('status') == PR_PAID:
+            await ws.send_str(f'paid')
+            await ws.close()
+            return ws
+        if info.get('status') == PR_EXPIRED:
+            await ws.send_str(f'expired')
+            await ws.close()
+            return ws
+        while True:
+            try:
+                await asyncio.wait_for(self.pending[key].wait(), 1)
+                break
+            except asyncio.TimeoutError:
+                # send data on the websocket, to keep it alive
+                await ws.send_str('waiting')
+        await ws.send_str('paid')
+        await ws.close()
+        return ws
+
+
+
+class Daemon(Logger):
+
+    network: Optional[Network]
+
+    @profiler
+    def __init__(self, config: SimpleConfig, fd=None, *, listen_jsonrpc=True):
+        Logger.__init__(self)
+        self.running = False
+        self.running_lock = threading.Lock()
+        self.config = config
+        if fd is None and listen_jsonrpc:
+            fd = get_file_descriptor(config)
+            if fd is None:
+                raise Exception('failed to lock daemon; already running?')
+        if 'wallet_path' in config.cmdline_options:
+            self.logger.warning("Ignoring parameter 'wallet_path' for daemon. "
+                                "Use the load_wallet command instead.")
+        self.asyncio_loop = asyncio.get_event_loop()
+        self.network = None
+        if not config.get('offline'):
+            self.network = Network(config, daemon=self)
+        self.fx = FxThread(config, self.network)
+        self.gui_object = None
+        # path -> wallet;   make sure path is standardized.
+        self._wallets = {}  # type: Dict[str, Abstract_Wallet]
+        daemon_jobs = []
+        # Setup commands server
+        self.commands_server = None
+        if listen_jsonrpc:
+            self.commands_server = CommandsServer(self, fd)
+            daemon_jobs.append(self.commands_server.run())
+        # pay server
+        self.pay_server = None
+        payserver_address = self.config.get_netaddress('payserver_address')
+        if not config.get('offline') and payserver_address:
+            self.pay_server = PayServer(self, payserver_address)
+            daemon_jobs.append(self.pay_server.run())
+        # server-side watchtower
+        self.watchtower = None
+        watchtower_address = self.config.get_netaddress('watchtower_address')
+        if not config.get('offline') and watchtower_address:
+            self.watchtower = WatchTowerServer(self.network, watchtower_address)
+            daemon_jobs.append(self.watchtower.run)
+        if self.network:
+            self.network.start(jobs=[self.fx.run])
+            # prepare lightning functionality, also load channel db early
+            self.network.init_channel_db()
+
+        self.taskgroup = TaskGroup()
+        asyncio.run_coroutine_threadsafe(self._run(jobs=daemon_jobs), self.asyncio_loop)
+
+    @log_exceptions
+    async def _run(self, jobs: Iterable = None):
+        if jobs is None:
+            jobs = []
+        self.logger.info("starting taskgroup.")
+        try:
+            async with self.taskgroup as group:
+                [await group.spawn(job) for job in jobs]
+                await group.spawn(asyncio.Event().wait)  # run forever (until cancel)
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:
+            self.logger.exception("taskgroup died.")
+        finally:
+            self.logger.info("taskgroup stopped.")
+
     def load_wallet(self, path, password, *, manual_upgrades=True) -> Optional[Abstract_Wallet]:
         path = standardize_path(path)
         # wizard will be launched if we return
@@ -419,7 +495,6 @@ class Daemon(Logger):
         wallet = Wallet(db, storage, config=self.config)
         wallet.start_network(self.network)
         self._wallets[path] = wallet
-        self.wallet = wallet
         return wallet
 
     def add_wallet(self, wallet: Abstract_Wallet) -> None:
@@ -427,7 +502,7 @@ class Daemon(Logger):
         path = standardize_path(path)
         self._wallets[path] = wallet
 
-    def get_wallet(self, path: str) -> Abstract_Wallet:
+    def get_wallet(self, path: str) -> Optional[Abstract_Wallet]:
         path = standardize_path(path)
         return self._wallets.get(path)
 
@@ -447,30 +522,9 @@ class Daemon(Logger):
         wallet = self._wallets.pop(path, None)
         if not wallet:
             return False
-        wallet.stop_threads()
+        wallet.stop()
         return True
 
-    async def run_cmdline(self, config_options):
-        cmdname = config_options['cmd']
-        cmd = known_commands[cmdname]
-        # arguments passed to function
-        args = [config_options.get(x) for x in cmd.params]
-        # decode json arguments
-        args = [json_decode(i) for i in args]
-        # options
-        kwargs = {}
-        for x in cmd.options:
-            kwargs[x] = config_options.get(x)
-        if cmd.requires_wallet:
-            kwargs['wallet_path'] = config_options.get('wallet_path')
-        func = getattr(self.cmd_runner, cmd.name)
-        # fixme: not sure how to retrieve message in jsonrpcclient
-        try:
-            result = await func(*args, **kwargs)
-        except Exception as e:
-            result = {'error':str(e)}
-        return result
-
     def run_daemon(self):
         self.running = True
         try:
@@ -493,7 +547,7 @@ class Daemon(Logger):
             self.gui_object.stop()
         # stop network/wallets
         for k, wallet in self._wallets.items():
-            wallet.stop_threads()
+            wallet.stop()
         if self.network:
             self.logger.info("shutting down network")
             self.network.stop()
@@ -501,7 +555,7 @@ class Daemon(Logger):
         fut = asyncio.run_coroutine_threadsafe(self.taskgroup.cancel_remaining(), self.asyncio_loop)
         try:
             fut.result(timeout=2)
-        except (asyncio.TimeoutError, asyncio.CancelledError):
+        except (concurrent.futures.TimeoutError, concurrent.futures.CancelledError, asyncio.CancelledError):
             pass
         self.logger.info("removing lockfile")
         remove_lockfile(get_lockfile(self.config))

electrum/dns_hacks.py

@@ -32,8 +32,12 @@ def configure_dns_depending_on_proxy(is_proxy: bool) -> None:
             # On Windows, socket.getaddrinfo takes a mutex, and might hold it for up to 10 seconds
             # when dns-resolving. To speed it up drastically, we resolve dns ourselves, outside that lock.
             # See https://github.com/spesmilo/electrum/issues/4421
-            _prepare_windows_dns_hack()
-            socket.getaddrinfo = _fast_getaddrinfo
+            try:
+                _prepare_windows_dns_hack()
+            except Exception as e:
+                _logger.exception('failed to apply windows dns hack.')
+            else:
+                socket.getaddrinfo = _fast_getaddrinfo
         else:
             socket.getaddrinfo = socket._getaddrinfo
 
@@ -43,6 +47,8 @@ def _prepare_windows_dns_hack():
     resolver = dns.resolver.get_default_resolver()
     if resolver.cache is None:
         resolver.cache = dns.resolver.Cache()
+    # ensure overall timeout for requests is long enough
+    resolver.lifetime = max(resolver.lifetime or 1, 30.0)
     # prepare threads
     global _dns_threads_executor
     if _dns_threads_executor is None:
@@ -65,8 +71,8 @@ def _fast_getaddrinfo(host, *args, **kwargs):
         addrs = []
         expected_errors = (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer,
                            concurrent.futures.CancelledError, concurrent.futures.TimeoutError)
-        ipv6_fut = _dns_threads_executor.submit(dns.resolver.query, host, dns.rdatatype.AAAA)
-        ipv4_fut = _dns_threads_executor.submit(dns.resolver.query, host, dns.rdatatype.A)
+        ipv6_fut = _dns_threads_executor.submit(dns.resolver.resolve, host, dns.rdatatype.AAAA)
+        ipv4_fut = _dns_threads_executor.submit(dns.resolver.resolve, host, dns.rdatatype.A)
         # try IPv6
         try:
             answers = ipv6_fut.result()

electrum/dnssec.py

@@ -35,6 +35,7 @@
 # import sys
 import time
 import struct
+import hashlib
 
 
 import dns.name
@@ -57,122 +58,6 @@ import dns.rdtypes.ANY.TXT
 import dns.rdtypes.IN.A
 import dns.rdtypes.IN.AAAA
 
-
-# Pure-Python version of dns.dnssec._validate_rsig
-import ecdsa
-from . import rsakey
-
-
-def python_validate_rrsig(rrset, rrsig, keys, origin=None, now=None):
-    from dns.dnssec import ValidationFailure, ECDSAP256SHA256, ECDSAP384SHA384
-    from dns.dnssec import _find_candidate_keys, _make_hash, _is_ecdsa, _is_rsa, _to_rdata, _make_algorithm_id
-
-    if isinstance(origin, str):
-        origin = dns.name.from_text(origin, dns.name.root)
-
-    for candidate_key in _find_candidate_keys(keys, rrsig):
-        if not candidate_key:
-            raise ValidationFailure('unknown key')
-
-        # For convenience, allow the rrset to be specified as a (name, rdataset)
-        # tuple as well as a proper rrset
-        if isinstance(rrset, tuple):
-            rrname = rrset[0]
-            rdataset = rrset[1]
-        else:
-            rrname = rrset.name
-            rdataset = rrset
-
-        if now is None:
-            now = time.time()
-        if rrsig.expiration < now:
-            raise ValidationFailure('expired')
-        if rrsig.inception > now:
-            raise ValidationFailure('not yet valid')
-
-        hash = _make_hash(rrsig.algorithm)
-
-        if _is_rsa(rrsig.algorithm):
-            keyptr = candidate_key.key
-            (bytes,) = struct.unpack('!B', keyptr[0:1])
-            keyptr = keyptr[1:]
-            if bytes == 0:
-                (bytes,) = struct.unpack('!H', keyptr[0:2])
-                keyptr = keyptr[2:]
-            rsa_e = keyptr[0:bytes]
-            rsa_n = keyptr[bytes:]
-            n = int.from_bytes(rsa_n, byteorder='big', signed=False)
-            e = int.from_bytes(rsa_e, byteorder='big', signed=False)
-            pubkey = rsakey.RSAKey(n, e)
-            sig = rrsig.signature
-
-        elif _is_ecdsa(rrsig.algorithm):
-            if rrsig.algorithm == ECDSAP256SHA256:
-                curve = ecdsa.curves.NIST256p
-                key_len = 32
-            elif rrsig.algorithm == ECDSAP384SHA384:
-                curve = ecdsa.curves.NIST384p
-                key_len = 48
-            else:
-                # shouldn't happen
-                raise ValidationFailure('unknown ECDSA curve')
-            keyptr = candidate_key.key
-            x = int.from_bytes(keyptr[0:key_len], byteorder='big', signed=False)
-            y = int.from_bytes(keyptr[key_len:key_len * 2], byteorder='big', signed=False)
-            assert ecdsa.ecdsa.point_is_valid(curve.generator, x, y)
-            point = ecdsa.ellipticcurve.Point(curve.curve, x, y, curve.order)
-            verifying_key = ecdsa.keys.VerifyingKey.from_public_point(point, curve)
-            r = rrsig.signature[:key_len]
-            s = rrsig.signature[key_len:]
-            sig = ecdsa.ecdsa.Signature(int.from_bytes(r, byteorder='big', signed=False),
-                                        int.from_bytes(s, byteorder='big', signed=False))
-
-        else:
-            raise ValidationFailure('unknown algorithm %u' % rrsig.algorithm)
-
-        hash.update(_to_rdata(rrsig, origin)[:18])
-        hash.update(rrsig.signer.to_digestable(origin))
-
-        if rrsig.labels < len(rrname) - 1:
-            suffix = rrname.split(rrsig.labels + 1)[1]
-            rrname = dns.name.from_text('*', suffix)
-        rrnamebuf = rrname.to_digestable(origin)
-        rrfixed = struct.pack('!HHI', rdataset.rdtype, rdataset.rdclass,
-                              rrsig.original_ttl)
-        rrlist = sorted(rdataset)
-        for rr in rrlist:
-            hash.update(rrnamebuf)
-            hash.update(rrfixed)
-            rrdata = rr.to_digestable(origin)
-            rrlen = struct.pack('!H', len(rrdata))
-            hash.update(rrlen)
-            hash.update(rrdata)
-
-        digest = hash.digest()
-
-        if _is_rsa(rrsig.algorithm):
-            digest = _make_algorithm_id(rrsig.algorithm) + digest
-            if pubkey.verify(bytearray(sig), bytearray(digest)):
-                return
-
-        elif _is_ecdsa(rrsig.algorithm):
-            diglong = int.from_bytes(digest, byteorder='big', signed=False)
-            if verifying_key.pubkey.verifies(diglong, sig):
-                return
-
-        else:
-            raise ValidationFailure('unknown algorithm %s' % rrsig.algorithm)
-
-    raise ValidationFailure('verify failure')
-
-
-# replace validate_rrsig
-dns.dnssec._validate_rrsig = python_validate_rrsig
-dns.dnssec.validate_rrsig = python_validate_rrsig
-dns.dnssec.validate = dns.dnssec._validate
-
-
-
 from .logging import get_logger
 
 
@@ -266,7 +151,6 @@ def query(url, rtype):
         validated = True
     except BaseException as e:
         _logger.info(f"DNSSEC error: {repr(e)}")
-        resolver = dns.resolver.get_default_resolver()
-        out = resolver.query(url, rtype)
+        out = dns.resolver.resolve(url, rtype)
         validated = False
     return out, validated

electrum/exchange_rate.py

@@ -11,7 +11,9 @@ from decimal import Decimal
 from typing import Sequence, Optional
 
 from aiorpcx.curio import timeout_after, TaskTimeout, TaskGroup
+import aiohttp
 
+from . import util
 from .bitcoin import COIN
 from .i18n import _
 from .util import (ThreadJob, make_dir, log_exceptions,
@@ -81,9 +83,12 @@ class ExchangeBase(Logger):
         except asyncio.CancelledError:
             # CancelledError must be passed-through for cancellation to work
             raise
-        except BaseException as e:
+        except aiohttp.ClientError as e:
             self.logger.info(f"failed fx quotes: {repr(e)}")
             self.quotes = {}
+        except Exception as e:
+            self.logger.exception(f"failed fx quotes: {repr(e)}")
+            self.quotes = {}
         self.on_quotes()
 
     def read_historical_rates(self, ccy, cache_dir) -> Optional[dict]:
@@ -109,9 +114,12 @@ class ExchangeBase(Logger):
             self.logger.info(f"requesting fx history for {ccy}")
             h = await self.request_history(ccy)
             self.logger.info(f"received fx history for {ccy}")
-        except BaseException as e:
+        except aiohttp.ClientError as e:
             self.logger.info(f"failed fx history: {repr(e)}")
             return
+        except Exception as e:
+            self.logger.exception(f"failed fx history: {repr(e)}")
+            return
         filename = os.path.join(cache_dir, self.name() + '_' + ccy)
         with open(filename, 'w', encoding='utf-8') as f:
             f.write(json.dumps(h))
@@ -314,6 +322,13 @@ class CoinGecko(ExchangeBase):
                      for h in history['prices']])
 
 
+class CointraderMonitor(ExchangeBase):
+
+    async def get_rates(self, ccy):
+        json = await self.get_json('cointradermonitor.com', '/api/pbb/v1/ticker')
+        return {'BRL': Decimal(json['last'])}
+
+
 class itBit(ExchangeBase):
 
     async def get_rates(self, ccy):
@@ -351,12 +366,6 @@ class MercadoBitcoin(ExchangeBase):
         return {'BRL': Decimal(json['ticker_1h']['exchanges']['MBT']['last'])}
 
 
-class NegocieCoins(ExchangeBase):
-
-    async def get_rates(self,ccy):
-        json = await self.get_json('api.bitvalor.com', '/v1/ticker.json')
-        return {'BRL': Decimal(json['ticker_1h']['exchanges']['NEG']['last'])}
-
 class TheRockTrading(ExchangeBase):
 
     async def get_rates(self, ccy):
@@ -388,6 +397,28 @@ class Zaif(ExchangeBase):
         return {'JPY': Decimal(json['last_price'])}
 
 
+class Bitragem(ExchangeBase):
+
+    async def get_rates(self,ccy):
+        json = await self.get_json('api.bitragem.com', '/v1/index?asset=BTC&market=BRL')
+        return {'BRL': Decimal(json['response']['index'])}
+
+
+class Biscoint(ExchangeBase):
+
+    async def get_rates(self,ccy):
+        json = await self.get_json('api.biscoint.io', '/v1/ticker?base=BTC&quote=BRL')
+        return {'BRL': Decimal(json['data']['last'])}
+
+
+class Walltime(ExchangeBase):
+
+    async def get_rates(self, ccy):
+        json = await self.get_json('s3.amazonaws.com', 
+                             '/data-production-walltime-info/production/dynamic/walltime-info.json')
+        return {'BRL': Decimal(json['BRL_XBT']['last_inexact'])}
+
+
 def dictinvert(d):
     inv = {}
     for k, vlist in d.items():
@@ -452,12 +483,11 @@ def get_exchanges_by_ccy(history=True):
 
 class FxThread(ThreadJob):
 
-    def __init__(self, config: SimpleConfig, network: Network):
+    def __init__(self, config: SimpleConfig, network: Optional[Network]):
         ThreadJob.__init__(self)
         self.config = config
         self.network = network
-        if self.network:
-            self.network.register_callback(self.set_proxy, ['proxy_set'])
+        util.register_callback(self.set_proxy, ['proxy_set'])
         self.ccy = self.get_currency()
         self.history_used_spot = False
         self.ccy_combo = None
@@ -516,8 +546,11 @@ class FxThread(ThreadJob):
         self.config.set_key('use_exchange_rate', bool(b))
         self.trigger_update()
 
-    def get_history_config(self, *, default=False):
-        return bool(self.config.get('history_rates', default))
+    def get_history_config(self, *, allow_none=False):
+        val = self.config.get('history_rates', None)
+        if val is None and allow_none:
+            return None
+        return bool(val)
 
     def set_history_config(self, b):
         self.config.set_key('history_rates', bool(b))
@@ -567,12 +600,10 @@ class FxThread(ThreadJob):
         self.exchange.read_historical_rates(self.ccy, self.cache_dir)
 
     def on_quotes(self):
-        if self.network:
-            self.network.trigger_callback('on_quotes')
+        util.trigger_callback('on_quotes')
 
     def on_history(self):
-        if self.network:
-            self.network.trigger_callback('on_history')
+        util.trigger_callback('on_history')
 
     def exchange_rate(self) -> Decimal:
         """Returns the exchange rate as a Decimal"""

electrum/gui/kivy/Makefile

@@ -1,33 +0,0 @@
-PYTHON = python3
-
-# needs kivy installed or in PYTHONPATH
-
-.PHONY: theming apk clean
-
-theming:
-	#bash -c 'for i in network lightning; do convert -background none theming/light/$i.{svg,png}; done'
-	$(PYTHON) -m kivy.atlas theming/light 1024 theming/light/*.png
-prepare:
-	# running pre build setup
-	@cp tools/buildozer.spec ../../../buildozer.spec
-	# copy electrum to main.py
-	@cp ../../../run_electrum ../../../main.py
-	@-if [ ! -d "../../../.buildozer" ];then \
-		cd ../../..; buildozer android debug;\
-		cp -f electrum/gui/kivy/tools/blacklist.txt .buildozer/android/platform/python-for-android/src/blacklist.txt;\
-		rm -rf ./.buildozer/android/platform/python-for-android/dist;\
-	fi
-apk:
-	@make prepare
-	@-cd ../../..; buildozer android debug deploy run
-	@make clean
-release:
-	@make prepare
-	@-cd ../../..; buildozer android release
-	@make clean
-clean:
-	# Cleaning up
-	# rename main.py to electrum
-	@-rm ../../../main.py
-	# remove buildozer.spec
-	@-rm ../../../buildozer.spec

electrum/gui/kivy/__init__.py

@@ -29,17 +29,21 @@ import sys
 import os
 from typing import TYPE_CHECKING
 
+KIVY_GUI_PATH = os.path.abspath(os.path.dirname(__file__))
+os.environ['KIVY_DATA_DIR'] = os.path.join(KIVY_GUI_PATH, 'data')
+
 try:
     sys.argv = ['']
     import kivy
 except ImportError:
     # This error ideally shouldn't be raised with pre-built packages
-    sys.exit("Error: Could not import kivy. Please install it using the" + \
-             "instructions mentioned here `http://kivy.org/#download` .")
+    sys.exit("Error: Could not import kivy. Please install it using the "
+             "instructions mentioned here `https://kivy.org/#download` .")
 
 # minimum required version for kivy
 kivy.require('1.8.0')
-from kivy.logger import Logger
+
+from electrum.logging import Logger
 
 if TYPE_CHECKING:
     from electrum.simple_config import SimpleConfig
@@ -49,10 +53,11 @@ if TYPE_CHECKING:
 
 
 
-class ElectrumGui:
+class ElectrumGui(Logger):
 
     def __init__(self, config: 'SimpleConfig', daemon: 'Daemon', plugins: 'Plugins'):
-        Logger.debug('ElectrumGUI: initialising')
+        Logger.__init__(self)
+        self.logger.debug('ElectrumGUI: initialising')
         self.daemon = daemon
         self.network = daemon.network
         self.config = config

electrum/gui/kivy/data/style.kv

@@ -436,6 +436,7 @@
 # Popup widget
 <Popup>:
     _container: container
+    background_color: (0, 0, 0, 0.7)
     GridLayout:
         padding: '12dp'
         cols: 1

electrum/gui/kivy/main.kv

@@ -2,6 +2,7 @@
 #:import Window kivy.core.window.Window
 #:import Factory kivy.factory.Factory
 #:import _ electrum.gui.kivy.i18n._
+#:import KIVY_GUI_PATH electrum.gui.kivy.KIVY_GUI_PATH
 
 
 ###########################
@@ -58,6 +59,7 @@
 
 <IconButton@Button>:
     icon: ''
+    icon_size: '30dp'
     AnchorLayout:
         pos: self.parent.pos
         size: self.parent.size
@@ -65,7 +67,7 @@
         Image:
             source: self.parent.parent.icon
             size_hint_x: None
-            size: '30dp', '30dp'
+            size: root.icon_size, root.icon_size
 
             
 <BackgroundColor@Widget>
@@ -148,14 +150,18 @@
     font_size: '6pt'
     name: ''
     data: ''
+    visible: True
+    opacity: 1 if self.visible else 0
+    disabled: not self.visible
     text: self.data if self.data else _('Tap to show')
     touched: False
     padding: '10dp', '10dp'
     background_color: .3, .3, .3, 1
+    touch_callback: lambda: app.on_ref_label(self)
     on_touch_down:
         touch = args[1]
         touched = bool(self.collide_point(*touch.pos))
-        if touched: app.on_ref_label(self)
+        if touched: self.touch_callback()
         if touched: self.touched = True
     canvas.before:
         Color:
@@ -206,7 +212,7 @@
         Color:
             rgba: 0.192, .498, 0.745, 1
         BorderImage:
-            source: 'atlas://electrum/gui/kivy/theming/light/card_bottom'
+            source: f'atlas://{KIVY_GUI_PATH}/theming/light/card_bottom'
             size: self.size
             pos: self.pos
 
@@ -220,7 +226,7 @@
 		Color:
 			rgba: 0.192, .498, 0.745, 1
 		BorderImage:
-			source: 'atlas://electrum/gui/kivy/theming/light/card_bottom'
+			source: f'atlas://{KIVY_GUI_PATH}/theming/light/card_bottom'
 			size: self.size
 			pos: self.pos
 
@@ -233,7 +239,7 @@
 		Color:
 			rgba: 0.192, .498, 0.745, 1
         BorderImage:
-            source: 'atlas://electrum/gui/kivy/theming/light/card_bottom'
+            source: f'atlas://{KIVY_GUI_PATH}/theming/light/card_bottom'
             size: self.size
             pos: self.pos
 
@@ -282,25 +288,6 @@
             size: self.size
             pos: self.pos
 
-<AddressButton@Button>:
-	background_color: 1, .585, .878, 0
-	halign: 'center'
-	text_size: (self.width, None)
-	shorten: True
-	size_hint: 0.5, None
-	default_text: ''
-	text: self.default_text
-	padding: '5dp', '5dp'
-	height: '40dp'
-	text_color: self.foreground_color
-	disabled_color: 1, 1, 1, 1
-	foreground_color: 1, 1, 1, 1
-	canvas.before:
-		Color:
-			rgba: (0.9, .498, 0.745, 1) if self.state == 'down' else self.background_color
-		Rectangle:
-			size: self.size
-			pos: self.pos
 
 <KButton@Button>:
     size_hint: 1, None
@@ -340,8 +327,8 @@
     valign: 'middle'
     bold: True
     font_size: '12.5sp'
-    background_normal: 'atlas://electrum/gui/kivy/theming/light/tab_btn'
-    background_down: 'atlas://electrum/gui/kivy/theming/light/tab_btn_pressed'
+    background_normal: f'atlas://{KIVY_GUI_PATH}/theming/light/tab_btn'
+    background_down: f'atlas://{KIVY_GUI_PATH}/theming/light/tab_btn_pressed'
 
 
 <ColoredLabel@Label>:
@@ -431,7 +418,7 @@ BoxLayout:
             rgb: .6, .6, .6
         Rectangle:
             size: self.size
-            source: 'electrum/gui/kivy/data/background.png'
+            source: f'{KIVY_GUI_PATH}/data/background.png'
 
     ActionBar:
 
@@ -443,7 +430,7 @@ BoxLayout:
                 size: 0, 0
 
             ActionButton:
-                size_hint_x: 0.5
+                size_hint_x: None
                 text: app.wallet_name
                 bold: True
                 color: 0.7, 0.7, 0.7, 1
@@ -453,13 +440,13 @@ BoxLayout:
                     self.state = 'normal'
 
             ActionButton:
-                size_hint_x: 0.4
+                size_hint_x: 0.8
                 text: ''
                 opacity:0
 
             ActionOverflow:
                 id: ao
-                size_hint_x: 0.15
+                size_hint_x: 0.2
                 ActionOvrButton:
                     name: 'about'
                     text: _('About')
@@ -469,9 +456,6 @@ BoxLayout:
                 ActionOvrButton:
                     name: 'network'
                     text: _('Network')
-                ActionOvrButton:
-                    name: 'lightning'
-                    text: _('Lightning')
                 ActionOvrButton:
                     name: 'addresses_dialog'
                     text: _('Addresses')

electrum/gui/kivy/main_window.py

@@ -13,16 +13,20 @@ from electrum.storage import WalletStorage, StorageReadWriteError
 from electrum.wallet_db import WalletDB
 from electrum.wallet import Wallet, InternalAddressCorruption, Abstract_Wallet
 from electrum.plugin import run_hook
+from electrum import util
 from electrum.util import (profiler, InvalidPassword, send_exception_to_crash_reporter,
                            format_satoshis, format_satoshis_plain, format_fee_satoshis,
-                           PR_PAID, PR_FAILED, maybe_extract_bolt11_invoice)
+                           maybe_extract_bolt11_invoice)
+from electrum.invoices import PR_PAID, PR_FAILED
 from electrum import blockchain
 from electrum.network import Network, TxBroadcastError, BestEffortRequestFailed
+from electrum.interface import PREFERRED_NETWORK_PROTOCOL, ServerAddr
+from electrum.logging import Logger
 from .i18n import _
+from . import KIVY_GUI_PATH
 
 from kivy.app import App
 from kivy.core.window import Window
-from kivy.logger import Logger
 from kivy.utils import platform
 from kivy.properties import (OptionProperty, AliasProperty, ObjectProperty,
                              StringProperty, ListProperty, BooleanProperty, NumericProperty)
@@ -31,7 +35,7 @@ from kivy.clock import Clock
 from kivy.factory import Factory
 from kivy.metrics import inch
 from kivy.lang import Builder
-from .uix.dialogs.password_dialog import PasswordDialog
+from .uix.dialogs.password_dialog import OpenWalletDialog, ChangePasswordDialog, PincodeDialog
 
 ## lazy imports for factory so that widgets can be used in kv
 #Factory.register('InstallWizard', module='electrum.gui.kivy.uix.dialogs.installwizard')
@@ -50,7 +54,6 @@ from .uix.dialogs.question import Question
 
 # delayed imports: for startup speed on android
 notification = app = ref = None
-util = False
 
 # register widget cache for keeping memory down timeout to forever to cache
 # the data
@@ -66,16 +69,17 @@ Factory.register('TabbedCarousel', module='electrum.gui.kivy.uix.screens')
 # Register fonts without this you won't be able to use bold/italic...
 # inside markup.
 from kivy.core.text import Label
-Label.register('Roboto',
-               'electrum/gui/kivy/data/fonts/Roboto.ttf',
-               'electrum/gui/kivy/data/fonts/Roboto.ttf',
-               'electrum/gui/kivy/data/fonts/Roboto-Bold.ttf',
-               'electrum/gui/kivy/data/fonts/Roboto-Bold.ttf')
+Label.register(
+    'Roboto',
+    KIVY_GUI_PATH + '/data/fonts/Roboto.ttf',
+    KIVY_GUI_PATH + '/data/fonts/Roboto.ttf',
+    KIVY_GUI_PATH + '/data/fonts/Roboto-Bold.ttf',
+    KIVY_GUI_PATH + '/data/fonts/Roboto-Bold.ttf',
+)
 
 
-from electrum.util import (base_units, NoDynamicFeeEstimates, decimal_point_to_base_unit_name,
-                           base_unit_name_to_decimal_point, NotEnoughFunds, UnknownBaseUnit,
-                           DECIMAL_POINT_DEFAULT)
+from electrum.util import (NoDynamicFeeEstimates, NotEnoughFunds,
+                           BITCOIN_BIP21_URI_SCHEME, LIGHTNING_URI_SCHEME)
 
 from .uix.dialogs.lightning_open_channel import LightningOpenChannelDialog
 from .uix.dialogs.lightning_channels import LightningChannelsDialog
@@ -87,7 +91,7 @@ if TYPE_CHECKING:
     from electrum.paymentrequest import PaymentRequest
 
 
-class ElectrumWindow(App):
+class ElectrumWindow(App, Logger):
 
     electrum_config = ObjectProperty(None)
     language = StringProperty('en')
@@ -135,15 +139,28 @@ class ElectrumWindow(App):
 
     def choose_server_dialog(self, popup):
         from .uix.dialogs.choice_dialog import ChoiceDialog
-        protocol = 's'
-        def cb2(host):
-            from electrum import constants
-            pp = servers.get(host, constants.net.DEFAULT_PORTS)
-            port = pp.get(protocol, '')
-            popup.ids.host.text = host
-            popup.ids.port.text = port
+        protocol = PREFERRED_NETWORK_PROTOCOL
+        def cb2(server_str):
+            popup.ids.server_str.text = server_str
         servers = self.network.get_servers()
-        ChoiceDialog(_('Choose a server'), sorted(servers), popup.ids.host.text, cb2).open()
+        server_choices = {}
+        for _host, d in sorted(servers.items()):
+            port = d.get(protocol)
+            if port:
+                server = ServerAddr(_host, port, protocol=protocol)
+                server_choices[server.net_addr_str()] = _host
+        ChoiceDialog(_('Choose a server'), server_choices, popup.ids.server_str.text, cb2).open()
+
+    def maybe_switch_to_server(self, server_str: str):
+        net_params = self.network.get_parameters()
+        try:
+            server = ServerAddr.from_str_with_inference(server_str)
+            if not server: raise Exception("failed to parse")
+        except Exception as e:
+            self.show_error(_("Invalid server details: {}").format(repr(e)))
+            return
+        net_params = net_params._replace(server=server)
+        self.network.run_from_another_thread(self.network.set_parameters(net_params))
 
     def choose_blockchain_dialog(self, dt):
         from .uix.dialogs.choice_dialog import ChoiceDialog
@@ -179,23 +196,31 @@ class ElectrumWindow(App):
     def on_use_unconfirmed(self, instance, x):
         self.electrum_config.set_key('confirmed_only', not self.use_unconfirmed, True)
 
+    def switch_to_send_screen(func):
+        # try until send_screen is available
+        def wrapper(self, *args):
+            f = lambda dt: (bool(func(self, *args) and False) if self.send_screen else bool(self.switch_to('send') or True)) if self.wallet else True
+            Clock.schedule_interval(f, 0.1)
+        return wrapper
+
+    @switch_to_send_screen
     def set_URI(self, uri):
-        self.switch_to('send')
         self.send_screen.set_URI(uri)
 
+    @switch_to_send_screen
     def set_ln_invoice(self, invoice):
-        self.switch_to('send')
         self.send_screen.set_ln_invoice(invoice)
 
     def on_new_intent(self, intent):
-        data = intent.getDataString()
-        if intent.getScheme() == 'bitcoin':
+        data = str(intent.getDataString())
+        scheme = str(intent.getScheme()).lower()
+        if scheme == BITCOIN_BIP21_URI_SCHEME:
             self.set_URI(data)
-        elif intent.getScheme() == 'lightning':
+        elif scheme == LIGHTNING_URI_SCHEME:
             self.set_ln_invoice(data)
 
     def on_language(self, instance, language):
-        Logger.info('language: {}'.format(language))
+        self.logger.info('language: {}'.format(language))
         _.switch_lang(language)
 
     def update_history(self, *dt):
@@ -203,12 +228,12 @@ class ElectrumWindow(App):
             self.history_screen.update()
 
     def on_quotes(self, d):
-        Logger.info("on_quotes")
+        self.logger.info("on_quotes")
         self._trigger_update_status()
         self._trigger_update_history()
 
     def on_history(self, d):
-        Logger.info("on_history")
+        self.logger.info("on_history")
         if self.wallet:
             self.wallet.clear_coin_price_cache()
         self._trigger_update_history()
@@ -216,42 +241,39 @@ class ElectrumWindow(App):
     def on_fee_histogram(self, *args):
         self._trigger_update_history()
 
-    def on_request_status(self, event, key, status):
+    def on_request_status(self, event, wallet, key, status):
         if key not in self.wallet.receive_requests:
             return
         self.update_tab('receive')
         if self.request_popup and self.request_popup.key == key:
-            self.request_popup.set_status(status)
+            self.request_popup.update_status()
         if status == PR_PAID:
             self.show_info(_('Payment Received') + '\n' + key)
             self._trigger_update_history()
 
-    def on_invoice_status(self, event, key):
+    def on_invoice_status(self, event, wallet, key):
         req = self.wallet.get_invoice(key)
         if req is None:
             return
-        status = req['status']
+        status = self.wallet.get_invoice_status(req)
         # todo: update single item
         self.update_tab('send')
         if self.invoice_popup and self.invoice_popup.key == key:
-            self.invoice_popup.set_status(status)
-        if status == PR_PAID:
-            self.show_info(_('Payment was sent'))
-            self._trigger_update_history()
-        elif status == PR_FAILED:
-            self.show_info(_('Payment failed'))
+            self.invoice_popup.update_status()
+
+    def on_payment_succeeded(self, event, wallet, key):
+        description = self.wallet.get_label(key)
+        self.show_info(_('Payment succeeded') + '\n\n' + description)
+        self._trigger_update_history()
+
+    def on_payment_failed(self, event, wallet, key, reason):
+        self.show_info(_('Payment failed') + '\n\n' + reason)
 
     def _get_bu(self):
-        decimal_point = self.electrum_config.get('decimal_point', DECIMAL_POINT_DEFAULT)
-        try:
-            return decimal_point_to_base_unit_name(decimal_point)
-        except UnknownBaseUnit:
-            return decimal_point_to_base_unit_name(DECIMAL_POINT_DEFAULT)
+        return self.electrum_config.get_base_unit()
 
     def _set_bu(self, value):
-        assert value in base_units.keys()
-        decimal_point = base_unit_name_to_decimal_point(value)
-        self.electrum_config.set_key('decimal_point', decimal_point, True)
+        self.electrum_config.set_base_unit(value)
         self._trigger_update_status()
         self._trigger_update_history()
 
@@ -263,7 +285,7 @@ class ElectrumWindow(App):
         self._trigger_update_history()
 
     def decimal_point(self):
-        return base_units[self.base_unit]
+        return self.electrum_config.get_decimal_point()
 
     def btc_to_fiat(self, amount_str):
         if not amount_str:
@@ -283,7 +305,7 @@ class ElectrumWindow(App):
         if rate.is_nan():
             return ''
         satoshis = int(pow(10,8) * Decimal(fiat_amount) / Decimal(rate))
-        return format_satoshis_plain(satoshis, self.decimal_point())
+        return format_satoshis_plain(satoshis, decimal_point=self.decimal_point())
 
     def get_amount(self, amount_str):
         a, u = amount_str.split()
@@ -338,6 +360,7 @@ class ElectrumWindow(App):
         self.password = None
 
         App.__init__(self)#, **kwargs)
+        Logger.__init__(self)
 
         self.electrum_config = config = kwargs.get('config', None)  # type: SimpleConfig
         self.language = config.get('language', 'en')
@@ -346,8 +369,8 @@ class ElectrumWindow(App):
             self.num_blocks = self.network.get_local_height()
             self.num_nodes = len(self.network.get_interfaces())
             net_params = self.network.get_parameters()
-            self.server_host = net_params.host
-            self.server_port = net_params.port
+            self.server_host = net_params.server.host
+            self.server_port = str(net_params.server.port)
             self.auto_connect = net_params.auto_connect
             self.oneserver = net_params.oneserver
             self.proxy_config = net_params.proxy if net_params.proxy else {}
@@ -358,6 +381,7 @@ class ElectrumWindow(App):
         self.daemon = self.gui_object.daemon
         self.fx = self.daemon.fx
         self.use_rbf = config.get('use_rbf', True)
+        self.android_backups = config.get('android_backups', False)
         self.use_unconfirmed = not config.get('confirmed_only', False)
 
         # create triggers so as to minimize updating a max of 2 times a sec
@@ -370,7 +394,6 @@ class ElectrumWindow(App):
 
         # cached dialogs
         self._settings_dialog = None
-        self._password_dialog = None
         self._channels_dialog = None
         self._addresses_dialog = None
         self.fee_status = self.electrum_config.get_fee_status()
@@ -384,7 +407,7 @@ class ElectrumWindow(App):
         if pr.verify(self.wallet.contacts):
             key = pr.get_id()
             invoice = self.wallet.get_invoice(key)  # FIXME wrong key...
-            if invoice and invoice['status'] == PR_PAID:
+            if invoice and self.wallet.get_invoice_status(invoice) == PR_PAID:
                 self.show_error("invoice already paid")
                 self.send_screen.do_clear()
             elif pr.has_expired():
@@ -402,9 +425,12 @@ class ElectrumWindow(App):
         if is_address(data):
             self.set_URI(data)
             return
-        if data.startswith('bitcoin:'):
+        if data.lower().startswith(BITCOIN_BIP21_URI_SCHEME + ':'):
             self.set_URI(data)
             return
+        if data.lower().startswith('channel_backup:'):
+            self.import_channel_backup(data)
+            return
         bolt11_invoice = maybe_extract_bolt11_invoice(data)
         if bolt11_invoice is not None:
             self.set_ln_invoice(bolt11_invoice)
@@ -428,24 +454,18 @@ class ElectrumWindow(App):
 
     @profiler
     def update_tabs(self):
-        for tab in ['invoices', 'send', 'history', 'receive', 'address']:
-            self.update_tab(tab)
+        for name in ['send', 'history', 'receive']:
+            self.update_tab(name)
 
     def switch_to(self, name):
         s = getattr(self, name + '_screen', None)
-        if s is None:
-            s = self.tabs.ids[name + '_screen']
-            s.load_screen()
         panel = self.tabs.ids.panel
         tab = self.tabs.ids[name + '_tab']
         panel.switch_to(tab)
 
     def show_request(self, is_lightning, key):
         from .uix.dialogs.request_dialog import RequestDialog
-        request = self.wallet.get_request(key)
-        data = request['invoice'] if is_lightning else request['URI']
-        self.request_popup = RequestDialog('Request', data, key, is_lightning=is_lightning)
-        self.request_popup.set_status(request['status'])
+        self.request_popup = RequestDialog('Request', key)
         self.request_popup.open()
 
     def show_invoice(self, is_lightning, key):
@@ -453,13 +473,11 @@ class ElectrumWindow(App):
         invoice = self.wallet.get_invoice(key)
         if not invoice:
             return
-        status = invoice['status']
-        data = invoice['invoice'] if is_lightning else key
+        data = invoice.invoice if is_lightning else key
         self.invoice_popup = InvoiceDialog('Invoice', data, key)
-        self.invoice_popup.set_status(status)
         self.invoice_popup.open()
 
-    def qr_dialog(self, title, data, show_text=False, text_for_clipboard=None):
+    def qr_dialog(self, title, data, show_text=False, text_for_clipboard=None, help_text=None):
         from .uix.dialogs.qr_dialog import QRDialog
         def on_qr_failure():
             popup.dismiss()
@@ -468,8 +486,11 @@ class ElectrumWindow(App):
                 msg += '\n' + _('Text copied to clipboard.')
                 self._clipboard.copy(text_for_clipboard)
             Clock.schedule_once(lambda dt: self.show_info(msg))
-        popup = QRDialog(title, data, show_text, failure_cb=on_qr_failure,
-                         text_for_clipboard=text_for_clipboard)
+        popup = QRDialog(
+            title, data, show_text,
+            failure_cb=on_qr_failure,
+            text_for_clipboard=text_for_clipboard,
+            help_text=help_text)
         popup.open()
 
     def scan_qr(self, on_complete):
@@ -513,7 +534,7 @@ class ElectrumWindow(App):
         currentActivity.startActivity(it)
 
     def build(self):
-        return Builder.load_file('electrum/gui/kivy/main.kv')
+        return Builder.load_file(KIVY_GUI_PATH + '/main.kv')
 
     def _pause(self):
         if platform == 'android':
@@ -528,6 +549,7 @@ class ElectrumWindow(App):
             try:
                 return func(self, *args, **kwargs)
             except Exception as e:
+                self.logger.exception('crash on startup')
                 from .uix.dialogs.crash_reporter import CrashReporter
                 # show the crash reporter, and when it's closed, shutdown the app
                 cr = CrashReporter(self, exctype=type(e), value=e, tb=e.__traceback__)
@@ -540,9 +562,8 @@ class ElectrumWindow(App):
         ''' This is the start point of the kivy ui
         '''
         import time
-        Logger.info('Time to on_start: {} <<<<<<<<'.format(time.process_time()))
+        self.logger.info('Time to on_start: {} <<<<<<<<'.format(time.process_time()))
         Window.bind(size=self.on_size, on_keyboard=self.on_keyboard)
-        Window.bind(on_key_down=self.on_key_down)
         #Window.softinput_mode = 'below_target'
         self.on_size(Window, Window.size)
         self.init_ui()
@@ -565,18 +586,20 @@ class ElectrumWindow(App):
         if self.network:
             interests = ['wallet_updated', 'network_updated', 'blockchain_updated',
                          'status', 'new_transaction', 'verified']
-            self.network.register_callback(self.on_network_event, interests)
-            self.network.register_callback(self.on_fee, ['fee'])
-            self.network.register_callback(self.on_fee_histogram, ['fee_histogram'])
-            self.network.register_callback(self.on_quotes, ['on_quotes'])
-            self.network.register_callback(self.on_history, ['on_history'])
-            self.network.register_callback(self.on_channels, ['channels_updated'])
-            self.network.register_callback(self.on_channel, ['channel'])
-            self.network.register_callback(self.on_invoice_status, ['invoice_status'])
-            self.network.register_callback(self.on_request_status, ['request_status'])
-            self.network.register_callback(self.on_channel_db, ['channel_db'])
-            self.network.register_callback(self.set_num_peers, ['gossip_peers'])
-            self.network.register_callback(self.set_unknown_channels, ['unknown_channels'])
+            util.register_callback(self.on_network_event, interests)
+            util.register_callback(self.on_fee, ['fee'])
+            util.register_callback(self.on_fee_histogram, ['fee_histogram'])
+            util.register_callback(self.on_quotes, ['on_quotes'])
+            util.register_callback(self.on_history, ['on_history'])
+            util.register_callback(self.on_channels, ['channels_updated'])
+            util.register_callback(self.on_channel, ['channel'])
+            util.register_callback(self.on_invoice_status, ['invoice_status'])
+            util.register_callback(self.on_request_status, ['request_status'])
+            util.register_callback(self.on_payment_failed, ['payment_failed'])
+            util.register_callback(self.on_payment_succeeded, ['payment_succeeded'])
+            util.register_callback(self.on_channel_db, ['channel_db'])
+            util.register_callback(self.set_num_peers, ['gossip_peers'])
+            util.register_callback(self.set_unknown_channels, ['unknown_channels'])
         # load wallet
         self.load_wallet_by_name(self.electrum_config.get_wallet_path(use_gui_last_wallet=True))
         # URI passed in config
@@ -600,84 +623,39 @@ class ElectrumWindow(App):
         else:
             return ''
 
-    def on_wizard_complete(self, wizard, storage, db):
-        if storage:
-            wallet = Wallet(db, storage, config=self.electrum_config)
-            wallet.start_network(self.daemon.network)
-            self.daemon.add_wallet(wallet)
-            self.load_wallet(wallet)
-        elif not self.wallet:
-            # wizard did not return a wallet; and there is no wallet open atm
-            # try to open last saved wallet (potentially start wizard again)
-            self.load_wallet_by_name(self.electrum_config.get_wallet_path(use_gui_last_wallet=True),
-                                     ask_if_wizard=True)
+    def on_wizard_success(self, storage, db, password):
+        self.password = password
+        wallet = Wallet(db, storage, config=self.electrum_config)
+        wallet.start_network(self.daemon.network)
+        self.daemon.add_wallet(wallet)
+        self.load_wallet(wallet)
 
-    def _on_decrypted_storage(self, storage: WalletStorage):
-        assert storage.is_past_initial_decryption()
-        db = WalletDB(storage.read(), manual_upgrades=False)
-        if db.requires_upgrade():
-            wizard = Factory.InstallWizard(self.electrum_config, self.plugins)
-            wizard.path = storage.path
-            wizard.bind(on_wizard_complete=self.on_wizard_complete)
-            wizard.upgrade_storage(storage, db)
-        else:
-            self.on_wizard_complete(None, storage, db)
+    def on_wizard_aborted(self):
+        # wizard did not return a wallet; and there is no wallet open atm
+        if not self.wallet:
+            self.stop()
 
-    def load_wallet_by_name(self, path, ask_if_wizard=False):
+    def load_wallet_by_name(self, path):
         if not path:
             return
         if self.wallet and self.wallet.storage.path == path:
             return
-        wallet = self.daemon.load_wallet(path, None)
-        if wallet:
-            if wallet.has_password():
-                def on_success(x):
-                    # save pin_code so that we can create backups
-                    self.password = x
-                    self.load_wallet(wallet)
-                self.password_dialog(
-                    check_password=wallet.check_password,
-                    on_success=on_success,
-                    on_failure=self.stop)
-            else:
-                self.load_wallet(wallet)
+        d = OpenWalletDialog(self, path, self.on_open_wallet)
+        d.open()
+
+    def on_open_wallet(self, password, storage):
+        if not storage.file_exists():
+            wizard = InstallWizard(self.electrum_config, self.plugins)
+            wizard.path = storage.path
+            wizard.run('new')
         else:
-            def launch_wizard():
-                storage = WalletStorage(path)
-                if not storage.file_exists():
-                    wizard = Factory.InstallWizard(self.electrum_config, self.plugins)
-                    wizard.path = path
-                    wizard.bind(on_wizard_complete=self.on_wizard_complete)
-                    wizard.run('new')
-                else:
-                    if storage.is_encrypted():
-                        if not storage.is_encrypted_with_user_pw():
-                            raise Exception("Kivy GUI does not support this type of encrypted wallet files.")
-                        def on_password(pw):
-                            self.password = pw
-                            storage.decrypt(pw)
-                            self._on_decrypted_storage(storage)
-                        self.password_dialog(
-                            check_password=storage.check_password,
-                            on_success=on_password,
-                            on_failure=self.stop)
-                        return
-                    self._on_decrypted_storage(storage)
-            if not ask_if_wizard:
-                launch_wizard()
-            else:
-                def handle_answer(b: bool):
-                    if b:
-                        launch_wizard()
-                    else:
-                        try: os.unlink(path)
-                        except FileNotFoundError: pass
-                        self.stop()
-                d = Question(_('Do you want to launch the wizard again?'), handle_answer)
-                d.open()
+            assert storage.is_past_initial_decryption()
+            db = WalletDB(storage.read(), manual_upgrades=False)
+            assert not db.requires_upgrade()
+            self.on_wizard_success(storage, db, password)
 
     def on_stop(self):
-        Logger.info('on_stop')
+        self.logger.info('on_stop')
         self.stop_wallet()
 
     def stop_wallet(self):
@@ -685,25 +663,6 @@ class ElectrumWindow(App):
             self.daemon.stop_wallet(self.wallet.storage.path)
             self.wallet = None
 
-    def on_key_down(self, instance, key, keycode, codepoint, modifiers):
-        if 'ctrl' in modifiers:
-            # q=24 w=25
-            if keycode in (24, 25):
-                self.stop()
-            elif keycode == 27:
-                # r=27
-                # force update wallet
-                self.update_wallet()
-            elif keycode == 112:
-                # pageup
-                #TODO move to next tab
-                pass
-            elif keycode == 117:
-                # pagedown
-                #TODO move to prev tab
-                pass
-        #TODO: alt+tab_number to activate the particular tab
-
     def on_keyboard(self, instance, key, keycode, codepoint, modifiers):
         if key == 27 and self.is_exit is False:
             self.is_exit = True
@@ -722,18 +681,33 @@ class ElectrumWindow(App):
         self._settings_dialog.open()
 
     def lightning_open_channel_dialog(self):
-        d = LightningOpenChannelDialog(self)
-        d.open()
+        if not self.wallet.has_lightning():
+            self.show_error(_('Lightning is not enabled for this wallet'))
+            return
+        if not self.wallet.lnworker.channels:
+            warning1 = _("Lightning support in Electrum is experimental. "
+                         "Do not put large amounts in lightning channels.")
+            warning2 = _("Funds stored in lightning channels are not recoverable "
+                         "from your seed. You must backup your wallet file everytime "
+                         "you create a new channel.")
+            d = Question(_('Do you want to create your first channel?') +
+                         '\n\n' + warning1 + '\n\n' + warning2, self.open_channel_dialog_with_warning)
+            d.open()
+        else:
+            d = LightningOpenChannelDialog(self)
+            d.open()
+
+    def open_channel_dialog_with_warning(self, b):
+        if b:
+            d = LightningOpenChannelDialog(self)
+            d.open()
 
     def lightning_channels_dialog(self):
-        if not self.wallet.has_lightning():
-            self.show_error('Lightning not enabled on this wallet')
-            return
         if self._channels_dialog is None:
             self._channels_dialog = LightningChannelsDialog(self)
         self._channels_dialog.open()
 
-    def on_channel(self, evt, chan):
+    def on_channel(self, evt, wallet, chan):
         if self._channels_dialog:
             Clock.schedule_once(lambda dt: self._channels_dialog.update())
 
@@ -741,15 +715,19 @@ class ElectrumWindow(App):
         if self._channels_dialog:
             Clock.schedule_once(lambda dt: self._channels_dialog.update())
 
+    def wallets_dialog(self):
+        from .uix.dialogs.wallets import WalletDialog
+        dirname = os.path.dirname(self.electrum_config.get_wallet_path())
+        d = WalletDialog(dirname, self.load_wallet_by_name)
+        d.open()
+
     def popup_dialog(self, name):
         if name == 'settings':
             self.settings_dialog()
         elif name == 'wallets':
-            from .uix.dialogs.wallets import WalletDialog
-            d = WalletDialog()
-            d.open()
+            self.wallets_dialog()
         elif name == 'status':
-            popup = Builder.load_file('electrum/gui/kivy/uix/ui_screens/'+name+'.kv')
+            popup = Builder.load_file(KIVY_GUI_PATH + f'/uix/ui_screens/{name}.kv')
             master_public_keys_layout = popup.ids.master_public_keys
             for xpub in self.wallet.get_master_public_keys()[1:]:
                 master_public_keys_layout.add_widget(TopLabel(text=_('Master Public Key')))
@@ -758,10 +736,13 @@ class ElectrumWindow(App):
                 ref.data = xpub
                 master_public_keys_layout.add_widget(ref)
             popup.open()
+        elif name == 'lightning_channels_dialog' and not self.wallet.can_have_lightning():
+            self.show_error(_("Not available for this wallet.") + "\n\n" +
+                            _("Lightning is currently restricted to HD wallets with p2wpkh addresses."))
         elif name.endswith("_dialog"):
             getattr(self, name)()
         else:
-            popup = Builder.load_file('electrum/gui/kivy/uix/ui_screens/'+name+'.kv')
+            popup = Builder.load_file(KIVY_GUI_PATH + f'/uix/ui_screens/{name}.kv')
             popup.open()
 
     @profiler
@@ -789,13 +770,9 @@ class ElectrumWindow(App):
         self.root.manager = self.root.ids['manager']
 
         self.history_screen = None
-        self.contacts_screen = None
         self.send_screen = None
-        self.invoices_screen = None
         self.receive_screen = None
-        self.requests_screen = None
-        self.address_screen = None
-        self.icon = "electrum/gui/icons/electrum.png"
+        self.icon = os.path.dirname(KIVY_GUI_PATH) + "/icons/electrum.png"
         self.tabs = self.root.ids['tabs']
 
     def update_interfaces(self, dt):
@@ -809,12 +786,12 @@ class ElectrumWindow(App):
         if interface:
             self.server_host = interface.host
         else:
-            self.server_host = str(net_params.host) + ' (connecting...)'
+            self.server_host = str(net_params.server.host) + ' (connecting...)'
         self.proxy_config = net_params.proxy or {}
         self.update_proxy_str(self.proxy_config)
 
     def on_network_event(self, event, *args):
-        Logger.info('network event: '+ event)
+        self.logger.info('network event: '+ event)
         if event == 'network_updated':
             self._trigger_update_interfaces()
             self._trigger_update_status()
@@ -852,6 +829,17 @@ class ElectrumWindow(App):
             return
         self.use_change = self.wallet.use_change
         self.electrum_config.save_last_wallet(wallet)
+        self.request_focus_for_main_view()
+
+    def request_focus_for_main_view(self):
+        if platform != 'android':
+            return
+        # The main view of the activity might be not have focus
+        # in which case e.g. the OS "back" button would not work.
+        # see #6276 (specifically "method 2" and "method 3")
+        from jnius import autoclass
+        PythonActivity = autoclass('org.kivy.android.PythonActivity')
+        PythonActivity.requestFocusForMainView()
 
     def update_status(self, *dt):
         if not self.wallet:
@@ -877,9 +865,11 @@ class ElectrumWindow(App):
             self.fiat_balance = status
         else:
             c, u, x = self.wallet.get_balance()
-            text = self.format_amount(c+x+u)
+            l = int(self.wallet.lnworker.get_balance()) if self.wallet.lnworker else 0
+            balance_sat = c + u + x + l
+            text = self.format_amount(balance_sat)
             self.balance = str(text.strip()) + ' [size=22dp]%s[/size]'% self.base_unit
-            self.fiat_balance = self.fx.format_amount(c+u+x) + ' [size=22dp]%s[/size]'% self.fx.ccy
+            self.fiat_balance = self.fx.format_amount(balance_sat) + ' [size=22dp]%s[/size]'% self.fx.ccy
 
     def update_wallet_synchronizing_progress(self, *dt):
         if not self.wallet:
@@ -896,7 +886,7 @@ class ElectrumWindow(App):
             return ''
         addr = None
         if self.send_screen:
-            addr = str(self.send_screen.screen.address)
+            addr = str(self.send_screen.address)
         if not addr:
             addr = self.wallet.dummy_address()
         outputs = [PartialTxOutput.from_address_and_value(addr, '!')]
@@ -914,13 +904,23 @@ class ElectrumWindow(App):
         amount = tx.output_value()
         __, x_fee_amount = run_hook('get_tx_extra_fee', self.wallet, tx) or (None, 0)
         amount_after_all_fees = amount - x_fee_amount
-        return format_satoshis_plain(amount_after_all_fees, self.decimal_point())
+        return format_satoshis_plain(amount_after_all_fees, decimal_point=self.decimal_point())
 
     def format_amount(self, x, is_diff=False, whitespaces=False):
-        return format_satoshis(x, 0, self.decimal_point(), is_diff=is_diff, whitespaces=whitespaces)
+        return format_satoshis(
+            x,
+            num_zeros=0,
+            decimal_point=self.decimal_point(),
+            is_diff=is_diff,
+            whitespaces=whitespaces,
+        )
 
-    def format_amount_and_units(self, x):
-        return format_satoshis_plain(x, self.decimal_point()) + ' ' + self.base_unit
+    def format_amount_and_units(self, x) -> str:
+        if x is None:
+            return 'none'
+        if x == '!':
+            return 'max'
+        return format_satoshis_plain(x, decimal_point=self.decimal_point()) + ' ' + self.base_unit
 
     def format_fee_rate(self, fee_rate):
         # fee_rate is in sat/kB
@@ -942,7 +942,7 @@ class ElectrumWindow(App):
             notification.notify('Electrum', message,
                             app_icon=icon, app_name='Electrum')
         except ImportError:
-            Logger.Error('Notification: needs plyer; `sudo python3 -m pip install plyer`')
+            self.logger.Error('Notification: needs plyer; `sudo python3 -m pip install plyer`')
 
     def on_pause(self):
         self.pause_time = time.time()
@@ -953,8 +953,13 @@ class ElectrumWindow(App):
 
     def on_resume(self):
         now = time.time()
-        if self.wallet and self.wallet.has_password() and now - self.pause_time > 5*60:
-            self.password_dialog(check_password=self.check_pin_code, on_success=None, on_failure=self.stop, is_password=False)
+        if self.wallet and self.has_pin_code() and now - self.pause_time > 5*60:
+            d = PincodeDialog(
+                self,
+                check_password=self.check_pin_code,
+                on_success=None,
+                on_failure=self.stop)
+            d.open()
         if self.nfcscanner:
             self.nfcscanner.nfc_enable()
 
@@ -969,8 +974,8 @@ class ElectrumWindow(App):
         self.qr_dialog(label.name, label.data, True)
 
     def show_error(self, error, width='200dp', pos=None, arrow_pos=None,
-        exit=False, icon='atlas://electrum/gui/kivy/theming/light/error', duration=0,
-        modal=False):
+                   exit=False, icon=f'atlas://{KIVY_GUI_PATH}/theming/light/error', duration=0,
+                   modal=False):
         ''' Show an error Message Bubble.
         '''
         self.show_info_bubble( text=error, icon=icon, width=width,
@@ -978,15 +983,15 @@ class ElectrumWindow(App):
             duration=duration, modal=modal)
 
     def show_info(self, error, width='200dp', pos=None, arrow_pos=None,
-        exit=False, duration=0, modal=False):
+                  exit=False, duration=0, modal=False):
         ''' Show an Info Message Bubble.
         '''
-        self.show_error(error, icon='atlas://electrum/gui/kivy/theming/light/important',
+        self.show_error(error, icon=f'atlas://{KIVY_GUI_PATH}/theming/light/important',
             duration=duration, modal=modal, exit=exit, pos=pos,
             arrow_pos=arrow_pos)
 
     def show_info_bubble(self, text=_('Hello World'), pos=None, duration=0,
-        arrow_pos='bottom_mid', width=None, icon='', modal=False, exit=False):
+                         arrow_pos='bottom_mid', width=None, icon='', modal=False, exit=False):
         '''Method to show an Information Bubble
 
         .. parameters::
@@ -996,6 +1001,7 @@ class ElectrumWindow(App):
             width: width of the Bubble
             arrow_pos: arrow position for the bubble
         '''
+        text = str(text)  # so that we also handle e.g. Exception
         info_bubble = self.info_bubble
         if not info_bubble:
             info_bubble = self.info_bubble = Factory.InfoBubble()
@@ -1021,7 +1027,7 @@ class ElectrumWindow(App):
             info_bubble.show_arrow = False
             img.allow_stretch = True
             info_bubble.dim_background = True
-            info_bubble.background_image = 'atlas://electrum/gui/kivy/theming/light/card'
+            info_bubble.background_image = f'atlas://{KIVY_GUI_PATH}/theming/light/card'
         else:
             info_bubble.fs = False
             info_bubble.icon = icon
@@ -1040,6 +1046,15 @@ class ElectrumWindow(App):
         d = TxDialog(self, tx)
         d.open()
 
+    def show_transaction(self, txid):
+        tx = self.wallet.db.get_transaction(txid)
+        if not tx and self.wallet.lnworker:
+            tx = self.wallet.lnworker.lnwatcher.db.get_transaction(txid)
+        if tx:
+            self.tx_dialog(tx)
+        else:
+            self.show_error(f'Transaction not found {txid}')
+
     def lightning_tx_dialog(self, tx):
         from .uix.dialogs.lightning_tx_dialog import LightningTxDialog
         d = LightningTxDialog(self, tx)
@@ -1100,7 +1115,13 @@ class ElectrumWindow(App):
             amount, u = str(amount).split()
             assert u == self.base_unit
         def cb(amount):
-            screen.amount = amount
+            if amount == '!':
+                screen.is_max = True
+                max_amt = self.get_max_amount()
+                screen.amount = (max_amt + ' ' + self.base_unit) if max_amt else ''
+            else:
+                screen.amount = amount
+                screen.is_max = False
         popup = AmountDialog(show_max, amount, cb)
         popup.open()
 
@@ -1123,47 +1144,23 @@ class ElectrumWindow(App):
 
     def protected(self, msg, f, args):
         if self.electrum_config.get('pin_code'):
-            on_success = lambda pw: f(*(args + (self.password,)))
-            self.password_dialog(
+            msg += "\n" + _("Enter your PIN code to proceed")
+            on_success = lambda pw: f(*args, self.password)
+            d = PincodeDialog(
+                self,
                 message = msg,
                 check_password=self.check_pin_code,
                 on_success=on_success,
-                on_failure=lambda: None,
-                is_password=False)
-        else:
-            f(*(args + (self.password,)))
-
-    def toggle_lightning(self):
-        if self.wallet.has_lightning():
-            if not bool(self.wallet.lnworker.channels):
-                warning = _('This will delete your lightning private keys')
-                d = Question(_('Disable Lightning?') + '\n\n' + warning, self._disable_lightning)
-                d.open()
-            else:
-                self.show_info('This wallet has channels')
-        else:
-            warning1 = _("Lightning support in Electrum is experimental. Do not put large amounts in lightning channels.")
-            warning2 = _("Funds stored in lightning channels are not recoverable from your seed. You must backup your wallet file everytime you create a new channel.")
-            d = Question(_('Enable Lightning?') + '\n\n' + warning1 + '\n\n' + warning2, self._enable_lightning)
+                on_failure=lambda: None)
+            d.open()
+        else:
+            d = Question(
+                msg,
+                lambda b: f(*args, self.password) if b else None,
+                yes_str=_("OK"),
+                no_str=_("Cancel"),
+                title=_("Confirm action"))
             d.open()
-
-    def _enable_lightning(self, b):
-        if not b:
-            return
-        wallet_path = self.get_wallet_path()
-        self.wallet.init_lightning()
-        self.show_info(_('Lightning keys have been initialized.'))
-        self.stop_wallet()
-        self.load_wallet_by_name(wallet_path)
-
-    def _disable_lightning(self, b):
-        if not b:
-            return
-        wallet_path = self.get_wallet_path()
-        self.wallet.remove_lightning()
-        self.show_info(_('Lightning keys have been removed.'))
-        self.stop_wallet()
-        self.load_wallet_by_name(wallet_path)
 
     def delete_wallet(self):
         basename = os.path.basename(self.wallet.storage.path)
@@ -1173,7 +1170,8 @@ class ElectrumWindow(App):
     def _delete_wallet(self, b):
         if b:
             basename = self.wallet.basename()
-            self.protected(_("Enter your PIN code to confirm deletion of {}").format(basename), self.__delete_wallet, ())
+            self.protected(_("Are you sure you want to delete wallet {}?").format(basename),
+                           self.__delete_wallet, ())
 
     def __delete_wallet(self, pw):
         wallet_path = self.get_wallet_path()
@@ -1181,8 +1179,8 @@ class ElectrumWindow(App):
         if self.wallet.has_password():
             try:
                 self.wallet.check_password(pw)
-            except:
-                self.show_error("Invalid PIN")
+            except InvalidPassword:
+                self.show_error("Invalid password")
                 return
         self.stop_wallet()
         os.unlink(wallet_path)
@@ -1191,7 +1189,7 @@ class ElectrumWindow(App):
         self.load_wallet_by_name(new_path)
 
     def show_seed(self, label):
-        self.protected(_("Enter PIN code to display your seed"), self._show_seed, (label,))
+        self.protected(_("Display your seed?"), self._show_seed, (label,))
 
     def _show_seed(self, label, password):
         if self.wallet.has_password() and password is None:
@@ -1210,38 +1208,29 @@ class ElectrumWindow(App):
         if pin != self.electrum_config.get('pin_code'):
             raise InvalidPassword
 
-    def password_dialog(self, **kwargs):
-        if self._password_dialog is None:
-            self._password_dialog = PasswordDialog()
-        self._password_dialog.init(self, **kwargs)
-        self._password_dialog.open()
-
     def change_password(self, cb):
         def on_success(old_password, new_password):
             self.wallet.update_password(old_password, new_password)
             self.password = new_password
             self.show_info(_("Your password was updated"))
         on_failure = lambda: self.show_error(_("Password not updated"))
-        self.password_dialog(
-            check_password = self.wallet.check_password,
-            on_success=on_success, on_failure=on_failure,
-            is_change=True, is_password=True,
-            has_password=self.wallet.has_password())
+        d = ChangePasswordDialog(self, self.wallet, on_success, on_failure)
+        d.open()
 
     def change_pin_code(self, cb):
-        if self._password_dialog is None:
-            self._password_dialog = PasswordDialog()
         def on_success(old_password, new_password):
             self.electrum_config.set_key('pin_code', new_password)
             cb()
             self.show_info(_("PIN updated") if new_password else _('PIN disabled'))
         on_failure = lambda: self.show_error(_("PIN not updated"))
-        self._password_dialog.init(
-            self, check_password=self.check_pin_code,
-            on_success=on_success, on_failure=on_failure,
-            is_change=True, is_password=False,
+        d = PincodeDialog(
+            self,
+            check_password=self.check_pin_code,
+            on_success=on_success,
+            on_failure=on_failure,
+            is_change=True,
             has_password = self.has_pin_code())
-        self._password_dialog.open()
+        d.open()
 
     def save_backup(self):
         if platform != 'android':
@@ -1259,7 +1248,12 @@ class ElectrumWindow(App):
         request_permissions([Permission.WRITE_EXTERNAL_STORAGE], cb)
 
     def _save_backup(self):
-        new_path = self.wallet.save_backup()
+        try:
+            new_path = self.wallet.save_backup()
+        except Exception as e:
+            self.logger.exception("Failed to save wallet backup")
+            self.show_error("Failed to save wallet backup" + '\n' + str(e))
+            return
         if new_path:
             self.show_info(_("Backup saved:") + f"\n{new_path}")
         else:
@@ -1280,4 +1274,19 @@ class ElectrumWindow(App):
             except InvalidPassword:
                 self.show_error("Invalid PIN")
                 return
-        self.protected(_("Enter your PIN code in order to decrypt your private key"), show_private_key, (addr, pk_label))
+        self.protected(_("Decrypt your private key?"), show_private_key, (addr, pk_label))
+
+    def import_channel_backup(self, encrypted):
+        d = Question(_('Import Channel Backup?'), lambda b: self._import_channel_backup(b, encrypted))
+        d.open()
+
+    def _import_channel_backup(self, b, encrypted):
+        if not b:
+            return
+        try:
+            self.wallet.lnbackups.import_channel_backup(encrypted)
+        except Exception as e:
+            self.logger.exception("failed to import backup")
+            self.show_error("failed to import backup" + '\n' + str(e))
+            return
+        self.lightning_channels_dialog()

electrum/gui/kivy/uix/dialogs/__init__.py

@@ -81,9 +81,6 @@ class EventsDialog(Factory.Popup):
     def on_press(self, instance):
         pass
 
-    def close(self):
-        self.dismiss()
-
 
 class SelectionDialog(EventsDialog):
 

electrum/gui/kivy/uix/dialogs/addresses.py

@@ -1,6 +1,7 @@
 from typing import TYPE_CHECKING
 
 from kivy.app import App
+from kivy.clock import Clock
 from kivy.factory import Factory
 from kivy.properties import ObjectProperty
 from kivy.lang import Builder
@@ -41,6 +42,26 @@ Builder.load_string('''
             shorten: True
         Widget
 
+<AddressButton@Button>:
+    background_color: 1, .585, .878, 0
+    halign: 'center'
+    text_size: (self.width, None)
+    shorten: True
+    size_hint: 0.5, None
+    default_text: ''
+    text: self.default_text
+    padding: '5dp', '5dp'
+    height: '40dp'
+    text_color: self.foreground_color
+    disabled_color: 1, 1, 1, 1
+    foreground_color: 1, 1, 1, 1
+    canvas.before:
+        Color:
+            rgba: (0.9, .498, 0.745, 1) if self.state == 'down' else self.background_color
+        Rectangle:
+            size: self.size
+            pos: self.pos
+
 <AddressesDialog@Popup>
     id: popup
     title: _('Addresses')
@@ -52,12 +73,12 @@ Builder.load_string('''
         self.update()
     BoxLayout:
         id:box
-        padding: '12dp', '70dp', '12dp', '12dp'
+        padding: '12dp', '12dp', '12dp', '12dp'
         spacing: '12dp'
         orientation: 'vertical'
-        size_hint: 1, 1.1
         BoxLayout:
             spacing: '6dp'
+            height: self.minimum_height
             size_hint: 1, None
             orientation: 'horizontal'
             AddressFilter:
@@ -186,7 +207,8 @@ class AddressPopup(Popup):
         self.dismiss()
         self.parent_dialog.dismiss()
         self.app.switch_to('receive')
-        self.app.receive_screen.set_address(self.address)
+        # retry until receive_screen is set
+        Clock.schedule_interval(lambda dt: bool(self.app.receive_screen.set_address(self.address) and False) if self.app.receive_screen else True, 0.1)
 
     def do_export(self, pk_label):
         self.app.export_private_keys(pk_label, self.address)
@@ -220,7 +242,7 @@ class AddressesDialog(Factory.Popup):
         n = 0
         cards = []
         for address in _list:
-            label = wallet.labels.get(address, '')
+            label = wallet.get_label(address)
             balance = sum(wallet.get_addr_balance(address))
             is_used_and_empty = wallet.is_used(address) and balance == 0
             if self.show_used == 1 and (balance or is_used_and_empty):

electrum/gui/kivy/uix/dialogs/amount_dialog.py

@@ -49,6 +49,7 @@ Builder.load_string('''
                 amount: ''
                 fiat_amount: ''
                 is_fiat: False
+                is_max: False
                 on_fiat_amount: if self.is_fiat: self.amount = app.fiat_to_btc(self.fiat_amount)
                 on_amount: if not self.is_fiat: self.fiat_amount = app.btc_to_fiat(self.amount)
                 size_hint: 1, None
@@ -92,6 +93,7 @@ Builder.load_string('''
                     on_release:
                         kb.is_fiat = False
                         kb.amount = app.get_max_amount()
+                        kb.is_max = True
                 Button:
                     size_hint: 1, None
                     height: '48dp'
@@ -99,6 +101,7 @@ Builder.load_string('''
                     on_release:
                         kb.amount = ''
                         kb.fiat_amount = ''
+                        kb.is_max = False
             Widget:
                 size_hint: 1, 0.2
             BoxLayout:
@@ -112,7 +115,7 @@ Builder.load_string('''
                     height: '48dp'
                     text: _('OK')
                     on_release:
-                        root.callback(btc.text if kb.amount else '')
+                        root.callback('!' if kb.is_max else btc.text if kb.amount else '')
                         popup.dismiss()
 ''')
 

electrum/gui/kivy/uix/dialogs/bump_fee_dialog.py

@@ -1,3 +1,5 @@
+from typing import TYPE_CHECKING
+
 from kivy.app import App
 from kivy.factory import Factory
 from kivy.properties import ObjectProperty
@@ -5,6 +7,10 @@ from kivy.lang import Builder
 
 from electrum.gui.kivy.i18n import _
 
+if TYPE_CHECKING:
+    from ...main_window import ElectrumWindow
+
+
 Builder.load_string('''
 <BumpFeeDialog@Popup>
     title: _('Bump fee')
@@ -68,7 +74,7 @@ Builder.load_string('''
 
 class BumpFeeDialog(Factory.Popup):
 
-    def __init__(self, app, fee, size, callback):
+    def __init__(self, app: 'ElectrumWindow', fee, size, callback):
         Factory.Popup.__init__(self)
         self.app = app
         self.init_fee = fee

electrum/gui/kivy/uix/dialogs/crash_reporter.py

@@ -36,7 +36,7 @@ Builder.load_string('''
             text: 'Show report contents'
             height: '48dp'
             size_hint: 1, None
-            on_press: root.show_contents()
+            on_release: root.show_contents()
         BoxLayout:
             size_hint: 1, 0.1
         Label:
@@ -131,6 +131,9 @@ class CrashReporter(BaseCrashReporter, Factory.Popup):
                 self.open_url(response["location"])
         self.dismiss()
 
+    def on_dismiss(self):
+        self.main_window.on_wizard_aborted()
+
     def open_url(self, url):
         if platform != 'android':
             return

electrum/gui/kivy/uix/dialogs/dscancel_dialog.py

@@ -0,0 +1,111 @@
+from typing import TYPE_CHECKING
+
+from kivy.app import App
+from kivy.factory import Factory
+from kivy.properties import ObjectProperty
+from kivy.lang import Builder
+
+from electrum.gui.kivy.i18n import _
+
+if TYPE_CHECKING:
+    from ...main_window import ElectrumWindow
+
+
+Builder.load_string('''
+<DSCancelDialog@Popup>
+    title: _('Cancel (double-spend) transaction')
+    size_hint: 0.8, 0.8
+    pos_hint: {'top':0.9}
+    BoxLayout:
+        orientation: 'vertical'
+        padding: '10dp'
+
+        GridLayout:
+            height: self.minimum_height
+            size_hint_y: None
+            cols: 1
+            spacing: '10dp'
+            BoxLabel:
+                id: old_fee
+                text: _('Current Fee')
+                value: ''
+            BoxLabel:
+                id: old_feerate
+                text: _('Current Fee rate')
+                value: ''
+        Label:
+            id: tooltip1
+            text: ''
+            size_hint_y: None
+        Label:
+            id: tooltip2
+            text: ''
+            size_hint_y: None
+        Slider:
+            id: slider
+            range: 0, 4
+            step: 1
+            on_value: root.on_slider(self.value)
+        Widget:
+            size_hint: 1, 1
+        BoxLayout:
+            orientation: 'horizontal'
+            size_hint: 1, 0.5
+            Button:
+                text: 'Cancel'
+                size_hint: 0.5, None
+                height: '48dp'
+                on_release: root.dismiss()
+            Button:
+                text: 'OK'
+                size_hint: 0.5, None
+                height: '48dp'
+                on_release:
+                    root.dismiss()
+                    root.on_ok()
+''')
+
+class DSCancelDialog(Factory.Popup):
+
+    def __init__(self, app: 'ElectrumWindow', fee, size, callback):
+        Factory.Popup.__init__(self)
+        self.app = app
+        self.init_fee = fee
+        self.tx_size = size
+        self.callback = callback
+        self.config = app.electrum_config
+        self.mempool = self.config.use_mempool_fees()
+        self.dynfees = self.config.is_dynfee() and bool(self.app.network) and self.config.has_dynamic_fees_ready()
+        self.ids.old_fee.value = self.app.format_amount_and_units(self.init_fee)
+        self.ids.old_feerate.value = self.app.format_fee_rate(fee / self.tx_size * 1000)
+        self.update_slider()
+        self.update_text()
+
+    def update_text(self):
+        pos = int(self.ids.slider.value)
+        new_fee_rate = self.get_fee_rate()
+        text, tooltip = self.config.get_fee_text(pos, self.dynfees, self.mempool, new_fee_rate)
+        self.ids.tooltip1.text = text
+        self.ids.tooltip2.text = tooltip
+
+    def update_slider(self):
+        slider = self.ids.slider
+        maxp, pos, fee_rate = self.config.get_fee_slider(self.dynfees, self.mempool)
+        slider.range = (0, maxp)
+        slider.step = 1
+        slider.value = pos
+
+    def get_fee_rate(self):
+        pos = int(self.ids.slider.value)
+        if self.dynfees:
+            fee_rate = self.config.depth_to_fee(pos) if self.mempool else self.config.eta_to_fee(pos)
+        else:
+            fee_rate = self.config.static_fee(pos)
+        return fee_rate  # sat/kbyte
+
+    def on_ok(self):
+        new_fee_rate = self.get_fee_rate() / 1000
+        self.callback(new_fee_rate)
+
+    def on_slider(self, value):
+        self.update_text()

electrum/gui/kivy/uix/dialogs/fx_dialog.py

@@ -89,7 +89,12 @@ class FxDialog(Factory.Popup):
         self.config = config
         self.callback = callback
         self.fx = self.app.fx
-        self.has_history_rates = self.fx.get_history_config(default=True)
+        if self.fx.get_history_config(allow_none=True) is None:
+            # If nothing is set, force-enable it. (Note that as fiat rates itself
+            # are disabled by default, it is enough to set this here. If they
+            # were enabled by default, this would be too late.)
+            self.fx.set_history_config(True)
+        self.has_history_rates = self.fx.get_history_config()
 
         Factory.Popup.__init__(self)
         self.add_currencies()