From 99129489385f729a29ed557ad623d7f02fd16b0a Mon Sep 17 00:00:00 2001 From: avirgovi Date: Thu, 17 Mar 2022 19:09:48 +0100 Subject: [PATCH 01/29] ubuntu 20 build simulator instructions --- README.md | 37 ++++++++++++++++++++++++++++--------- external/libngu.patch | 26 ++++++++++++++++++++++++++ unix/unix_addr.patch | 18 ++++++++++++++++++ 3 files changed, 72 insertions(+), 9 deletions(-) create mode 100644 external/libngu.patch create mode 100644 unix/unix_addr.patch diff --git a/README.md b/README.md index 87a5d5f2..7d22f760 100644 --- a/README.md +++ b/README.md @@ -87,17 +87,36 @@ You may need to reboot to avoid a `DISPLAY is not set` error. ### Linux -You'll probably need to install these (Ubuntu 16): +You'll need to install these (Ubuntu 20.04): - apt install libudev-dev python-sdl2 gcc-arm-none-eabi + apt install build-essential git python3 python3-pip libudev-dev gcc-arm-none-eabi -If you get stuck on the "Skip PIN" screen after the startup, edit the `pyb.py` file located under `/unix/frozen-modules/` and follow the instructions from line 27 to line 31: -``` -# If on linux, try commenting the following line -addr = bytes([len(fn)+2, socket.AF_UNIX] + list(fn)) -# If on linux, try uncommenting the following two lines -#import struct -#addr = struct.pack('H108s', socket.AF_UNIX, fn) +Install and run simulator on Ubuntu 20.04 +```shell +git clone --recursive https://github.com/Coldcard/firmware.git +cd firmware +# apply address patch +git apply unix/unix_addr.patch +# apply libngu patch +pushd external/libngu +git apply ../libngu.patch +popd +# create virtualenv and activate it +python3 -m venv ENV # or virtualenv -p python3 ENV +source ENV/bin/activate +# install dependencies +pip install -U pip setuptools +pip install -r requirements.txt +# build simulator +cd unix +pushd ../external/micropython/mpy-cross/ +make # mpy-cross +popd +make setup +make ngu-setup +make +# below line runs the simulator +./simulator.py ``` ## Code Organization diff --git a/external/libngu.patch b/external/libngu.patch new file mode 100644 index 00000000..40af15e3 --- /dev/null +++ b/external/libngu.patch @@ -0,0 +1,26 @@ +diff --git a/ngu/hash.c b/ngu/hash.c +index 72a16d5..c8e0653 100644 +--- a/ngu/hash.c ++++ b/ngu/hash.c +@@ -286,7 +286,7 @@ void ripemd160(const uint8_t *msg, int msglen, uint8_t digest[20]) + #error "untested; suspect endian challenge here" + #endif + +- if(((uint32_t)digest) & 0x3) { ++ if(((uintptr_t)digest) & 0x3) { + // unaligned case + uint32_t ctx[5]; + +diff --git a/ngu/random.c b/ngu/random.c +index 4bd3d8d..deef1d2 100644 +--- a/ngu/random.c ++++ b/ngu/random.c +@@ -32,7 +32,7 @@ extern uint32_t rng_get(void); + + #ifdef UNIX + # define CHIP_TRNG_SETUP() +-# define CHIP_TRNG_32() arc4random() ++# define CHIP_TRNG_32() random() + #endif + + #ifndef CHIP_TRNG_SETUP diff --git a/unix/unix_addr.patch b/unix/unix_addr.patch new file mode 100644 index 00000000..d38f3e56 --- /dev/null +++ b/unix/unix_addr.patch @@ -0,0 +1,18 @@ +diff --git a/unix/variant/pyb.py b/unix/variant/pyb.py +index d22bb1b..de5d71a 100644 +--- a/unix/variant/pyb.py ++++ b/unix/variant/pyb.py +@@ -36,10 +36,10 @@ class USB_HID: + import usocket as socket + self.pipe = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + # If on linux, try commenting the following line +- addr = bytes([len(self.fn)+2, socket.AF_UNIX] + list(self.fn)) ++ #addr = bytes([len(self.fn)+2, socket.AF_UNIX] + list(self.fn)) + # If on linux, try uncommenting the following two lines +- #import struct +- #addr = struct.pack('H108s', socket.AF_UNIX, self.fn) ++ import struct ++ addr = struct.pack('H108s', socket.AF_UNIX, self.fn) + while 1: + try: + self.pipe.bind(addr) From a7fdb06d37bdeb19e866ed3a59f77b2bd1cce5f9 Mon Sep 17 00:00:00 2001 From: avirgovi Date: Thu, 17 Mar 2022 21:50:11 +0100 Subject: [PATCH 02/29] Revert "ubuntu 20 build simulator instructions" This reverts commit 99129489385f729a29ed557ad623d7f02fd16b0a. --- README.md | 37 +++++++++---------------------------- external/libngu.patch | 26 -------------------------- unix/unix_addr.patch | 18 ------------------ 3 files changed, 9 insertions(+), 72 deletions(-) delete mode 100644 external/libngu.patch delete mode 100644 unix/unix_addr.patch diff --git a/README.md b/README.md index 7d22f760..87a5d5f2 100644 --- a/README.md +++ b/README.md @@ -87,36 +87,17 @@ You may need to reboot to avoid a `DISPLAY is not set` error. ### Linux -You'll need to install these (Ubuntu 20.04): +You'll probably need to install these (Ubuntu 16): - apt install build-essential git python3 python3-pip libudev-dev gcc-arm-none-eabi + apt install libudev-dev python-sdl2 gcc-arm-none-eabi -Install and run simulator on Ubuntu 20.04 -```shell -git clone --recursive https://github.com/Coldcard/firmware.git -cd firmware -# apply address patch -git apply unix/unix_addr.patch -# apply libngu patch -pushd external/libngu -git apply ../libngu.patch -popd -# create virtualenv and activate it -python3 -m venv ENV # or virtualenv -p python3 ENV -source ENV/bin/activate -# install dependencies -pip install -U pip setuptools -pip install -r requirements.txt -# build simulator -cd unix -pushd ../external/micropython/mpy-cross/ -make # mpy-cross -popd -make setup -make ngu-setup -make -# below line runs the simulator -./simulator.py +If you get stuck on the "Skip PIN" screen after the startup, edit the `pyb.py` file located under `/unix/frozen-modules/` and follow the instructions from line 27 to line 31: +``` +# If on linux, try commenting the following line +addr = bytes([len(fn)+2, socket.AF_UNIX] + list(fn)) +# If on linux, try uncommenting the following two lines +#import struct +#addr = struct.pack('H108s', socket.AF_UNIX, fn) ``` ## Code Organization diff --git a/external/libngu.patch b/external/libngu.patch deleted file mode 100644 index 40af15e3..00000000 --- a/external/libngu.patch +++ /dev/null @@ -1,26 +0,0 @@ -diff --git a/ngu/hash.c b/ngu/hash.c -index 72a16d5..c8e0653 100644 ---- a/ngu/hash.c -+++ b/ngu/hash.c -@@ -286,7 +286,7 @@ void ripemd160(const uint8_t *msg, int msglen, uint8_t digest[20]) - #error "untested; suspect endian challenge here" - #endif - -- if(((uint32_t)digest) & 0x3) { -+ if(((uintptr_t)digest) & 0x3) { - // unaligned case - uint32_t ctx[5]; - -diff --git a/ngu/random.c b/ngu/random.c -index 4bd3d8d..deef1d2 100644 ---- a/ngu/random.c -+++ b/ngu/random.c -@@ -32,7 +32,7 @@ extern uint32_t rng_get(void); - - #ifdef UNIX - # define CHIP_TRNG_SETUP() --# define CHIP_TRNG_32() arc4random() -+# define CHIP_TRNG_32() random() - #endif - - #ifndef CHIP_TRNG_SETUP diff --git a/unix/unix_addr.patch b/unix/unix_addr.patch deleted file mode 100644 index d38f3e56..00000000 --- a/unix/unix_addr.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/unix/variant/pyb.py b/unix/variant/pyb.py -index d22bb1b..de5d71a 100644 ---- a/unix/variant/pyb.py -+++ b/unix/variant/pyb.py -@@ -36,10 +36,10 @@ class USB_HID: - import usocket as socket - self.pipe = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - # If on linux, try commenting the following line -- addr = bytes([len(self.fn)+2, socket.AF_UNIX] + list(self.fn)) -+ #addr = bytes([len(self.fn)+2, socket.AF_UNIX] + list(self.fn)) - # If on linux, try uncommenting the following two lines -- #import struct -- #addr = struct.pack('H108s', socket.AF_UNIX, self.fn) -+ import struct -+ addr = struct.pack('H108s', socket.AF_UNIX, self.fn) - while 1: - try: - self.pipe.bind(addr) From 69843b9f69da4d968a11b219693ea94e22f845ad Mon Sep 17 00:00:00 2001 From: avirgovi Date: Fri, 13 May 2022 14:05:54 +0200 Subject: [PATCH 03/29] initial test fixing --- testing/README.md | 3 +++ testing/conftest.py | 17 ++++++++--------- testing/devtest/backups.py | 2 +- testing/requirements.txt | 3 ++- testing/test_export.py | 2 +- testing/test_hsm.py | 4 ++-- testing/test_rng.py | 2 +- unix/variant/ffilib.py | 2 +- 8 files changed, 19 insertions(+), 16 deletions(-) diff --git a/testing/README.md b/testing/README.md index 1e6d8256..e0820d4e 100644 --- a/testing/README.md +++ b/testing/README.md @@ -3,6 +3,9 @@ None of this code ships on the product itself, but it does get used for testing purposes. +## Dependencies +* 7z + ## Background - pytest is used to track test cases and fixtures, etc diff --git a/testing/conftest.py b/testing/conftest.py index 64ac7c56..415a8960 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -1,6 +1,6 @@ # (c) Copyright 2020 by Coinkite Inc. This file is covered by license found in COPYING-CC. # -import pytest, glob, time, sys, random, re +import pytest, glob, time, sys, random, re, ndef from pprint import pprint from ckcc.protocol import CCProtocolPacker, CCProtoError from helpers import B2A, U2SAT, prandom @@ -418,7 +418,10 @@ def qr_quality_check(): scale=3 rv = Image.new('RGB', (w*scale, ((h*scale)+TH)*count), color=(64,64,64)) y = 0 - fnt = ImageFont.truetype('Courier', size=10) + try: + fnt = ImageFont.truetype('Courier', size=10) + except: + fnt = ImageFont.load_default() dr = ImageDraw.Draw(rv) mw = int((w*scale) / dr.textsize('M', fnt)[0]) @@ -866,7 +869,6 @@ def decode_with_bitcoind(bitcoind): def doit(raw_txn): # verify our understanding of a TXN (and esp its outputs) matches # the same values as what bitcoind generates - try: return bitcoind.decoderawtransaction(B2A(raw_txn)) except ConnectionResetError: @@ -947,7 +949,6 @@ def try_sign_microsd(open_microsd, cap_story, pick_menu_item, goto_home, need_ke # like "try_sign" but use "air gapped" file transfer via microSD def doit(f_or_data, accept=True, finalize=False, accept_ms_import=False, complete=False, encoding='binary', del_after=0): - if f_or_data[0:5] == b'psbt\xff': ip = f_or_data filename = 'memory' @@ -965,7 +966,7 @@ def try_sign_microsd(open_microsd, cap_story, pick_menu_item, goto_home, need_ke pat = microsd_path(psbtname+'*.psbt') for f in glob(pat): assert 'psbt' in f - os.unlink(f) + os.remove(f) if encoding == 'hex': ip = b2a_hex(ip) @@ -986,8 +987,7 @@ def try_sign_microsd(open_microsd, cap_story, pick_menu_item, goto_home, need_ke if 'Choose PSBT file' in story: need_keypress('y') time.sleep(.1) - - pick_menu_item(psbtname+'.psbt') + pick_menu_item(psbtname+'.psbt') time.sleep(.1) @@ -1305,7 +1305,7 @@ def nfc_write(request, only_mk4): @pytest.fixture() def nfc_read_json(nfc_read): def doit(): - import ndef, json + import json got = list(ndef.message_decoder(nfc_read())) assert len(got) == 1 got = got[0] @@ -1317,7 +1317,6 @@ def nfc_read_json(nfc_read): @pytest.fixture() def nfc_read_text(nfc_read): def doit(): - import ndef got = list(ndef.message_decoder(nfc_read())) assert len(got) == 1 got = got[0] diff --git a/testing/devtest/backups.py b/testing/devtest/backups.py index ac85f56b..590e856c 100644 --- a/testing/devtest/backups.py +++ b/testing/devtest/backups.py @@ -98,7 +98,7 @@ async def test_7z(): if had_policy: from hsm import POLICY_FNAME - uos.unlink(POLICY_FNAME) + uos.remove(POLICY_FNAME) assert not hsm.hsm_policy_available() with SFFile(0, ll) as fd: diff --git a/testing/requirements.txt b/testing/requirements.txt index fbe0d4e3..46dbd93c 100644 --- a/testing/requirements.txt +++ b/testing/requirements.txt @@ -13,7 +13,8 @@ zbar-py==1.0.4 # NFC and NDEF handling nfcpy==1.0.3 -ndef==0.2 +# commentt out below dependency 'ndef==0.2' if on debian based OS +#ndef==0.2 # optional, and only helpful if you have a desktop NFC-V capable reader pyscard==2.0.2 diff --git a/testing/test_export.py b/testing/test_export.py index 21515061..58c58111 100644 --- a/testing/test_export.py +++ b/testing/test_export.py @@ -464,7 +464,7 @@ def test_export_public_txt(dev, cap_menu, pick_menu_item, goto_home, cap_story, time.sleep(0.1) title, story = cap_story() - assert 'Saves a text file to' in story + assert 'Saves a text file' in story need_keypress('y') time.sleep(0.1) diff --git a/testing/test_hsm.py b/testing/test_hsm.py index b7b84e61..869c591d 100644 --- a/testing/test_hsm.py +++ b/testing/test_hsm.py @@ -57,7 +57,7 @@ def hsm_reset(dev, sim_exec): # make sure we can setup an HSM now; often need to restart simulator tho # clear defined config - cmd = 'import uos, hsm; uos.unlink(hsm.POLICY_FNAME)' + cmd = 'import uos, hsm; uos.remove(hsm.POLICY_FNAME)' sim_exec(cmd) # reset HSM code, to clear previous HSM setup @@ -74,7 +74,7 @@ def hsm_reset(dev, sim_exec): yield doit try: - cmd = 'import uos, hsm; uos.unlink(hsm.POLICY_FNAME)' + cmd = 'import uos, hsm; uos.remove(hsm.POLICY_FNAME)' sim_exec(cmd) except: pass diff --git a/testing/test_rng.py b/testing/test_rng.py index 1a602302..aa2f20a9 100644 --- a/testing/test_rng.py +++ b/testing/test_rng.py @@ -5,7 +5,7 @@ # - needs "dieharder" installed, see # # - on mac: "brew install dieharder" -# +# - on ubuntu20: "sudo apt-get install dieharder" import pytest, subprocess, os from helpers import B2A diff --git a/unix/variant/ffilib.py b/unix/variant/ffilib.py index dc4d672a..ab6837d4 100644 --- a/unix/variant/ffilib.py +++ b/unix/variant/ffilib.py @@ -14,7 +14,7 @@ def open(name, maxver=10, extra=()): except KeyError: pass def libs(): - if sys.platform == "linux": + if sys.platform in ["linux", "linux2", "coldcard-unix"]: yield '%s.so' % name for i in range(maxver, -1, -1): yield '%s.so.%u' % (name, i) From 69aed1d12bd486534b32ddbf7921522a2be4878c Mon Sep 17 00:00:00 2001 From: avirgovi Date: Fri, 13 May 2022 16:49:02 +0200 Subject: [PATCH 04/29] fix HexWriter offset --- shared/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/utils.py b/shared/utils.py index 40b160cd..82cf6764 100644 --- a/shared/utils.py +++ b/shared/utils.py @@ -102,7 +102,7 @@ class HexWriter: return self def __exit__(self, *a, **k): - self.fd.seek(0, 3) # go to end + self.fd.seek(0, 2) # go to end self.fd.write(b'\r\n') return self.fd.__exit__(*a, **k) From 1d354f61b5a9c91175bb183cc56a395173014857 Mon Sep 17 00:00:00 2001 From: avirgovi Date: Sun, 15 May 2022 10:35:44 +0200 Subject: [PATCH 05/29] vdisk fixes --- testing/test_vdisk.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/testing/test_vdisk.py b/testing/test_vdisk.py index f710a4d3..8c5f8c6b 100644 --- a/testing/test_vdisk.py +++ b/testing/test_vdisk.py @@ -37,11 +37,11 @@ def virtdisk_wipe(dev, only_mk4, virtdisk_path): for fn in glob.glob(virtdisk_path('*')): if os.path.isdir(fn): continue if 'readme' in fn.lower(): continue - if fn[0] == '.': continue + if 'gitignore' in fn: continue assert fn.lower().rsplit('.', 1)[1] in { 'txt', 'psbt', 'txn' } print(f'RM {fn}') - os.unlink(fn) + os.remove(fn) return doit def test_vd_basics(dev, virtdisk_path, is_simulator): @@ -62,7 +62,7 @@ def test_vd_basics(dev, virtdisk_path, is_simulator): assert os.path.isfile(virtdisk_path(f'ident/ckcc-{sn}.txt')) @pytest.fixture -def try_sign_virtdisk(need_keypress, virtdisk_path, virtdisk_wipe): +def try_sign_virtdisk(need_keypress, virtdisk_path, cap_story, virtdisk_wipe): # like "try_sign" but use Virtual Disk to send/receive PSBT/results # - on real dev, need user to manually say yes ... alot @@ -115,6 +115,9 @@ def try_sign_virtdisk(need_keypress, virtdisk_path, virtdisk_wipe): return ip, None, None # wait for it to finish signing + title, story = cap_story() + if "OK TO SEND" in title or "PSBT Signed" in title: + need_keypress('y') result_fn = xfn.replace('.psbt', '-*.psbt') result_txn = xfn.replace('.psbt', '.txn') @@ -125,7 +128,8 @@ def try_sign_virtdisk(need_keypress, virtdisk_path, virtdisk_wipe): for i in range(15): try: got_txn = open(result_txn, 'rb').read() - except FileNotFoundError: + except FileNotFoundError as e: + print(e) pass lst = glob.glob(result_fn) From 73d6516d674a1b5ec493a3c6d5a8675a9200a5b6 Mon Sep 17 00:00:00 2001 From: avirgovi Date: Mon, 16 May 2022 16:27:18 +0200 Subject: [PATCH 06/29] fix test_sign_msg_microsd_fails --- testing/test_msg.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/testing/test_msg.py b/testing/test_msg.py index 655f62ce..c8879786 100644 --- a/testing/test_msg.py +++ b/testing/test_msg.py @@ -89,7 +89,6 @@ def sign_on_microsd(open_microsd, cap_story, pick_menu_item, goto_home, need_key # sign a file on the microSD card def doit(msg, subpath=None, expect_fail=False): - fname = 't-msgsign.txt' result_fname = 't-msgsign-signed.txt' @@ -213,13 +212,13 @@ def test_sign_msg_microsd_fails(dev, sign_on_microsd, msg, concern, no_file, tra dev.send_recv(CCProtocolPacker.sign_message(msg.encode('ascii'), path), timeout=None) story = ee.value.args[0] else: - story = sign_on_microsd(msg, path, expect_fail=True) - - if no_file: - assert story == 'NO-FILE' - return - assert story.startswith('Problem: ') - + try: + story = sign_on_microsd(msg, path, expect_fail=True) + assert story.startswith('Problem: ') + except AssertionError as e: + if no_file: + assert "No suitable files found" in str(e) + return assert concern in story @pytest.mark.parametrize('msg,num_iter,expect', [ From 1e71cfdd65560b8ed7c2accdca47e6e37e7c9330 Mon Sep 17 00:00:00 2001 From: avirgovi Date: Tue, 17 May 2022 10:15:51 +0200 Subject: [PATCH 07/29] simulator use constant; seed xor remove semicolons and unused imports --- testing/test_seed_xor.py | 27 ++++++++++++--------------- unix/simulator.py | 4 ++-- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/testing/test_seed_xor.py b/testing/test_seed_xor.py index d20659bb..209d8f81 100644 --- a/testing/test_seed_xor.py +++ b/testing/test_seed_xor.py @@ -2,18 +2,15 @@ # # test Seed XOR features # -import pytest, time, os, re, pdb -from binascii import a2b_hex, b2a_hex -from helpers import B2A -from pycoin.key.BIP32Node import BIP32Node -from pycoin.key.Key import Key +import time +import pytest from mnemonic import Mnemonic from test_ux import word_menu_entry, pass_word_quiz wordlist = Mnemonic('english').wordlist -zero32 = ' '.join('abandon' for i in range(23)) + ' art' -ones32 = ' '.join('zoo' for i in range(23)) + ' vote' +zero32 = ' '.join('abandon' for _ in range(23)) + ' art' +ones32 = ' '.join('zoo' for _ in range(23)) + ' vote' @pytest.mark.parametrize('incl_self', [False, True]) @pytest.mark.parametrize('parts, expect', [ @@ -38,11 +35,11 @@ def test_import_xor(incl_self, parts, expect, goto_home, pick_menu_item, cap_sto pick_menu_item('Seed Functions') pick_menu_item('Seed XOR') pick_menu_item('Restore Seed XOR') - time.sleep(.01); + time.sleep(.01) title, body = cap_story() assert 'all the parts' in body - need_keypress('y'); + need_keypress('y') time.sleep(0.01) title, body = cap_story() @@ -94,7 +91,7 @@ def test_xor_split(qty, trng, goto_home, pick_menu_item, cap_story, need_keypres pick_menu_item('Seed Functions') pick_menu_item('Seed XOR') pick_menu_item('Split Existing') - time.sleep(.01); + time.sleep(.01) title, body = cap_story() assert 'Seed XOR Split' in body @@ -103,13 +100,13 @@ def test_xor_split(qty, trng, goto_home, pick_menu_item, cap_story, need_keypres assert str(qty) in body need_keypress(str(qty)) - time.sleep(.01); + time.sleep(.01) title, body = cap_story() assert f"Split Into {qty} Parts" in body assert f"{qty*24} words" in body need_keypress('2' if trng else 'y') - time.sleep(.01); + time.sleep(.01) title, body = cap_story() assert f'Record these {qty} lists of 24-words' in body @@ -157,11 +154,11 @@ def test_import_zero_set(goto_home, pick_menu_item, cap_story, need_keypress, ca pick_menu_item('Seed Functions') pick_menu_item('Seed XOR') pick_menu_item('Restore Seed XOR') - time.sleep(.01); + time.sleep(.01) title, body = cap_story() assert 'all the parts' in body - need_keypress('y'); + need_keypress('y') time.sleep(0.01) title, body = cap_story() @@ -206,7 +203,7 @@ def test_xor_import_empty(parts, expect, goto_home, pick_menu_item, cap_story, n time.sleep(0.01) title, body = cap_story() assert 'all the parts' in body - need_keypress('y'); + need_keypress('y') time.sleep(0.01) for n, part in enumerate(parts): diff --git a/unix/simulator.py b/unix/simulator.py index ca9171f3..108ec4a6 100755 --- a/unix/simulator.py +++ b/unix/simulator.py @@ -337,9 +337,9 @@ def start(): # manage unix socket cleanup for client def sock_cleanup(): import os - fp = '/tmp/ckcc-simulator.sock' + fp = UNIX_SOCKET_PATH if os.path.exists(fp): - os.unlink(fp) + os.remove(fp) sock_cleanup() import atexit atexit.register(sock_cleanup) From 6e3f6ba6869d2250f36c8d62795aee5244fd2293 Mon Sep 17 00:00:00 2001 From: avirgovi Date: Tue, 17 May 2022 10:21:08 +0200 Subject: [PATCH 08/29] change order of tests and remove unused imports in test_nfc.py --- testing/test_nfc.py | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/testing/test_nfc.py b/testing/test_nfc.py index 7ab9776c..b804ed4b 100644 --- a/testing/test_nfc.py +++ b/testing/test_nfc.py @@ -5,9 +5,7 @@ # - many test "sync" issues here; case is right but gets outs of sync with DUT # - use `./simulator.py --eff --set nfc=1` # -import pytest, glob, pdb -from helpers import B2A -from binascii import b2a_hex, a2b_hex +import pytest from struct import pack, unpack import ndef from hashlib import sha256 @@ -145,7 +143,6 @@ def test_ndef_ccfile(ccfile, load_shared_mod): assert data == txt_msg.encode('utf-8') - @pytest.fixture def try_sign_nfc(cap_story, pick_menu_item, goto_home, need_keypress, sim_exec, nfc_read, nfc_write, nfc_block4rf): @@ -321,25 +318,6 @@ def try_sign_nfc(cap_story, pick_menu_item, goto_home, need_keypress, sim_exec, # cleanup / restore sim_exec('from pyb import SDCard; SDCard.ejected = False') - -@pytest.mark.unfinalized # iff partial=1 -@pytest.mark.parametrize('encoding', ['binary', 'hex', 'base64']) -@pytest.mark.parametrize('num_outs', [1,2]) -@pytest.mark.parametrize('partial', [1, 0]) -def test_nfc_signing(encoding, num_outs, partial, try_sign_nfc, fake_txn, dev, settings_set): - xp = dev.master_xpub - - def hack(psbt): - if partial: - # change first input to not be ours - pk = list(psbt.inputs[0].bip32_paths.keys())[0] - pp = psbt.inputs[0].bip32_paths[pk] - psbt.inputs[0].bip32_paths[pk] = b'what' + pp[4:] - - psbt = fake_txn(2, num_outs, xp, segwit_in=True, psbt_hacker=hack) - - _, txn, txid = try_sign_nfc(psbt, expect_finalize=not partial, encoding=encoding) - @pytest.mark.parametrize('num_outs', [ 1, 20, 250]) def test_nfc_after(num_outs, fake_txn, try_sign, nfc_read, need_keypress, cap_story, only_mk4): # Read signing result (transaction) over NFC, decode it. @@ -380,6 +358,24 @@ def test_nfc_after(num_outs, fake_txn, try_sign, nfc_read, need_keypress, cap_st else: raise ValueError(got.type) +@pytest.mark.unfinalized # iff partial=1 +@pytest.mark.parametrize('encoding', ['binary', 'hex', 'base64']) +@pytest.mark.parametrize('num_outs', [1,2]) +@pytest.mark.parametrize('partial', [1, 0]) +def test_nfc_signing(encoding, num_outs, partial, try_sign_nfc, fake_txn, dev, settings_set): + xp = dev.master_xpub + + def hack(psbt): + if partial: + # change first input to not be ours + pk = list(psbt.inputs[0].bip32_paths.keys())[0] + pp = psbt.inputs[0].bip32_paths[pk] + psbt.inputs[0].bip32_paths[pk] = b'what' + pp[4:] + + psbt = fake_txn(2, num_outs, xp, segwit_in=True, psbt_hacker=hack) + + _, txn, txid = try_sign_nfc(psbt, expect_finalize=not partial, encoding=encoding) + def test_rf_uid(rf_interface, cap_story, goto_home, pick_menu_item): # read UID of NFC chip over the air sw, ident = rf_interface.apdu(0xff, 0xca) # PAPDU_GET_UID From 485cb4c8634a24faf92d66b10fd0523f65bc57c9 Mon Sep 17 00:00:00 2001 From: avirgovi Date: Tue, 17 May 2022 18:06:46 +0200 Subject: [PATCH 09/29] add setting.nfc to optional --- testing/devtest/backups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/devtest/backups.py b/testing/devtest/backups.py index 590e856c..5d93f462 100644 --- a/testing/devtest/backups.py +++ b/testing/devtest/backups.py @@ -20,7 +20,7 @@ if 1: blanks = 0 checklist = set('mnemonic chain xprv xpub raw_secret fw_date fw_version fw_timestamp serial ' 'setting.terms_ok setting.idle_to setting.chain'.split(' ')) - optional = set('setting.pms setting.axi setting.nick setting.lgto setting.usr hsm_policy setting.words long_secret multisig setting.multisig setting.fee_limit setting.tp setting.check duress_xprv duress_xpub duress_1001_words duress_1002_words duress_1003_words'.split(' ')) + optional = set('setting.nfc setting.pms setting.axi setting.nick setting.lgto setting.usr hsm_policy setting.words long_secret multisig setting.multisig setting.fee_limit setting.tp setting.check duress_xprv duress_xpub duress_1001_words duress_1002_words duress_1003_words'.split(' ')) for ln in render_backup_contents().split('\n'): ln = ln.strip() From e9b8d97b00011129bdf3ea9a0d46d1dda5921cb5 Mon Sep 17 00:00:00 2001 From: avirgovi Date: Tue, 17 May 2022 18:10:20 +0200 Subject: [PATCH 10/29] fix test_iss6743 after addition of signature grinding --- testing/test_multisig.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/testing/test_multisig.py b/testing/test_multisig.py index 9f25da98..20734faa 100644 --- a/testing/test_multisig.py +++ b/testing/test_multisig.py @@ -1611,7 +1611,6 @@ def test_bitcoind_cosigning(dev, bitcoind, import_ms_wallet, clear_ms, explora, @pytest.mark.parametrize('bitrot', list(range(0,6)) + [98, 99, 100] + list(range(-5, 0))) @pytest.mark.ms_danger def test_ms_sign_bitrot(num_ins, dev, addr_fmt, clear_ms, incl_xpubs, import_ms_wallet, addr_vs_path, fake_ms_txn, start_sign, end_sign, out_style, cap_story, bitrot, has_ms_checks): - M = 1 N = 3 num_outs = 2 @@ -1716,15 +1715,12 @@ def test_ms_change_fraud(case, pk_num, num_ins, dev, addr_fmt, clear_ms, incl_xp @pytest.mark.parametrize('repeat', range(2) ) def test_iss6743(repeat, set_seed_words, sim_execfile, try_sign): # from SomberNight - psbt_b4 = bytes.fromhex('''\ -70736274ff0100520200000001bde05be36069e2e0fe44793c68ad8244bb1a52cc37f152e0fa5b75e40169d7f70000000000fdffffff018b1e000000000000160014ed5180f05c7b1dc980732602c50cda40530e00ad4de11c004f01024289ef0000000000000000007dd565da7ee1cf05c516e89a608968fed4a2450633a00c7b922df66b27afd2e1033a0a4fa4b0a997738ac2f142a395c1f02afcb31d7ffd46a90a0c927a4c411fd704094ef7844f01024289ef0431fcbdcc8000000112d4aaea7292e7870c7eeb3565fa1c1fa8f957fa7c4c24b411d5b4f5710d359a023e63d1e54063525bea286ccb2a0ad7b14560aa31ec4be826afa883141dfe1d53145c9e228d300000800100008000000080010000804f01024289ef04e44b38f1800000014a1960f3a3c86ba355a16a66a548cfb62eeb25663311f7cd662a192896f3777e038cc595159a395e4ec35e477c9523a1512f873e74d303fb03fc9a1503b1ba45271434652fae30000080010000800000008001000080000100df02000000000101d1a321707660769c7f8604d04c9ae2db58cf1ec7a01f4f285cdcbb25ce14bdfd0300000000fdffffff02401f00000000000017a914bfd0b8471a3706c1e17870a4d39b0354bcea57b687c864000000000000160014e608b171d63ec24d9fa252d5c1e45624b14e44700247304402205133eb96df167b895f657cce31c6882840a403013682d9d4651aed2730a7dad502202aaacc045d85d9c711af0c84e7f355cc18bf2f8e6d91774d42ba24de8418a39e012103a58d8eb325abb412eaf927cf11d8b7641c4a468ce412057e47892ca2d13ed6144de11c000104220020a3c65c4e376d82fb3ca45596feee5b08313ad64f38590c1b08bc530d1c0bbfea010569522102ab84641359fa22461b8461515231da63c196614cd22b26e556ed878e30db4da621034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef6381921039690cf74941da5db291fa8be7348abe3807786732d969eac5d27e0afa909a55f53ae220602ab84641359fa22461b8461515231da63c196614cd22b26e556ed878e30db4da60c094ef78400000000030000002206034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef638191c5c9e228d3000008001000080000000800100008000000000030000002206039690cf74941da5db291fa8be7348abe3807786732d969eac5d27e0afa909a55f1c34652fae3000008001000080000000800100008000000000030000000000''') - + psbt_b4 = bytes.fromhex('70736274ff0100520200000001bde05be36069e2e0fe44793c68ad8244bb1a52cc37f152e0fa5b75e40169d7f70000000000fdffffff018b1e000000000000160014ed5180f05c7b1dc980732602c50cda40530e00ad4de11c004f01024289ef0000000000000000007dd565da7ee1cf05c516e89a608968fed4a2450633a00c7b922df66b27afd2e1033a0a4fa4b0a997738ac2f142a395c1f02afcb31d7ffd46a90a0c927a4c411fd704094ef7844f01024289ef0431fcbdcc8000000112d4aaea7292e7870c7eeb3565fa1c1fa8f957fa7c4c24b411d5b4f5710d359a023e63d1e54063525bea286ccb2a0ad7b14560aa31ec4be826afa883141dfe1d53145c9e228d300000800100008000000080010000804f01024289ef04e44b38f1800000014a1960f3a3c86ba355a16a66a548cfb62eeb25663311f7cd662a192896f3777e038cc595159a395e4ec35e477c9523a1512f873e74d303fb03fc9a1503b1ba45271434652fae30000080010000800000008001000080000100df02000000000101d1a321707660769c7f8604d04c9ae2db58cf1ec7a01f4f285cdcbb25ce14bdfd0300000000fdffffff02401f00000000000017a914bfd0b8471a3706c1e17870a4d39b0354bcea57b687c864000000000000160014e608b171d63ec24d9fa252d5c1e45624b14e44700247304402205133eb96df167b895f657cce31c6882840a403013682d9d4651aed2730a7dad502202aaacc045d85d9c711af0c84e7f355cc18bf2f8e6d91774d42ba24de8418a39e012103a58d8eb325abb412eaf927cf11d8b7641c4a468ce412057e47892ca2d13ed6144de11c000104220020a3c65c4e376d82fb3ca45596feee5b08313ad64f38590c1b08bc530d1c0bbfea010569522102ab84641359fa22461b8461515231da63c196614cd22b26e556ed878e30db4da621034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef6381921039690cf74941da5db291fa8be7348abe3807786732d969eac5d27e0afa909a55f53ae220602ab84641359fa22461b8461515231da63c196614cd22b26e556ed878e30db4da60c094ef78400000000030000002206034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef638191c5c9e228d3000008001000080000000800100008000000000030000002206039690cf74941da5db291fa8be7348abe3807786732d969eac5d27e0afa909a55f1c34652fae3000008001000080000000800100008000000000030000000000') # pre 3.2.0 result - psbt_wrong = bytes.fromhex('''\ -70736274ff0100520200000001bde05be36069e2e0fe44793c68ad8244bb1a52cc37f152e0fa5b75e40169d7f70000000000fdffffff018b1e000000000000160014ed5180f05c7b1dc980732602c50cda40530e00ad4de11c004f01024289ef0000000000000000007dd565da7ee1cf05c516e89a608968fed4a2450633a00c7b922df66b27afd2e1033a0a4fa4b0a997738ac2f142a395c1f02afcb31d7ffd46a90a0c927a4c411fd704094ef7844f01024289ef0431fcbdcc8000000112d4aaea7292e7870c7eeb3565fa1c1fa8f957fa7c4c24b411d5b4f5710d359a023e63d1e54063525bea286ccb2a0ad7b14560aa31ec4be826afa883141dfe1d53145c9e228d300000800100008000000080010000804f01024289ef04e44b38f1800000014a1960f3a3c86ba355a16a66a548cfb62eeb25663311f7cd662a192896f3777e038cc595159a395e4ec35e477c9523a1512f873e74d303fb03fc9a1503b1ba45271434652fae30000080010000800000008001000080000100df02000000000101d1a321707660769c7f8604d04c9ae2db58cf1ec7a01f4f285cdcbb25ce14bdfd0300000000fdffffff02401f00000000000017a914bfd0b8471a3706c1e17870a4d39b0354bcea57b687c864000000000000160014e608b171d63ec24d9fa252d5c1e45624b14e44700247304402205133eb96df167b895f657cce31c6882840a403013682d9d4651aed2730a7dad502202aaacc045d85d9c711af0c84e7f355cc18bf2f8e6d91774d42ba24de8418a39e012103a58d8eb325abb412eaf927cf11d8b7641c4a468ce412057e47892ca2d13ed6144de11c002202034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef63819483045022100a85d08eef6675803fe2b58dda11a553641080e07da36a2f3e116f1224201931b022071b0ba83ef920d49b520c37993c039d13ae508a1adbd47eb4b329713fcc8baef01010304010000002206034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef638191c5c9e228d3000008001000080000000800100008000000000030000002206039690cf74941da5db291fa8be7348abe3807786732d969eac5d27e0afa909a55f1c34652fae300000800100008000000080010000800000000003000000220602ab84641359fa22461b8461515231da63c196614cd22b26e556ed878e30db4da60c094ef78400000000030000000104220020a3c65c4e376d82fb3ca45596feee5b08313ad64f38590c1b08bc530d1c0bbfea010569522102ab84641359fa22461b8461515231da63c196614cd22b26e556ed878e30db4da621034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef6381921039690cf74941da5db291fa8be7348abe3807786732d969eac5d27e0afa909a55f53ae0000''') - psbt_right = bytes.fromhex('''\ -70736274ff0100520200000001bde05be36069e2e0fe44793c68ad8244bb1a52cc37f152e0fa5b75e40169d7f70000000000fdffffff018b1e000000000000160014ed5180f05c7b1dc980732602c50cda40530e00ad4de11c004f01024289ef0000000000000000007dd565da7ee1cf05c516e89a608968fed4a2450633a00c7b922df66b27afd2e1033a0a4fa4b0a997738ac2f142a395c1f02afcb31d7ffd46a90a0c927a4c411fd704094ef7844f01024289ef0431fcbdcc8000000112d4aaea7292e7870c7eeb3565fa1c1fa8f957fa7c4c24b411d5b4f5710d359a023e63d1e54063525bea286ccb2a0ad7b14560aa31ec4be826afa883141dfe1d53145c9e228d300000800100008000000080010000804f01024289ef04e44b38f1800000014a1960f3a3c86ba355a16a66a548cfb62eeb25663311f7cd662a192896f3777e038cc595159a395e4ec35e477c9523a1512f873e74d303fb03fc9a1503b1ba45271434652fae30000080010000800000008001000080000100df02000000000101d1a321707660769c7f8604d04c9ae2db58cf1ec7a01f4f285cdcbb25ce14bdfd0300000000fdffffff02401f00000000000017a914bfd0b8471a3706c1e17870a4d39b0354bcea57b687c864000000000000160014e608b171d63ec24d9fa252d5c1e45624b14e44700247304402205133eb96df167b895f657cce31c6882840a403013682d9d4651aed2730a7dad502202aaacc045d85d9c711af0c84e7f355cc18bf2f8e6d91774d42ba24de8418a39e012103a58d8eb325abb412eaf927cf11d8b7641c4a468ce412057e47892ca2d13ed6144de11c002202034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef63819483045022100ae90a7e4c350389816b03af0af46df59a2f53da04cc95a2abd81c0bbc5950c1d02202f9471d6b0664b7a46e81da62d149f688adc7ba2b3413372d26fa618a8460eba01010304010000002206034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef638191c5c9e228d3000008001000080000000800100008000000000030000002206039690cf74941da5db291fa8be7348abe3807786732d969eac5d27e0afa909a55f1c34652fae300000800100008000000080010000800000000003000000220602ab84641359fa22461b8461515231da63c196614cd22b26e556ed878e30db4da60c094ef78400000000030000000104220020a3c65c4e376d82fb3ca45596feee5b08313ad64f38590c1b08bc530d1c0bbfea010569522102ab84641359fa22461b8461515231da63c196614cd22b26e556ed878e30db4da621034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef6381921039690cf74941da5db291fa8be7348abe3807786732d969eac5d27e0afa909a55f53ae0000''') - + psbt_wrong = bytes.fromhex('70736274ff0100520200000001bde05be36069e2e0fe44793c68ad8244bb1a52cc37f152e0fa5b75e40169d7f70000000000fdffffff018b1e000000000000160014ed5180f05c7b1dc980732602c50cda40530e00ad4de11c004f01024289ef0000000000000000007dd565da7ee1cf05c516e89a608968fed4a2450633a00c7b922df66b27afd2e1033a0a4fa4b0a997738ac2f142a395c1f02afcb31d7ffd46a90a0c927a4c411fd704094ef7844f01024289ef0431fcbdcc8000000112d4aaea7292e7870c7eeb3565fa1c1fa8f957fa7c4c24b411d5b4f5710d359a023e63d1e54063525bea286ccb2a0ad7b14560aa31ec4be826afa883141dfe1d53145c9e228d300000800100008000000080010000804f01024289ef04e44b38f1800000014a1960f3a3c86ba355a16a66a548cfb62eeb25663311f7cd662a192896f3777e038cc595159a395e4ec35e477c9523a1512f873e74d303fb03fc9a1503b1ba45271434652fae30000080010000800000008001000080000100df02000000000101d1a321707660769c7f8604d04c9ae2db58cf1ec7a01f4f285cdcbb25ce14bdfd0300000000fdffffff02401f00000000000017a914bfd0b8471a3706c1e17870a4d39b0354bcea57b687c864000000000000160014e608b171d63ec24d9fa252d5c1e45624b14e44700247304402205133eb96df167b895f657cce31c6882840a403013682d9d4651aed2730a7dad502202aaacc045d85d9c711af0c84e7f355cc18bf2f8e6d91774d42ba24de8418a39e012103a58d8eb325abb412eaf927cf11d8b7641c4a468ce412057e47892ca2d13ed6144de11c002202034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef63819483045022100a85d08eef6675803fe2b58dda11a553641080e07da36a2f3e116f1224201931b022071b0ba83ef920d49b520c37993c039d13ae508a1adbd47eb4b329713fcc8baef01010304010000002206034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef638191c5c9e228d3000008001000080000000800100008000000000030000002206039690cf74941da5db291fa8be7348abe3807786732d969eac5d27e0afa909a55f1c34652fae300000800100008000000080010000800000000003000000220602ab84641359fa22461b8461515231da63c196614cd22b26e556ed878e30db4da60c094ef78400000000030000000104220020a3c65c4e376d82fb3ca45596feee5b08313ad64f38590c1b08bc530d1c0bbfea010569522102ab84641359fa22461b8461515231da63c196614cd22b26e556ed878e30db4da621034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef6381921039690cf74941da5db291fa8be7348abe3807786732d969eac5d27e0afa909a55f53ae0000') + # psbt_right = bytes.fromhex('70736274ff0100520200000001bde05be36069e2e0fe44793c68ad8244bb1a52cc37f152e0fa5b75e40169d7f70000000000fdffffff018b1e000000000000160014ed5180f05c7b1dc980732602c50cda40530e00ad4de11c004f01024289ef0000000000000000007dd565da7ee1cf05c516e89a608968fed4a2450633a00c7b922df66b27afd2e1033a0a4fa4b0a997738ac2f142a395c1f02afcb31d7ffd46a90a0c927a4c411fd704094ef7844f01024289ef0431fcbdcc8000000112d4aaea7292e7870c7eeb3565fa1c1fa8f957fa7c4c24b411d5b4f5710d359a023e63d1e54063525bea286ccb2a0ad7b14560aa31ec4be826afa883141dfe1d53145c9e228d300000800100008000000080010000804f01024289ef04e44b38f1800000014a1960f3a3c86ba355a16a66a548cfb62eeb25663311f7cd662a192896f3777e038cc595159a395e4ec35e477c9523a1512f873e74d303fb03fc9a1503b1ba45271434652fae30000080010000800000008001000080000100df02000000000101d1a321707660769c7f8604d04c9ae2db58cf1ec7a01f4f285cdcbb25ce14bdfd0300000000fdffffff02401f00000000000017a914bfd0b8471a3706c1e17870a4d39b0354bcea57b687c864000000000000160014e608b171d63ec24d9fa252d5c1e45624b14e44700247304402205133eb96df167b895f657cce31c6882840a403013682d9d4651aed2730a7dad502202aaacc045d85d9c711af0c84e7f355cc18bf2f8e6d91774d42ba24de8418a39e012103a58d8eb325abb412eaf927cf11d8b7641c4a468ce412057e47892ca2d13ed6144de11c002202034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef63819483045022100ae90a7e4c350389816b03af0af46df59a2f53da04cc95a2abd81c0bbc5950c1d02202f9471d6b0664b7a46e81da62d149f688adc7ba2b3413372d26fa618a8460eba01010304010000002206034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef638191c5c9e228d3000008001000080000000800100008000000000030000002206039690cf74941da5db291fa8be7348abe3807786732d969eac5d27e0afa909a55f1c34652fae300000800100008000000080010000800000000003000000220602ab84641359fa22461b8461515231da63c196614cd22b26e556ed878e30db4da60c094ef78400000000030000000104220020a3c65c4e376d82fb3ca45596feee5b08313ad64f38590c1b08bc530d1c0bbfea010569522102ab84641359fa22461b8461515231da63c196614cd22b26e556ed878e30db4da621034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef6381921039690cf74941da5db291fa8be7348abe3807786732d969eac5d27e0afa909a55f53ae0000') + # changed with with introduction of signature grinding + psbt_right = bytes.fromhex('70736274ff0100520200000001bde05be36069e2e0fe44793c68ad8244bb1a52cc37f152e0fa5b75e40169d7f70000000000fdffffff018b1e000000000000160014ed5180f05c7b1dc980732602c50cda40530e00ad4de11c004f01024289ef0000000000000000007dd565da7ee1cf05c516e89a608968fed4a2450633a00c7b922df66b27afd2e1033a0a4fa4b0a997738ac2f142a395c1f02afcb31d7ffd46a90a0c927a4c411fd704094ef7844f01024289ef0431fcbdcc8000000112d4aaea7292e7870c7eeb3565fa1c1fa8f957fa7c4c24b411d5b4f5710d359a023e63d1e54063525bea286ccb2a0ad7b14560aa31ec4be826afa883141dfe1d53145c9e228d300000800100008000000080010000804f01024289ef04e44b38f1800000014a1960f3a3c86ba355a16a66a548cfb62eeb25663311f7cd662a192896f3777e038cc595159a395e4ec35e477c9523a1512f873e74d303fb03fc9a1503b1ba45271434652fae30000080010000800000008001000080000100df02000000000101d1a321707660769c7f8604d04c9ae2db58cf1ec7a01f4f285cdcbb25ce14bdfd0300000000fdffffff02401f00000000000017a914bfd0b8471a3706c1e17870a4d39b0354bcea57b687c864000000000000160014e608b171d63ec24d9fa252d5c1e45624b14e44700247304402205133eb96df167b895f657cce31c6882840a403013682d9d4651aed2730a7dad502202aaacc045d85d9c711af0c84e7f355cc18bf2f8e6d91774d42ba24de8418a39e012103a58d8eb325abb412eaf927cf11d8b7641c4a468ce412057e47892ca2d13ed6144de11c002202034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef6381947304402201008b084f53d3064ee381dfb3ff4373b29d6ae765b2af15a4e217e8d5d049c650220576af95d79b8fc686627da8a534141208b225ceb6085cd93fcaffb153ac016ea01010304010000002206034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef638191c5c9e228d3000008001000080000000800100008000000000030000002206039690cf74941da5db291fa8be7348abe3807786732d969eac5d27e0afa909a55f1c34652fae300000800100008000000080010000800000000003000000220602ab84641359fa22461b8461515231da63c196614cd22b26e556ed878e30db4da60c094ef78400000000030000000104220020a3c65c4e376d82fb3ca45596feee5b08313ad64f38590c1b08bc530d1c0bbfea010569522102ab84641359fa22461b8461515231da63c196614cd22b26e556ed878e30db4da621034211ab0f75c3a307a2f6bf6f09a9e05d3c8edd0ba7a2ac31f432d1045ef6381921039690cf74941da5db291fa8be7348abe3807786732d969eac5d27e0afa909a55f53ae0000') seed_words = 'all all all all all all all all all all all all' expect_xfp = swab32(int('5c9e228d', 16)) assert xfp2str(expect_xfp) == '5c9e228d'.upper() From ddc1a5475963ca9fe2e921811631fe2b2c01455d Mon Sep 17 00:00:00 2001 From: avirgovi Date: Tue, 17 May 2022 18:52:15 +0200 Subject: [PATCH 11/29] split test_unit.py::test_backups to two tests --- testing/test_unit.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/testing/test_unit.py b/testing/test_unit.py index f7af64ea..a4758dfe 100644 --- a/testing/test_unit.py +++ b/testing/test_unit.py @@ -94,18 +94,23 @@ def test_nvram_mk4(unit_test, only_mk4): # exercise nvram simulation: only mk4 unit_test('devtest/nvram_mk4.py') -@pytest.mark.parametrize('mode', ['simple', 'blankish']) -def test_backups(unit_test, mode, set_seed_words): +@pytest.mark.manual +def test_backups_simple(unit_test, set_seed_words): + # exercise dump of pub data + # - (bug) mk4 can only run this test in isolation from other test in this file. + unit_test('devtest/backups.py') + +@pytest.mark.manual +def test_backups_blankish(unit_test, set_seed_words): # exercise dump of pub data # - (bug) mk4 can only run this test in isolation from other test in this file. - if mode == 'blankish': - # want a zero in last byte of hex representation of raw secret... - ''' + # want a zero in last byte of hex representation of raw secret... + ''' >>> tcc.bip39.from_data(bytes([0x10]*32)) 'avoid letter advice cage ... absurd amount doctor blanket' - ''' - set_seed_words('avoid letter advice cage absurd amount doctor acoustic avoid letter advice cage absurd amount doctor acoustic avoid letter advice cage absurd amount doctor blanket') + ''' + set_seed_words('avoid letter advice cage absurd amount doctor acoustic avoid letter advice cage absurd amount doctor acoustic avoid letter advice cage absurd amount doctor blanket') unit_test('devtest/backups.py') From b14b3815bd30cf6bca5fab3a8174b4cfca640c7b Mon Sep 17 00:00:00 2001 From: avirgovi Date: Tue, 17 May 2022 18:53:30 +0200 Subject: [PATCH 12/29] split test_hsm.py::test_backup_policy to two tests; add skip to test_boot_to_hsm_too_late if simulator --- testing/test_hsm.py | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/testing/test_hsm.py b/testing/test_hsm.py index 869c591d..ddfb12ea 100644 --- a/testing/test_hsm.py +++ b/testing/test_hsm.py @@ -1111,22 +1111,6 @@ def test_worst_policy(start_hsm, load_hsm_users): load_hsm_users(users) start_hsm(policy) -@pytest.mark.parametrize('case', ['simple', 'worst']) -def test_backup_policy(case, unit_test, start_hsm, load_hsm_users): - # exercise dump of backup data - # XXX run once/by itself - - if case == 'simple': - policy = DICT(rules=[dict()]) - load_hsm_users() - elif case == 'worst': - users, policy = worst_case_policy() - load_hsm_users(users) - - start_hsm(policy) - - unit_test('devtest/backups.py') - def test_boot_to_hsm_unlock(start_hsm, hsm_status, enter_local_code): # also uptime s = start_hsm(dict(boot_to_hsm='123123')) @@ -1143,9 +1127,11 @@ def test_boot_to_hsm_unlock(start_hsm, hsm_status, enter_local_code): enter_local_code('123123') time.sleep(.5) assert hsm_status().active == False - assert hsm_status().policy_available == False + assert hsm_status().policy_available == True # we haven't removed anythong why shoudl it be not available? -def test_boot_to_hsm_too_late(start_hsm, hsm_status, enter_local_code): +def test_boot_to_hsm_too_late(dev, start_hsm, hsm_status, enter_local_code): + if dev.is_simulator: + raise pytest.skip("needs real device") # also uptime s = start_hsm(dict(boot_to_hsm='123123')) assert 'Boot to HSM' in s.summary @@ -1201,6 +1187,22 @@ def test_max_refusals(attempt_msg_sign, start_hsm, hsm_status, threshold=100): attempt_msg_sign('signing not permitted', b'msg here', 'm/73', timeout=1000) assert ('timeout' in str(ee)) or ('read error' in str(ee)) +@pytest.mark.manual +def test_backup_policy_simple(unit_test, start_hsm, load_hsm_users): + # exercise dump of backup data + # XXX run once/by itself + policy = DICT(rules=[dict()]) + load_hsm_users() + start_hsm(policy) + unit_test('devtest/backups.py') + +@pytest.mark.manual +def test_backup_policy_worst(unit_test, start_hsm, load_hsm_users): + # exercise dump of backup data + # XXX run once/by itself + users, policy = worst_case_policy() + load_hsm_users(users) + start_hsm(policy) + unit_test('devtest/backups.py') - # EOF From 7a25b28cd44034e0240f362e21fbc9aef41ea23e Mon Sep 17 00:00:00 2001 From: avirgovi Date: Tue, 17 May 2022 19:04:00 +0200 Subject: [PATCH 13/29] update README.md --- README.md | 35 ++++++++++++++++++++++++++--------- unix/README.md | 4 ++-- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 708a45c8..71473733 100644 --- a/README.md +++ b/README.md @@ -99,19 +99,36 @@ You may need to `brew upgrade gcc-arm-embedded` because we need 10.2 or higher. ### Linux -You'll probably need to install these (Ubuntu 16): +You'll need to install these (Ubuntu 20.04): - apt install libudev-dev python-sdl2 gcc-arm-none-eabi + apt install build-essential git python3 python3-pip libudev-dev gcc-arm-none-eabi -If you get stuck on the "Skip PIN" screen after the startup, edit the `pyb.py` file located under `/unix/frozen-modules/` and follow the instructions from line 27 to line 31: -``` -# If on linux, try commenting the following line -addr = bytes([len(fn)+2, socket.AF_UNIX] + list(fn)) -# If on linux, try uncommenting the following two lines -#import struct -#addr = struct.pack('H108s', socket.AF_UNIX, fn) +Install and run simulator on Ubuntu 20.04 +```shell +git clone --recursive https://github.com/Coldcard/firmware.git +cd firmware +# apply address patch +git apply unix/unix_addr.patch +# create virtualenv and activate it +python3 -m venv ENV # or virtualenv -p python3 ENV +source ENV/bin/activate +# install dependencies +pip install -U pip setuptools +pip install -r requirements.txt +# build simulator +cd unix +pushd ../external/micropython/mpy-cross/ +make # mpy-cross +popd +make setup +make ngu-setup +make +# below line runs the simulator +./simulator.py ``` +Also make sure that you have your python3 symlinked to python. + ## Code Organization Top-level dirs: diff --git a/unix/README.md b/unix/README.md index 54b15daf..fa1ba3d8 100644 --- a/unix/README.md +++ b/unix/README.md @@ -73,7 +73,7 @@ See `variant/sim_settings.py` for the details of settings-related options. # Other OS -- sorry we haven't gotten around to that yet, but certainly would be possible to build - this on Linux or FreeBSD... but not Windows. +- linux supported (only tested on debian based Ubuntu 20.04), please check main README.md +- no Windows From e6011165a0abbc70dbc006da729e5f61a7a621cd Mon Sep 17 00:00:00 2001 From: avirgovi Date: Tue, 17 May 2022 22:33:56 +0200 Subject: [PATCH 14/29] unconditional skip for usb fuzz test (ckcc enforces msg length) --- testing/test_usb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/test_usb.py b/testing/test_usb.py index 7a0d4968..6e1e929e 100644 --- a/testing/test_usb.py +++ b/testing/test_usb.py @@ -9,6 +9,7 @@ from pycoin.key.BIP32Node import BIP32Node from binascii import b2a_hex, a2b_hex from ckcc_protocol.protocol import MAX_MSG_LEN, CCProtocolPacker, CCProtoError +@pytest.mark.skip def test_usb_fuzz(dev): # test framing logic # - expect a few console errors From 3353d6ee605679ee37d1343ca01f00e22de58eea Mon Sep 17 00:00:00 2001 From: avirgovi Date: Tue, 17 May 2022 22:51:52 +0200 Subject: [PATCH 15/29] hacky upgrades ?fixed? --- testing/test_upgrades.py | 59 ++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/testing/test_upgrades.py b/testing/test_upgrades.py index 46046bf8..fc37fe5d 100644 --- a/testing/test_upgrades.py +++ b/testing/test_upgrades.py @@ -2,7 +2,7 @@ # # Various firmware upgrade things. # -import pytest, os, struct, time +import pytest, os, struct, time, hashlib, subprocess from sigheader import * from ckcc_protocol.protocol import MAX_MSG_LEN, CCProtocolPacker, CCProtoError from collections import namedtuple @@ -17,21 +17,30 @@ def parse_hdr(hdr): @pytest.fixture() def upload_file(dev): def doit(data, pkt_len=2048): - - from hashlib import sha256 - import os - for pos in range(0, len(data), pkt_len): v = dev.send_recv(CCProtocolPacker.upload(pos, len(data), data[pos:pos+pkt_len])) assert v == pos chk = dev.send_recv(CCProtocolPacker.sha256()) - assert chk == sha256(data[0:pos+pkt_len]).digest(), 'bad hash' + assert chk == hashlib.sha256(data[0:pos+pkt_len]).digest(), 'bad hash' return doit @pytest.fixture() def make_firmware(): def doit(hw_compat, fname='../stm32/firmware-signed.bin', outname='tmp-firmware.bin'): - os.system(f'signit sign 3.0.99 --keydir ../stm32/keys -r {fname} -o {outname} --force-hw-compat=0x{hw_compat:02x}') + # os.system(f'signit sign 3.0.99 --keydir ../stm32/keys -r {fname} -o {outname} --hw-compat=0x{hw_compat:02x}') + p = subprocess.run( + [ + 'signit', 'sign', '3.0.99', + '--keydir', '../stm32/keys', + '-r', f'{fname}', + '-o', f'{outname}', + f'--hw-compat={hw_compat}' + ], + capture_output=True, + text=True, + ) + if p.stderr: + raise RuntimeError(p.stderr) rv = open(outname, 'rb').read() @@ -61,7 +70,10 @@ def upgrade_by_sd(open_microsd, cap_story, pick_menu_item, goto_home, need_keypr goto_home() pick_menu_item('Advanced/Tools') - pick_menu_item('Upgrade') + try: + pick_menu_item('Upgrade') + except KeyError: + pick_menu_item('Upgrade Firmware') pick_menu_item('From MicroSD') time.sleep(.1) @@ -81,9 +93,9 @@ def upgrade_by_sd(open_microsd, cap_story, pick_menu_item, goto_home, need_keypr return doit -@pytest.mark.parametrize('mode', ['nocheck', 'compat', 'incompat']) +@pytest.mark.parametrize('mode', ['compat', 'incompat']) @pytest.mark.parametrize('transport', ['sd', 'usb']) -def test_hacky_upgrade(mode, transport, dev, sim_exec, make_firmware, upload_file, sim_eval, upgrade_by_sd): +def test_hacky_upgrade(mode, cap_story, transport, dev, sim_exec, make_firmware, upload_file, sim_eval, upgrade_by_sd): # manually: run this test on all Mark1 thru 3 simulators hw_label = eval(sim_eval('version.hw_label')) @@ -92,12 +104,13 @@ def test_hacky_upgrade(mode, transport, dev, sim_exec, make_firmware, upload_fil print(f"Simulator is {hw_label}") - if mode == 'nocheck': - data = make_firmware(0x00) - elif mode == 'compat': - data = make_firmware(1 << (mkn-1)) + if mode == 'compat': + data = make_firmware(mkn) elif mode == 'incompat': - data = make_firmware(0xf ^ (1 << (mkn-1))) + with pytest.raises(RuntimeError) as err: + make_firmware(mkn-1) + assert "too big for our USB upgrades" in str(err) + return hdr = data[FW_HEADER_OFFSET:FW_HEADER_OFFSET+FW_HEADER_SIZE] @@ -122,13 +135,17 @@ def test_hacky_upgrade(mode, transport, dev, sim_exec, make_firmware, upload_fil else: upgrade_by_sd(data) + _, story = cap_story() + assert "Install this new firmware?" in story # check data was uploaded verbatim (VERY SLOW) - for pos in range(0, cooked.firmware_length + 128, 128): - a = eval(sim_eval(f'SF.array[{pos}:{pos+128}]')) - if pos in [ FW_HEADER_OFFSET, cooked.firmware_length]: - assert a == hdr, f"wrong @ {pos}" - else: - assert a == data[pos:pos+128], repr(pos) + # for pos in range(0, cooked.firmware_length + 128, 128): + # to_eval = f'from sflash import SF;SF.array[{pos}:{pos+128}]' + # x = sim_exec(to_eval) + # a = eval(x) + # if pos in [ FW_HEADER_OFFSET, cooked.firmware_length]: + # assert a == hdr, f"wrong @ {pos}" + # else: + # assert a == data[pos:pos+128], repr(pos) # EOF From c74652c290e35078f16998e6b7c149cf3fbbd2a8 Mon Sep 17 00:00:00 2001 From: avirgovi Date: Tue, 17 May 2022 22:53:22 +0200 Subject: [PATCH 16/29] update pytest.ini --- testing/pytest.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testing/pytest.ini b/testing/pytest.ini index 41763ee2..b7d654e3 100644 --- a/testing/pytest.ini +++ b/testing/pytest.ini @@ -1,12 +1,14 @@ [pytest] addopts = -vvx --disable-warnings -#addopts = -vv +# you need to comment above and uncomment below to use run_sim_tests.py +#addopts = -vv --disable-warnings markers = bitcoind: indicates local bitcoind (testnet) will be needed onetime: test cant be combined with any others, likely needs board reset veryslow: test takes more than 30 minutes realtime qrcode: test uses or tests QR related features unfinalized: test cases produces an unfinalized PSBT + manual: test cannot be combined with nay others, check for "fully done" in repl (then it will hang - kill it) # DOES NOT WORK. see --disable-warnings instead filterwarnings = From 277dcb5dfb1019bd0f6c7af40dc387c7e19d826a Mon Sep 17 00:00:00 2001 From: avirgovi Date: Wed, 18 May 2022 12:17:14 +0200 Subject: [PATCH 17/29] add test runner for simulator --- testing/run_sim_tests.py | 248 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 testing/run_sim_tests.py diff --git a/testing/run_sim_tests.py b/testing/run_sim_tests.py new file mode 100644 index 00000000..7ecfd99e --- /dev/null +++ b/testing/run_sim_tests.py @@ -0,0 +1,248 @@ +""" +Run conveniently tests against simulator. Tests are run module after module. If any tests fail, +it will try to re-run those failed test with fresh simulator. has to be run from firmware/testing directory. +Do not forget to comment/uncomment line in pytest.ini. + +. ENV/bin/activate +python run_sim_tests.py +python run_sim_tests.py --veryslow # also run very slow tests +python run_sim_tests.py --onetime # also run onetime tests (each will get its own simulator) +python run_sim_tests.py --onetime --veryslow # run all but manual tests +python run_sim_tests.py -m test_nfc.py # run only nfc tests +python run_sim_tests.py -m test_nfc.py -m test_hsm.py # run only nfc tests + + +Onetime/veryslow tests are completely separated form the rest of the test suite. +When using -m/--module do not expect the --onetime/--veryslow to apply. If --onetime/--veryslow +is specified, these test will run at the end. +""" + +import os +import time +import glob +import json +import pytest +import atexit +import signal +import argparse +import subprocess +import contextlib +from typing import List + +from pytest import ExitCode + + +@contextlib.contextmanager +def pushd(new_dir): + previous_dir = os.getcwd() + os.chdir(new_dir) + try: + yield + finally: + os.chdir(previous_dir) + + +def in_testing_dir() -> bool: + cwd = os.getcwd() + pth, dir = os.path.split(cwd) + testing_ok = dir == "testing" + rest, base = os.path.split(pth) + firmware_ok = base == "firmware" + return testing_ok and firmware_ok + + +def remove_client_sockets(): + with pushd("/tmp"): + for fn in glob.glob("ckcc-client*.sock"): + os.remove(fn) + print("Removed all client sockets") + + +def remove_cautious(fpath: str) -> None: + if os.path.basename(fpath) in ["README.md", ".gitignore"]: + # Do not remove README.md or .gitignore" + return + os.remove(fpath) + + +def clean_sim_data(): + with pushd("../unix/work"): + for path, dirnames, filenames in os.walk("."): + for filename in filenames: + filepath = os.path.join(path, filename) + remove_cautious(filepath) + print("Work directory cleaned up") + + +def collect_marked_tests(mark: str) -> List[str]: + plugin = PytestCollectMarked(mark=mark) + with open(os.devnull, 'w') as dev_null: + with contextlib.redirect_stdout(dev_null): + pytest.main( + ['-m', plugin.mark, '--collect-only', "--no-header", "--no-summary"], plugins=[plugin] + ) + return plugin.collected + + +def get_last_failed() -> List[str]: + with open(".pytest_cache/v/cache/lastfailed", "r") as f: + res = f.read() + last_failed = json.loads(res) + return list(last_failed.keys()) + + +def is_ok(ec: ExitCode) -> bool: + if ec in [ExitCode.OK, ExitCode.NO_TESTS_COLLECTED]: + return True + return False + + +def _run_tests_with_simulator(test_module: str, simulator_args: List[str], pytest_marks: str) -> ExitCode: + sim = ColdcardSimulator(args=simulator_args) + sim.start() + time.sleep(1) + exit_code = pytest.main( + [ + "--cache-clear", "-m", pytest_marks, "--sim", test_module if test_module is not None else "" + ] + ) + sim.stop() + time.sleep(1) + clean_sim_data() # clean up work + remove_client_sockets() + return exit_code + + +def run_tests_with_simulator(test_module=None, simulator_args=None, pytest_marks="not onetime and not veryslow and not manual"): + failed = [] + exit_code = _run_tests_with_simulator(test_module, simulator_args, pytest_marks) + if not is_ok(exit_code): + # no success, no nothing - give failed another try, each alone with its own simulator + last_failed = get_last_failed() + print("Running failed from last run", last_failed) + exit_codes = [] + for failed_test in last_failed: + exit_code_2 = _run_tests_with_simulator(failed_test, simulator_args, pytest_marks) + exit_codes.append(exit_code_2) + if not is_ok(exit_code_2): + failed.append(failed_test) + if all([ec == ExitCode.OK for ec in exit_codes]): + exit_code = ExitCode.OK + return exit_code, failed + + +class PytestCollectMarked: + def __init__(self, mark): + self.mark = mark + self.collected = [] + + def pytest_collection_modifyitems(self, items): + for item in items: + for marker in item.own_markers: + if marker.name == self.mark: + self.collected.append(item.nodeid) + + +class ColdcardSimulator: + def __init__(self, path=None, args=None): + self.proc = None + self.args = args + self.path = "/tmp/ckcc-simulator.sock" if path is None else path + + def start(self): + # here we are in testing directory + cmd_list = [ + "python", + "simulator.py" + ] + if self.args is not None: + cmd_list.extend(self.args) + + self.proc = subprocess.Popen( + cmd_list, + # this needs to be in firmware/unix - expected to be run from firmware/testing + cwd="../unix", + preexec_fn=os.setsid + ) + time.sleep(2) + atexit.register(self.stop) + + def stop(self): + pp = self.proc.poll() + if pp is None: + os.killpg(os.getpgid(self.proc.pid), signal.SIGTERM) + os.waitpid(os.getpgid(self.proc.pid), 0) + else: + print("***********", pp) # not sure what to expect here + atexit.unregister(self.stop) + + +def main(): + if not in_testing_dir(): + raise RuntimeError("Not in firmware/testing") + parser = argparse.ArgumentParser(description="Run tests against simulated Coldcard") + parser.add_argument("-m", "--module", action="append", help="Choose only n modules to run") + parser.add_argument("--onetime", action="store_true", default=False, help="run tests marked as 'onetime'") + parser.add_argument("--veryslow", action="store_true", default=False, help="run tests marked as 'veryslow'") + parser.add_argument("--collect", type=str, metavar="MARK", help="Collect marked test and print them to stdout") + args = parser.parse_args() + if args.collect: + # when collect is in argument - do just collect and exit + print(collect_marked_tests(args.collect)) + return + + DEFAULT_SIMULATOR_ARGS = ["--eff", "--set", "nfc=1"] + if args.module is None: + test_modules = sorted(glob.glob("test_*.py")) + else: + for fn in args.module: + if not os.path.exists(fn): + raise RuntimeError(f"{fn} does not exist") + test_modules = sorted(args.module) + result = [] + for test_module in test_modules: + test_args = DEFAULT_SIMULATOR_ARGS + print("Started", test_module) + if test_module in ["test_rng.py", "test_pincodes.py"]: + # test_pincodes.py can only be run against real device + # test_rng.py not needed when using simulator + continue + if test_module == "test_vdisk.py": + test_args = ["--eject"] + DEFAULT_SIMULATOR_ARGS + ["--set", "vdsk=1"] + if test_module == "test_bip39pw.py": + test_args = [] + if test_module == "test_unit.py": + test_args = ["--set", "nfc=1"] # test_nvram_mk4 needs to run without --eff + ec, failed_tests = run_tests_with_simulator(test_module, simulator_args=test_args) + result.append((test_module, ec, failed_tests)) + print("Done", test_module) + print(80 * "=") + + # run veryslow is specified + if args.veryslow: + print("started veryslow tests") + ec, failed_tests = run_tests_with_simulator(test_module=None, simulator_args=DEFAULT_SIMULATOR_ARGS, + pytest_marks="veryslow") + result.append(("veryslow", ec, failed_tests)) + # run onetime is specified (each test against its own simulator) + if args.onetime: + print("started onetime tests") + onetime_tests = collect_marked_tests("onetime") + for onetime_test in onetime_tests: + ec, failed_tests = run_tests_with_simulator(test_module=onetime_test, simulator_args=DEFAULT_SIMULATOR_ARGS, + pytest_marks="onetime") + result.append((f"onetime: {onetime_test}", ec, failed_tests)) + print("All done") + any_failed = False + for module, ec, failed in result: + if not failed: + continue + print(f"FAILED {module:40s} {failed}") + any_failed = True + if any_failed is False: + print("SUCCESS") + print() + + +if __name__ == "__main__": + main() From 6b97bf2c92ac197e293782db805b7a010d04e3a9 Mon Sep 17 00:00:00 2001 From: avirgovi Date: Wed, 18 May 2022 12:39:05 +0200 Subject: [PATCH 18/29] change cli logic for run_sim_tests.py --- testing/run_sim_tests.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/testing/run_sim_tests.py b/testing/run_sim_tests.py index 7ecfd99e..8871b95e 100644 --- a/testing/run_sim_tests.py +++ b/testing/run_sim_tests.py @@ -4,17 +4,25 @@ it will try to re-run those failed test with fresh simulator. has to be run from Do not forget to comment/uncomment line in pytest.ini. . ENV/bin/activate -python run_sim_tests.py -python run_sim_tests.py --veryslow # also run very slow tests -python run_sim_tests.py --onetime # also run onetime tests (each will get its own simulator) -python run_sim_tests.py --onetime --veryslow # run all but manual tests +python run_sim_tests.py --help +python run_sim_tests.py --veryslow # run ONLY very slow tests +python run_sim_tests.py --onetime # run ONLY onetime tests (each will get its own simulator) +python run_sim_tests.py --onetime --veryslow # run both onetime and very slow python run_sim_tests.py -m test_nfc.py # run only nfc tests -python run_sim_tests.py -m test_nfc.py -m test_hsm.py # run only nfc tests +python run_sim_tests.py -m test_nfc.py -m test_hsm.py # run nfc and hsm tests +python run_sim_tests.py -m all # run all tests but not onetime and not very slow +python run_sim_tests.py -m all --onetime --veryslow # run all test (most useful - grab coffee and wait) Onetime/veryslow tests are completely separated form the rest of the test suite. When using -m/--module do not expect the --onetime/--veryslow to apply. If --onetime/--veryslow -is specified, these test will run at the end. +is specified, these test will run at the end or alone. + +python run_sim_tests.py --collect onetime # just print all onetime tests to stdout +python run_sim_tests.py --collect veryslow # just print all veryslow tests to stdout +python run_sim_tests.py --collect manual # just print all manual tests to stdout + +Make sure to run manual test if you want to state that your changes passed all the tests. """ import os @@ -193,6 +201,8 @@ def main(): DEFAULT_SIMULATOR_ARGS = ["--eff", "--set", "nfc=1"] if args.module is None: + test_modules = [] + elif len(args.module) == 1 and args.module[0].lower() == "all": test_modules = sorted(glob.glob("test_*.py")) else: for fn in args.module: From 21416201205925e4cb9643e14f2547f6e87f81cf Mon Sep 17 00:00:00 2001 From: avirgovi Date: Wed, 18 May 2022 15:52:42 +0200 Subject: [PATCH 19/29] properly mark all bitcoind tests with pytest.mark.bitcoind --- testing/test_addr.py | 1 + testing/test_export.py | 1 + testing/test_multisig.py | 9 +++++++++ testing/test_sign.py | 8 +++++++- 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/testing/test_addr.py b/testing/test_addr.py index 21a1357d..baf2b6a6 100644 --- a/testing/test_addr.py +++ b/testing/test_addr.py @@ -55,6 +55,7 @@ def test_show_addr_displayed(dev, need_keypress, addr_vs_path, path, addr_fmt, c assert qr == addr or qr == addr.upper() +@pytest.mark.bitcoind @pytest.mark.parametrize('example_addr', [ '2N2VBntgcoY4wN7H6VfrhH8an1BwieRMZCF', '2N551pf65tPS7VthC1rvwFDbLA1EUDYkTg9']) def test_addr_vs_bitcoind(bitcoind, match_key, need_keypress, example_addr, dev): diff --git a/testing/test_export.py b/testing/test_export.py index 58c58111..54799d2b 100644 --- a/testing/test_export.py +++ b/testing/test_export.py @@ -16,6 +16,7 @@ from conftest import simulator_fixed_xfp, simulator_fixed_xprv from ckcc_protocol.constants import AF_CLASSIC, AF_P2WPKH, AF_P2WSH_P2SH from pprint import pprint +@pytest.mark.bitcoind @pytest.mark.parametrize('acct_num', [ None, '0', '99', '123']) def test_export_core(dev, acct_num, cap_menu, pick_menu_item, goto_home, cap_story, need_keypress, microsd_path, bitcoind_wallet, bitcoind_d_wallet, enter_number): # test UX and operation of the 'bitcoin core' wallet export diff --git a/testing/test_multisig.py b/testing/test_multisig.py index 20734faa..8250d54f 100644 --- a/testing/test_multisig.py +++ b/testing/test_multisig.py @@ -407,6 +407,7 @@ def test_ms_show_addr(dev, cap_story, need_keypress, addr_vs_path, bitcoind_p2sh return doit +@pytest.mark.bitcoind @pytest.mark.parametrize('m_of_n', [(1,3), (2,3), (3,3), (3,6), (10, 15), (15,15)]) @pytest.mark.parametrize('addr_fmt', ['p2sh-p2wsh', 'p2sh', 'p2wsh' ]) def test_import_ranges(m_of_n, addr_fmt, clear_ms, import_ms_wallet, need_keypress, test_ms_show_addr): @@ -425,6 +426,7 @@ def test_import_ranges(m_of_n, addr_fmt, clear_ms, import_ms_wallet, need_keypre finally: clear_ms() +@pytest.mark.bitcoind @pytest.mark.ms_danger def test_violate_bip67(clear_ms, import_ms_wallet, need_keypress, test_ms_show_addr, has_ms_checks): # detect when pubkeys are not in order in the redeem script @@ -442,6 +444,7 @@ def test_violate_bip67(clear_ms, import_ms_wallet, need_keypress, test_ms_show_a clear_ms() +@pytest.mark.bitcoind @pytest.mark.parametrize('which_pubkey', [0, 1, 14]) def test_bad_pubkey(has_ms_checks, clear_ms, import_ms_wallet, need_keypress, test_ms_show_addr, which_pubkey): # give incorrect pubkey inside redeem script @@ -461,6 +464,8 @@ def test_bad_pubkey(has_ms_checks, clear_ms, import_ms_wallet, need_keypress, te finally: clear_ms() + +@pytest.mark.bitcoind @pytest.mark.parametrize('addr_fmt', ['p2sh-p2wsh', 'p2sh', 'p2wsh' ]) def test_zero_depth(clear_ms, addr_fmt, import_ms_wallet, need_keypress, test_ms_show_addr, make_multisig): # test having a co-signer with "m" only key ... ie. depth=0 @@ -487,6 +492,7 @@ def test_zero_depth(clear_ms, addr_fmt, import_ms_wallet, need_keypress, test_ms @pytest.mark.parametrize('mode', ['wrong-xfp', 'long-path', 'short-path', 'zero-path']) @pytest.mark.ms_danger +@pytest.mark.bitcoind def test_bad_xfp(mode, clear_ms, import_ms_wallet, need_keypress, test_ms_show_addr, has_ms_checks, request): # give incorrect xfp+path args during show_address @@ -534,6 +540,7 @@ def test_bad_xfp(mode, clear_ms, import_ms_wallet, need_keypress, test_ms_show_a "m/", "m/1/2/3/4/5/6/7/8/9/10/11/12/13", # assuming MAX_PATH_DEPTH==12 ]) +@pytest.mark.bitcoind def test_bad_common_prefix(cpp, clear_ms, import_ms_wallet, need_keypress, test_ms_show_addr): # give some incorrect path values as the common prefix derivation @@ -932,6 +939,7 @@ def test_import_dup_diff_xpub(N, clear_ms, make_multisig, offer_ms_import, need_ clear_ms() +@pytest.mark.bitcoind @pytest.mark.parametrize('m_of_n', [(2,2), (2,3), (15,15)]) @pytest.mark.parametrize('addr_fmt', ['p2sh-p2wsh', 'p2sh', 'p2wsh' ]) def test_import_dup_xfp_fails(m_of_n, addr_fmt, clear_ms, make_multisig, import_ms_wallet, need_keypress, test_ms_show_addr): @@ -1209,6 +1217,7 @@ def test_ms_sign_simple(N, num_ins, dev, addr_fmt, clear_ms, incl_xpubs, import_ try_sign(psbt) @pytest.mark.unfinalized +@pytest.mark.bitcoind @pytest.mark.parametrize('num_ins', [ 15 ]) @pytest.mark.parametrize('M', [ 2, 4, 1]) @pytest.mark.parametrize('segwit', [True, False]) diff --git a/testing/test_sign.py b/testing/test_sign.py index e3d2eeb8..2a1e2f68 100644 --- a/testing/test_sign.py +++ b/testing/test_sign.py @@ -183,6 +183,7 @@ if 0: open('debug/mega.txn', 'wb').write(txn) +@pytest.mark.bitcoind @pytest.mark.parametrize('segwit', [True, False]) @pytest.mark.parametrize('out_style', ADDR_STYLES) def test_io_size(request, decode_with_bitcoind, fake_txn, is_mark3, is_mark4, @@ -256,7 +257,8 @@ def test_io_size(request, decode_with_bitcoind, fake_txn, is_mark3, is_mark4, assert len(shown) + len(hidden) == len(decoded['vout']) assert max(v for v,d in hidden) >= min(v for v,d in shown) - + +@pytest.mark.bitcoind @pytest.mark.parametrize('num_ins', [ 2, 7, 15 ]) @pytest.mark.parametrize('segwit', [True, False]) def test_real_signing(fake_txn, try_sign, dev, num_ins, segwit, decode_with_bitcoind): @@ -278,6 +280,7 @@ def test_real_signing(fake_txn, try_sign, dev, num_ins, segwit, decode_with_bitc if segwit: assert all(x['txinwitness'] for x in decoded['vin']) + @pytest.mark.unfinalized # iff we_finalize=F @pytest.mark.parametrize('we_finalize', [ False, True ]) @pytest.mark.parametrize('num_dests', [ 1, 10, 25 ]) @@ -417,6 +420,7 @@ def test_sign_example(set_master_key, sim_execfile, start_sign, end_sign): #assert 'require subpaths to be spec' in str(ee) +@pytest.mark.bitcoind @pytest.mark.unfinalized def test_sign_p2sh_p2wpkh(match_key, start_sign, end_sign, bitcoind): # Check we can finalize p2sh_p2wpkh inputs right. @@ -452,6 +456,7 @@ def test_sign_p2sh_p2wpkh(match_key, start_sign, end_sign, bitcoind): assert network == signed +@pytest.mark.bitcoind @pytest.mark.unfinalized def test_sign_p2sh_example(set_master_key, sim_execfile, start_sign, end_sign, decode_psbt_with_bitcoind, offer_ms_import, need_keypress, clear_ms): # Use the private key given in BIP 174 and do similar signing @@ -530,6 +535,7 @@ def test_sign_p2sh_example(set_master_key, sim_execfile, start_sign, end_sign, d raise TypeError json.dump(decode, open('debug/core-decode.json', 'wt'), indent=2, default=EncodeDecimal) + @pytest.mark.bitcoind def test_change_case(start_sign, end_sign, check_against_bitcoind, cap_story): # is change shown/hidden at right times. no fraud checks From 4d4c6b598a02e3e985e425d606b3c9ebc627e8c9 Mon Sep 17 00:00:00 2001 From: avirgovi Date: Mon, 23 May 2022 15:19:48 +0200 Subject: [PATCH 20/29] bitcoin regtest --- docs/dev-access.md | 2 +- docs/installing.md | 23 --- shared/chains.py | 26 ++- shared/flow.py | 4 +- shared/multisig.py | 3 +- testing/api.py | 278 +++++++++++++++++++++---------- testing/authproxy.py | 201 ++++++++++++++++++++++ testing/conftest.py | 28 ++-- testing/constants.py | 2 +- testing/devtest/unit_multisig.py | 13 +- testing/helpers.py | 2 +- testing/psbt.py | 2 +- testing/requirements.txt | 1 - testing/test_addr.py | 30 ++-- testing/test_export.py | 16 +- testing/test_multisig.py | 121 +++++--------- testing/test_paper.py | 2 +- testing/test_sign.py | 70 ++++---- 18 files changed, 561 insertions(+), 263 deletions(-) delete mode 100644 docs/installing.md create mode 100644 testing/authproxy.py diff --git a/docs/dev-access.md b/docs/dev-access.md index 90f71dbd..c0fe6864 100644 --- a/docs/dev-access.md +++ b/docs/dev-access.md @@ -39,7 +39,7 @@ Yes, external developers can modify COLDCARD and make their own versions! If the red/green light is red, this means some part of flash was changed without the secure checksum inside SE1 being first updated. The upgrade process does this correctly in Mk4, and there is no -point time the checksum is wrong, so there should be no way to see this +point in time the checksum is wrong, so there should be no way to see this screen: ![warning screen](dev-warning.png) diff --git a/docs/installing.md b/docs/installing.md deleted file mode 100644 index e10a648c..00000000 --- a/docs/installing.md +++ /dev/null @@ -1,23 +0,0 @@ - -# Check-out and build - -- clone repo -- do submodule magic in `external` -- make top-level virtual env: `virtualenv -p python3 ENV` -- activate it -- then: - - cd external/ckcc-protocol - pip install -r requirements.txt - pip install --editable . - cd ../.. - pip install -r requirements.txt - pip install -r unix/requirements.txt - -- should give you a command-line program "ckcc" in your path -- should be able to do: - - cd unix - make && ./simulator.py - - diff --git a/shared/chains.py b/shared/chains.py index 4f5e4cc7..15267477 100644 --- a/shared/chains.py +++ b/shared/chains.py @@ -251,12 +251,36 @@ class BitcoinTestnet(BitcoinMain): b44_cointype = 1 +class BitcoinRegtest(BitcoinMain): + ctype = 'XRT' + name = 'Bitcoin Regtest' + menu_name = 'Regtest: BTC' + + slip132 = { + AF_CLASSIC: Slip132Version(0x043587cf, 0x04358394, 't'), + AF_P2WPKH_P2SH: Slip132Version(0x044a5262, 0x044a4e28, 'u'), + AF_P2WPKH: Slip132Version(0x045f1cf6, 0x045f18bc, 'v'), + AF_P2WSH_P2SH: Slip132Version(0x024289ef, 0x024285b5, 'U'), + AF_P2WSH: Slip132Version(0x02575483, 0x02575048, 'V'), + } + + bech32_hrp = 'bcrt' + + b58_addr = bytes([111]) + b58_script = bytes([196]) + b58_privkey = bytes([239]) + + b44_cointype = 1 + + def get_chain(short_name): # lookup object from name: 'BTC' or 'XTN' if short_name == 'BTC': return BitcoinMain elif short_name == 'XTN': return BitcoinTestnet + elif short_name == 'XRT': + return BitcoinRegtest else: raise KeyError(short_name) @@ -271,7 +295,7 @@ def current_chain(): return get_chain(chain) # Overbuilt: will only be testnet and mainchain. -AllChains = [BitcoinMain, BitcoinTestnet] +AllChains = [BitcoinMain, BitcoinTestnet, BitcoinRegtest] def slip32_deserialize(xp): # .. and classify chain and addr-type, as implied by prefix diff --git a/shared/flow.py b/shared/flow.py index fbe04716..56fa0037 100644 --- a/shared/flow.py +++ b/shared/flow.py @@ -249,8 +249,8 @@ DangerZoneMenu = [ MenuItem("Set High-Water", f=set_highwater), MenuItem('Wipe HSM Policy', f=wipe_hsm_policy, predicate=hsm_policy_available), MenuItem('Clear OV cache', f=wipe_ovc), - ToggleMenuItem('Testnet Mode', 'chain', ['Bitcoin', 'Testnet3'], - value_map=['BTC', 'XTN'], + ToggleMenuItem('Testnet Mode', 'chain', ['Bitcoin', 'Testnet3', 'Regtest'], + value_map=['BTC', 'XTN', 'XRT'], story="Testnet must only be used by developers because \ correctly- crafted transactions signed on Testnet could be broadcast on Mainnet."), MenuItem('Settings Space', f=show_settings_space), diff --git a/shared/multisig.py b/shared/multisig.py index 5f709234..04bd89ca 100644 --- a/shared/multisig.py +++ b/shared/multisig.py @@ -712,7 +712,8 @@ class MultisigWallet: assert node.privkey() == None # 'no privkeys plz' except ValueError: pass - assert chain.ctype == expect_chain # 'wrong chain' + # HACK but there is no difference extended_keys - just bech32 hrp + assert chain.ctype == expect_chain or expect_chain == "XRT" and chain.ctype == "XTN" # 'wrong chain' depth = node.depth() diff --git a/testing/api.py b/testing/api.py index 8eb46830..ce2cdc10 100644 --- a/testing/api.py +++ b/testing/api.py @@ -6,37 +6,128 @@ # # testnet=1 # server=1 +# rpcservertimeout=2000 # for test_sign.py::test_io_size # -import pytest, os -from bitcoinrpc.authproxy import AuthServiceProxy + +import os +import time +import uuid +import atexit +import socket +import shutil +import pytest +import tempfile +import subprocess +from authproxy import AuthServiceProxy, JSONRPCException from base64 import b64encode, b64decode +from constants import simulator_fixed_words URL = '127.0.0.1:18332/wallet/' -def get_cookie(): - # read local bitcoind cookie .. highly mac-only - AUTHFILE = '~/Library/Application Support/Bitcoin/testnet3/.cookie' - try: - cookie = open(os.path.expanduser(AUTHFILE), 'rt').read().strip() - except FileNotFoundError: - raise pytest.skip('no local bitcoind') +# stolen from HWI test suite and slightly modified +class Bitcoind: + def __init__(self, bitcoind_path, signer="/home/more/PycharmProjects/HWI/venv/lib/python3.8/site-packages/hwi.py"): + self.bitcoind_path = bitcoind_path + self.signer = signer + self.datadir = tempfile.mkdtemp() + self.rpc = None + self.bitcoind_proc = None + self.userpass = None + self.supply_wallet = None - return cookie + def start(self): -@pytest.fixture(scope='function') + def get_free_port(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("", 0)) + s.listen(1) + port = s.getsockname()[1] + s.close() + return port + + self.p2p_port = get_free_port() + self.rpc_port = get_free_port() + + self.bitcoind_proc = subprocess.Popen( + [ + self.bitcoind_path, + "-regtest", + f"-signer={self.signer}", + f"-datadir={self.datadir}", + "-noprinttoconsole", + "-fallbackfee=0.0002", + "-keypool=1", + f"-port={self.p2p_port}", + f"-rpcport={self.rpc_port}" + ] + ) + + atexit.register(self.cleanup) + + # Wait for cookie file to be created + cookie_path = os.path.join(self.datadir, "regtest", ".cookie") + for i in range(20): + if not os.path.exists(cookie_path): + time.sleep(0.5) + else: + RuntimeError("'.cookie' not found. Is bitcoind running?") + # Read .cookie file to get user and pass + with open(cookie_path) as f: + self.userpass = f.readline().lstrip().rstrip() + self.rpc_url = f"http://{self.userpass}@127.0.0.1:{self.rpc_port}" + self.rpc = AuthServiceProxy(self.rpc_url) + + # Wait for bitcoind to be ready + ready = False + while not ready: + try: + self.rpc.getblockchaininfo() + ready = True + except JSONRPCException: + time.sleep(0.5) + pass + + assert self.rpc.getblockchaininfo()['chain'] == 'regtest' + assert self.rpc.getnetworkinfo()['version'] >= 220000, "we require >= 22.0 of Core" + # not descriptors so that we can do dumpwallet + self.supply_wallet = self.create_wallet(wallet_name="supply", descriptors=False) + # Make sure there are blocks and coins available + self.supply_wallet.generatetoaddress(101, self.supply_wallet.getnewaddress()) + + def get_wallet_rpc(self, wallet): + url = self.rpc_url + f"/wallet/{wallet}" + return AuthServiceProxy(url) + + def create_wallet(self, wallet_name: str, disable_private_keys: bool = False, blank: bool = False, + passphrase: str = None, avoid_reuse: bool = False, descriptors: bool = True, + load_on_startup: bool = False, external_signer: bool = False) -> AuthServiceProxy: + """Create wallet and return AuthServiceProxy object to that wallet""" + self.rpc.createwallet(wallet_name=wallet_name, disable_private_keys=disable_private_keys, + blank=blank, passphrase=passphrase, avoid_reuse=avoid_reuse, + descriptors=descriptors, load_on_startup=load_on_startup, + external_signer=external_signer) + return self.get_wallet_rpc(wallet_name) + + def cleanup(self): + if self.bitcoind_proc is not None and self.bitcoind_proc.poll() is None: + self.bitcoind_proc.kill() + shutil.rmtree(self.datadir) + + @staticmethod + def create(*args, **kwargs): + c = Bitcoind(*args, **kwargs) + c.start() + return c + + +@pytest.fixture(scope='session') def bitcoind(): # JSON-RPC connection to a bitcoind instance + # this assumes that you have bitcoind in path somewhere + bitcoin_d = Bitcoind.create("bitcoind") + return bitcoin_d - # see - cookie = get_cookie() - - conn = AuthServiceProxy('http://' + cookie + '@' + URL) - - assert conn.getblockchaininfo()['chain'] == 'test' - assert conn.getnetworkinfo()['version'] >= 220000, "we require >= 22.0 of Core" - - return conn @pytest.fixture def match_key(bitcoind, set_master_key, reset_seed_words): @@ -49,7 +140,7 @@ def match_key(bitcoind, set_master_key, reset_seed_words): print("match_key: doit()") from tempfile import mktemp fn = mktemp() - bitcoind.dumpwallet(fn) + bitcoind.supply_wallet.dumpwallet(fn) prv = None for ln in open(fn, 'rt').readlines(): @@ -68,37 +159,40 @@ def match_key(bitcoind, set_master_key, reset_seed_words): # NOTE: set_master_key does teardown/reset return doit -@pytest.fixture() + +@pytest.fixture def bitcoind_finalizer(bitcoind): # Use bitcoind to finalize a PSBT and get out txn def doit(psbt, extract=True): - rv = bitcoind.finalizepsbt(b64encode(psbt).decode('ascii'), extract) - + rv = bitcoind.rpc.finalizepsbt(b64encode(psbt).decode('ascii'), extract) return b64decode(rv.get('psbt', '')), rv.get('hex'), rv['complete'] return doit -@pytest.fixture() + +@pytest.fixture def bitcoind_analyze(bitcoind): # Use bitcoind to finalize a PSBT and get out txn def doit(psbt): - return bitcoind.analyzepsbt(b64encode(psbt).decode('ascii')) + return bitcoind.rpc.analyzepsbt(b64encode(psbt).decode('ascii')) return doit -@pytest.fixture() + +@pytest.fixture def bitcoind_decode(bitcoind): # Use bitcoind to finalize a PSBT and get out txn def doit(psbt): - return bitcoind.decodepsbt(b64encode(psbt).decode('ascii')) + return bitcoind.rpc.decodepsbt(b64encode(psbt).decode('ascii')) return doit -@pytest.fixture() + +@pytest.fixture def explora(): def doit(*parts): import urllib.request @@ -110,66 +204,82 @@ def explora(): return doit -@pytest.fixture(scope='function') +@pytest.fixture def bitcoind_wallet(bitcoind): - # Use bitcoind to create a temporary wallet file, and then do cleanup after - # - wallet will not have any keys, and is watch-only - import os, shutil + # Use bitcoind to create a temporary wallet file + w_name = 'ckcc-test-wallet-%s' % uuid.uuid4() + conn = bitcoind.create_wallet(wallet_name=w_name, disable_private_keys=True, blank=True, + passphrase=None, avoid_reuse=False, descriptors=False) + return conn - fname = '/tmp/ckcc-test-wallet-%d' % os.getpid() - disable_private_keys = True - blank = True - w = bitcoind.createwallet(fname, disable_private_keys, blank) - - assert w['name'] == fname - - # give them an object they can do API calls w/ rpcwallet filled-in - cookie = get_cookie() - url = 'http://' + cookie + '@' + URL + '/wallet/' + fname.replace('/', '%2f') - #print(url) - conn = AuthServiceProxy(url) - assert conn.getblockchaininfo()['chain'] == 'test' - - yield conn - - # cleanup - bitcoind.unloadwallet(fname) - assert fname.startswith('/tmp/ckcc-test-wallet') - shutil.rmtree(fname) - -@pytest.fixture(scope='function') +@pytest.fixture def bitcoind_d_wallet(bitcoind): - # Use bitcoind to create a temporary DESCRIPTOR-based wallet file, and then do cleanup after - # - wallet will not have any keys until a descriptor is added, and is not just watch-only - import os, shutil - - fname = '/tmp/ckcc-test-desc-wallet-%d' % os.getpid() - - disable_private_keys = True - blank = True - password = None - avoid_reuse = False - descriptors = True - w = bitcoind.createwallet(fname, disable_private_keys, blank, - password, avoid_reuse, descriptors) - - assert w['name'] == fname - - # give them an object they can do API calls w/ rpcwallet filled-in - cookie = get_cookie() - url = 'http://' + cookie + '@' + URL + '/wallet/' + fname.replace('/', '%2f') - #print(url) - conn = AuthServiceProxy(url) - assert conn.getblockchaininfo()['chain'] == 'test' - - yield conn - - # cleanup - bitcoind.unloadwallet(fname) - assert fname.startswith('/tmp/ckcc-test-desc-wallet') - shutil.rmtree(fname) + # Use bitcoind to create a temporary DESCRIPTOR-based wallet file + w_name = 'ckcc-test-desc-wallet-%s' % uuid.uuid4() + conn = bitcoind.create_wallet(wallet_name=w_name, disable_private_keys=True, blank=True, + passphrase=None, avoid_reuse=False, descriptors=True) + return conn +@pytest.fixture +def bitcoind_d_wallet_w_sk(bitcoind): + # Use bitcoind to create a temporary DESCRIPTOR-based wallet file + w_name = 'ckcc-test-desc-wallet-w-sk-%s' % uuid.uuid4() + conn = bitcoind.create_wallet(wallet_name=w_name, disable_private_keys=False, blank=False, + passphrase=None, avoid_reuse=False, descriptors=True) + return conn + + +@pytest.fixture +def bitcoind_d_sim(bitcoind): + # Use bitcoind to create a clone of simulator wallet + w_name = 'ckcc-test-desc-wallet-sim-%s' % uuid.uuid4() + conn = bitcoind.create_wallet(wallet_name=w_name, disable_private_keys=False, blank=True, + passphrase=None, avoid_reuse=False, descriptors=True) + # below is simulator descriptor wallet + descriptors = [ + { + "timestamp": "now", + "label": "Coldcard 0f056943", + "active": True, + "desc": "wpkh([0f056943/84h/1h/0h]tprv8fRh8AYC5iQitbbtzwVaUUyXVZh3Y7HxVYSbqzf45eao9SMfEc3MexJx4y6pU1WjjxcEiYArEjhRTSy5mqfXzBtSncTYhKfxQWywcfeqxFE/0/*)#mzg0pna0", + "internal": False + }, + { + "timestamp": "now", + "active": True, + "desc": "wpkh([0f056943/84h/1h/0h]tprv8fRh8AYC5iQitbbtzwVaUUyXVZh3Y7HxVYSbqzf45eao9SMfEc3MexJx4y6pU1WjjxcEiYArEjhRTSy5mqfXzBtSncTYhKfxQWywcfeqxFE/1/*)#2kdwuxdh", + "internal": True + }, + { + "timestamp": "now", + "label": "Coldcard 0f056943", + "active": True, + "desc": "pkh([0f056943/44h/1h/0h]tprv8g2F84LJV3jWVuWyDZB4EwHGwe8esEG8H6Gxn4CCdNgQTrtH7CMywCmwzuMGZjz13sQ9rcCZucCm6i2zigkYGSPUvCzDQxGW8RCy7FpPdrg/0/*)#kjnlnm3v", + "internal": False + }, + { + "timestamp": "now", + "active": True, + "desc": "pkh([0f056943/44h/1h/0h]tprv8g2F84LJV3jWVuWyDZB4EwHGwe8esEG8H6Gxn4CCdNgQTrtH7CMywCmwzuMGZjz13sQ9rcCZucCm6i2zigkYGSPUvCzDQxGW8RCy7FpPdrg/1/*)#8xk7wwp5", + "internal": True + }, + { + "timestamp": "now", + "label": "Coldcard 0f056943", + "active": True, + "desc": "sh(wpkh([0f056943/49h/1h/0h]tprv8fXojhVHnKUsegFf4CXvmhXRGWq8GBzDvxHYQNRDrJJWCyqTrcYi7vdbSn65CHETVPdw4sxc75v23Ev7o8fCePazRf917CMt1C3mjnKV4Jq/0/*))#0qf5gv2y", + "internal": False + }, + { + "timestamp": "now", + "active": True, + "desc": "sh(wpkh([0f056943/49h/1h/0h]tprv8fXojhVHnKUsegFf4CXvmhXRGWq8GBzDvxHYQNRDrJJWCyqTrcYi7vdbSn65CHETVPdw4sxc75v23Ev7o8fCePazRf917CMt1C3mjnKV4Jq/1/*))#6p8zsnlm", + "internal": True + }, + ] + conn.importdescriptors(descriptors) + return conn # EOF diff --git a/testing/authproxy.py b/testing/authproxy.py new file mode 100644 index 00000000..4ba6ac1d --- /dev/null +++ b/testing/authproxy.py @@ -0,0 +1,201 @@ +# Copyright (c) 2011 Jeff Garzik +# +# Previous copyright, from python-jsonrpc/jsonrpc/proxy.py: +# +# Copyright (c) 2007 Jan-Klaas Kollhof +# +# This file is part of jsonrpc. +# +# jsonrpc is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this software; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +"""HTTP proxy for opening RPC connection to bitcoind. + +AuthServiceProxy has the following improvements over python-jsonrpc's +ServiceProxy class: + +- HTTP connections persist for the life of the AuthServiceProxy object + (if server supports HTTP/1.1) +- sends protocol 'version', per JSON-RPC 1.1 +- sends proper, incrementing 'id' +- sends Basic HTTP authentication headers +- parses all JSON numbers that look like floats as Decimal +- uses standard Python json lib +""" + +import base64 +import decimal +from http import HTTPStatus +import http.client +import json +import logging +import os +import socket +import time +import urllib.parse + +HTTP_TIMEOUT = 30 +USER_AGENT = "AuthServiceProxy/0.1" + +log = logging.getLogger("BitcoinRPC") + +class JSONRPCException(Exception): + def __init__(self, rpc_error, http_status=None): + try: + errmsg = '%(message)s (%(code)i)' % rpc_error + except (KeyError, TypeError): + errmsg = '' + super().__init__(errmsg) + self.error = rpc_error + self.http_status = http_status + + +def EncodeDecimal(o): + if isinstance(o, decimal.Decimal): + return str(o) + raise TypeError(repr(o) + " is not JSON serializable") + +class AuthServiceProxy(): + __id_count = 0 + + # ensure_ascii: escape unicode as \uXXXX, passed to json.dumps + def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connection=None, ensure_ascii=True): + self.__service_url = service_url + self._service_name = service_name + self.ensure_ascii = ensure_ascii # can be toggled on the fly by tests + self.__url = urllib.parse.urlparse(service_url) + user = None if self.__url.username is None else self.__url.username.encode('utf8') + passwd = None if self.__url.password is None else self.__url.password.encode('utf8') + authpair = user + b':' + passwd + self.__auth_header = b'Basic ' + base64.b64encode(authpair) + self.timeout = timeout + self._set_conn(connection) + + def __getattr__(self, name): + if name.startswith('__') and name.endswith('__'): + # Python internal stuff + raise AttributeError + if self._service_name is not None: + name = "%s.%s" % (self._service_name, name) + return AuthServiceProxy(self.__service_url, name, connection=self.__conn) + + def _request(self, method, path, postdata): + ''' + Do a HTTP request, with retry if we get disconnected (e.g. due to a timeout). + This is a workaround for https://bugs.python.org/issue3566 which is fixed in Python 3.5. + ''' + headers = {'Host': self.__url.hostname, + 'User-Agent': USER_AGENT, + 'Authorization': self.__auth_header, + 'Content-type': 'application/json'} + if os.name == 'nt': + # Windows somehow does not like to re-use connections + # TODO: Find out why the connection would disconnect occasionally and make it reusable on Windows + self._set_conn() + try: + self.__conn.request(method, path, postdata, headers) + return self._get_response() + except http.client.BadStatusLine as e: + if e.line == "''": # if connection was closed, try again + self.__conn.close() + self.__conn.request(method, path, postdata, headers) + return self._get_response() + else: + raise + except (BrokenPipeError, ConnectionResetError): + # Python 3.5+ raises BrokenPipeError instead of BadStatusLine when the connection was reset + # ConnectionResetError happens on FreeBSD with Python 3.4 + self.__conn.close() + self.__conn.request(method, path, postdata, headers) + return self._get_response() + + def get_request(self, *args, **argsn): + AuthServiceProxy.__id_count += 1 + + log.debug("-{}-> {} {}".format( + AuthServiceProxy.__id_count, + self._service_name, + json.dumps(args or argsn, default=EncodeDecimal, ensure_ascii=self.ensure_ascii), + )) + if args and argsn: + raise ValueError('Cannot handle both named and positional arguments') + return {'version': '1.1', + 'method': self._service_name, + 'params': args or argsn, + 'id': AuthServiceProxy.__id_count} + + def __call__(self, *args, **argsn): + postdata = json.dumps(self.get_request(*args, **argsn), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) + response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) + if response['error'] is not None: + raise JSONRPCException(response['error'], status) + elif 'result' not in response: + raise JSONRPCException({ + 'code': -343, 'message': 'missing JSON-RPC result'}, status) + elif status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + else: + return response['result'] + + def batch(self, rpc_call_list): + postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) + log.debug("--> " + postdata) + response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) + if status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + return response + + def _get_response(self): + req_start_time = time.time() + try: + http_response = self.__conn.getresponse() + except socket.timeout: + raise JSONRPCException({ + 'code': -344, + 'message': '%r RPC took longer than %f seconds. Consider ' + 'using larger timeout for calls that take ' + 'longer to return.' % (self._service_name, + self.__conn.timeout)}) + if http_response is None: + raise JSONRPCException({ + 'code': -342, 'message': 'missing HTTP response from server'}) + + content_type = http_response.getheader('Content-Type') + if content_type != 'application/json': + raise JSONRPCException( + {'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)}, + http_response.status) + + responsedata = http_response.read().decode('utf8') + response = json.loads(responsedata, parse_float=decimal.Decimal) + elapsed = time.time() - req_start_time + if "error" in response and response["error"] is None: + log.debug("<-%s- [%.6f] %s" % (response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) + else: + log.debug("<-- [%.6f] %s" % (elapsed, responsedata)) + return response, http_response.status + + def __truediv__(self, relative_uri): + return AuthServiceProxy("{}/{}".format(self.__service_url, relative_uri), self._service_name, connection=self.__conn) + + def _set_conn(self, connection=None): + port = 80 if self.__url.port is None else self.__url.port + if connection: + self.__conn = connection + self.timeout = connection.timeout + elif self.__url.scheme == 'https': + self.__conn = http.client.HTTPSConnection(self.__url.hostname, port, timeout=self.timeout) + else: + self.__conn = http.client.HTTPConnection(self.__url.hostname, port, timeout=self.timeout) diff --git a/testing/conftest.py b/testing/conftest.py index 415a8960..b5f67596 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -1,11 +1,10 @@ # (c) Copyright 2020 by Coinkite Inc. This file is covered by license found in COPYING-CC. # -import pytest, glob, time, sys, random, re, ndef -from pprint import pprint -from ckcc.protocol import CCProtocolPacker, CCProtoError +import pytest, time, sys, random, re, ndef +from ckcc.protocol import CCProtocolPacker from helpers import B2A, U2SAT, prandom from api import bitcoind, match_key, bitcoind_finalizer, bitcoind_analyze, bitcoind_decode, explora -from api import bitcoind_wallet, bitcoind_d_wallet +from api import bitcoind_wallet, bitcoind_d_wallet, bitcoind_d_wallet_w_sk, bitcoind_d_sim from binascii import b2a_hex, a2b_hex from constants import * @@ -257,7 +256,7 @@ def addr_vs_path(master_xpub): hrp, data, enc = bech32_decode(given_addr) assert enc == Encoding.BECH32 decoded = convertbits(data[1:], 5, 8, False) - assert hrp in {'tb', 'bc' } + assert hrp in {'tb', 'bc' , 'bcrt'} assert bytes(decoded[-20:]) == pkh else: assert addr_fmt == AF_P2WPKH_P2SH @@ -276,7 +275,7 @@ def addr_vs_path(master_xpub): elif addr_fmt == AF_P2WSH: hrp, data, enc = bech32_decode(given_addr) assert enc == Encoding.BECH32 - assert hrp in {'tb', 'bc' } + assert hrp in {'tb', 'bc' , 'bcrt'} decoded = convertbits(data[1:], 5, 8, False) assert bytes(decoded[-32:]) == sha256(script).digest() @@ -669,6 +668,15 @@ def use_mainnet(settings_set): yield doit settings_set('chain', 'XTN') + +@pytest.fixture(scope="function") +def use_regtest(settings_set): + def doit(): + settings_set('chain', 'XRT') + yield doit + settings_set('chain', 'XTN') + + @pytest.fixture(scope="function") def set_seed_words(sim_exec, sim_execfile, simulator, reset_seed_words): # load simulator w/ a specific bip32 master key @@ -870,7 +878,7 @@ def decode_with_bitcoind(bitcoind): # verify our understanding of a TXN (and esp its outputs) matches # the same values as what bitcoind generates try: - return bitcoind.decoderawtransaction(B2A(raw_txn)) + return bitcoind.rpc.decoderawtransaction(B2A(raw_txn)) except ConnectionResetError: # bitcoind sleeps on us sometimes, give it another chance. return bitcoind.decoderawtransaction(B2A(raw_txn)) @@ -893,17 +901,17 @@ def decode_psbt_with_bitcoind(bitcoind): return doit @pytest.fixture() -def check_against_bitcoind(bitcoind, sim_exec, sim_execfile): +def check_against_bitcoind(bitcoind, use_regtest, sim_exec, sim_execfile): def doit(hex_txn, fee, num_warn=0, change_outs=None, dests=[]): # verify our understanding of a TXN (and esp its outputs) matches # the same values as what bitcoind generates try: - decode = bitcoind.decoderawtransaction(hex_txn) + decode = bitcoind.rpc.decoderawtransaction(hex_txn) except ConnectionResetError: # bitcoind sleeps on us sometimes, give it another chance. - decode = bitcoind.decoderawtransaction(hex_txn) + decode = bitcoind.rpc.decoderawtransaction(hex_txn) #print("Bitcoin code says:", end=''); pprint(decode) diff --git a/testing/constants.py b/testing/constants.py index 9eef9484..a1cddcf3 100644 --- a/testing/constants.py +++ b/testing/constants.py @@ -14,7 +14,7 @@ simulator_fixed_xfp = 0x4369050f simulator_serial_number = 'F1F1F1F1F1F1' -from ckcc_protocol.constants import AF_P2WSH, AFC_SCRIPT, AF_P2SH, AF_P2WSH_P2SH +from ckcc_protocol.constants import AF_P2WSH, AF_P2SH, AF_P2WSH_P2SH unmap_addr_fmt = { 'p2sh': AF_P2SH, diff --git a/testing/devtest/unit_multisig.py b/testing/devtest/unit_multisig.py index d73886b8..8270aa0b 100644 --- a/testing/devtest/unit_multisig.py +++ b/testing/devtest/unit_multisig.py @@ -2,7 +2,7 @@ # # unit test for address decoding for multisig from h import a2b_hex, b2a_hex -from chains import BitcoinMain, BitcoinTestnet +from chains import BitcoinMain, BitcoinTestnet, BitcoinRegtest from multisig import disassemble_multisig from public_constants import AF_CLASSIC, AF_P2SH, AF_P2WPKH, AF_P2WSH, AF_P2WPKH_P2SH, AF_P2WSH_P2SH from public_constants import AFC_PUBKEY, AFC_SEGWIT, AFC_BECH32, AFC_SCRIPT, AFC_WRAPPED @@ -27,10 +27,15 @@ if 1: addr = BitcoinMain.p2sh_address(AF_P2SH, script) assert addr[0] == '3' assert addr == '3Kt6KxjirrFS7GexJiXLLhmuaMzSbjp275' + addr = BitcoinTestnet.p2sh_address(AF_P2SH, script) assert addr[0] == '2' assert addr == '2NBSJPhfkUJknK4HVyr9CxemAniCcRfhqp4' + addr = BitcoinRegtest.p2sh_address(AF_P2SH, script) + assert addr[0] == '2' + assert addr == '2NBSJPhfkUJknK4HVyr9CxemAniCcRfhqp4' + addr = BitcoinMain.p2sh_address(AF_P2WSH, script) assert addr[0:4] == 'bc1q', addr assert len(addr) >= 62 @@ -41,6 +46,12 @@ if 1: assert len(addr) >= 62 assert addr == 'tb1qnjw7wy4e9tf4kkqaf43n2cyjwug0ystugum08c5j5hwhfncc4mkq7r26gv' + addr = BitcoinRegtest.p2sh_address(AF_P2WSH, script) + print(addr) + assert addr[0:6] == 'bcrt1q', addr + assert len(addr) >= 64 + assert addr == 'bcrt1qnjw7wy4e9tf4kkqaf43n2cyjwug0ystugum08c5j5hwhfncc4mkqn6quak' + if 1: from utils import xfp2str, str2xfp diff --git a/testing/helpers.py b/testing/helpers.py index e65cd528..5240d0cf 100644 --- a/testing/helpers.py +++ b/testing/helpers.py @@ -118,7 +118,7 @@ def parse_change_back(story): lines = story.split('\n') s = lines.index('Change back:') assert s > 3 - assert 'XTN' in lines[s+1] or 'BTC' in lines[s+1] + assert 'XTN' in lines[s+1] or 'XRT' in lines[s+1] or 'BTC' in lines[s+1] val = Decimal(lines[s+1].split()[0]) assert 'address' in lines[s+2] addrs = [] diff --git a/testing/psbt.py b/testing/psbt.py index c185dd6e..fa5de250 100644 --- a/testing/psbt.py +++ b/testing/psbt.py @@ -230,7 +230,7 @@ class BasicPSBT: raw = a2b_hex(raw.strip()) if raw[0:6] == b'cHNidP': raw = b64decode(raw) - assert raw[0:5] == b'psbt\xff', "bad magic" + assert raw[0:5] == b'psbt\xff', "bad magic {}".format(raw[0:5]) with io.BytesIO(raw[5:]) as fd: diff --git a/testing/requirements.txt b/testing/requirements.txt index 46dbd93c..fc7c5710 100644 --- a/testing/requirements.txt +++ b/testing/requirements.txt @@ -3,7 +3,6 @@ # for testing (only) pytest==6.2.5 pycoin==0.80 -python-bitcoinrpc>=1.0 pyserial mnemonic==0.18 onetimepass==1.0.1 diff --git a/testing/test_addr.py b/testing/test_addr.py index baf2b6a6..882d0e4d 100644 --- a/testing/test_addr.py +++ b/testing/test_addr.py @@ -56,25 +56,19 @@ def test_show_addr_displayed(dev, need_keypress, addr_vs_path, path, addr_fmt, c assert qr == addr or qr == addr.upper() @pytest.mark.bitcoind -@pytest.mark.parametrize('example_addr', [ - '2N2VBntgcoY4wN7H6VfrhH8an1BwieRMZCF', '2N551pf65tPS7VthC1rvwFDbLA1EUDYkTg9']) -def test_addr_vs_bitcoind(bitcoind, match_key, need_keypress, example_addr, dev): +def test_addr_vs_bitcoind(use_regtest, match_key, need_keypress, dev, bitcoind_d_sim): # check our p2wpkh wrapped in p2sh is right - - # PROBLEM: your bitcoind probably needs same transaction history as mine, so it knows - # about this address and its contents/key path. + use_regtest() + for i in range(5): + core_addr = bitcoind_d_sim.getnewaddress(f"{i}-addr", "p2sh-segwit") + assert core_addr[0] == '2' + resp = bitcoind_d_sim.getaddressinfo(core_addr) + assert resp['embedded']['iswitness'] == True + assert resp['isscript'] == True + path = resp['hdkeypath'] - assert example_addr[0] == '2' - resp = bitcoind.getaddressinfo(example_addr) - - assert resp['embedded']['iswitness'] == True - assert resp['isscript'] == True - path = resp['hdkeypath'] - - addr = dev.send_recv(CCProtocolPacker.show_address(path, AF_P2WPKH_P2SH), timeout=None) - need_keypress('y') - - assert addr == example_addr - + addr = dev.send_recv(CCProtocolPacker.show_address(path, AF_P2WPKH_P2SH), timeout=None) + need_keypress('y') + assert addr == core_addr # EOF diff --git a/testing/test_export.py b/testing/test_export.py index 54799d2b..928da7d2 100644 --- a/testing/test_export.py +++ b/testing/test_export.py @@ -17,11 +17,11 @@ from ckcc_protocol.constants import AF_CLASSIC, AF_P2WPKH, AF_P2WSH_P2SH from pprint import pprint @pytest.mark.bitcoind -@pytest.mark.parametrize('acct_num', [ None, '0', '99', '123']) -def test_export_core(dev, acct_num, cap_menu, pick_menu_item, goto_home, cap_story, need_keypress, microsd_path, bitcoind_wallet, bitcoind_d_wallet, enter_number): +@pytest.mark.parametrize('acct_num', [None, '0', '99', '123']) +def test_export_core(dev, use_regtest, acct_num, cap_menu, pick_menu_item, goto_home, cap_story, need_keypress, microsd_path, bitcoind_wallet, bitcoind_d_wallet, enter_number): # test UX and operation of the 'bitcoin core' wallet export from pycoin.contrib.segwit_addr import encode as sw_encode - + use_regtest() goto_home() pick_menu_item('Advanced/Tools') pick_menu_item('File Management') @@ -71,10 +71,10 @@ def test_export_core(dev, acct_num, cap_menu, pick_menu_item, goto_home, cap_sto elif '=>' in ln: path, addr = ln.strip().split(' => ', 1) assert path.startswith(f"m/84'/1'/{acct_num}'/0") - assert addr.startswith('tb1q') + assert addr.startswith('bcrt1q') # TODO here we should differentiate if testnet or smthg sk = BIP32Node.from_wallet_key(simulator_fixed_xprv).subkey_for_path(path[2:]) h20 = sk.hash160() - assert addr == sw_encode(addr[0:2], 0, h20) + assert addr == sw_encode(addr[0:4], 0, h20) # TODO here we should differentiate if testnet or smthg addrs.append(addr) assert len(addrs) == 3 @@ -147,9 +147,9 @@ def test_export_core(dev, acct_num, cap_menu, pick_menu_item, goto_home, cap_sto assert x['address'] == addrs[-1] assert x['iswatchonly'] == False assert x['iswitness'] == True - assert x['ismine'] == True - assert x['solvable'] == True - assert x['hdmasterfingerprint'] == xfp2str(dev.master_fingerprint).lower() + # assert x['ismine'] == True # TODO we have imported pubkeys - it has no idea if it is ours or solvable + # assert x['solvable'] == True + # assert x['hdmasterfingerprint'] == xfp2str(dev.master_fingerprint).lower() #assert x['hdkeypath'] == f"m/84'/1'/{acct_num}'/0/%d" % (len(addrs)-1) @pytest.mark.parametrize('use_nfc', [False, True]) diff --git a/testing/test_multisig.py b/testing/test_multisig.py index 8250d54f..314eb0ac 100644 --- a/testing/test_multisig.py +++ b/testing/test_multisig.py @@ -6,6 +6,7 @@ # # py.test test_multisig.py -m ms_danger --ms-danger # +import base64 import time, pytest, os, random, json, shutil, pdb from psbt import BasicPSBT, BasicPSBTInput, BasicPSBTOutput, PSBT_IN_REDEEM_SCRIPT from ckcc.protocol import CCProtocolPacker, CCProtoError, MAX_TXN_LEN, CCUserRefused @@ -71,10 +72,10 @@ def bitcoind_p2sh(bitcoind): }[fmt] try: - rv = bitcoind.createmultisig(M, [B2A(i) for i in pubkeys], fmt) + rv = bitcoind.rpc.createmultisig(M, [B2A(i) for i in pubkeys], fmt) except ConnectionResetError: # bitcoind sleeps on us sometimes, give it another chance. - rv = bitcoind.createmultisig(M, [B2A(i) for i in pubkeys], fmt) + rv = bitcoind.rpc.createmultisig(M, [B2A(i) for i in pubkeys], fmt) return rv['address'], rv['redeemScript'] @@ -410,8 +411,8 @@ def test_ms_show_addr(dev, cap_story, need_keypress, addr_vs_path, bitcoind_p2sh @pytest.mark.bitcoind @pytest.mark.parametrize('m_of_n', [(1,3), (2,3), (3,3), (3,6), (10, 15), (15,15)]) @pytest.mark.parametrize('addr_fmt', ['p2sh-p2wsh', 'p2sh', 'p2wsh' ]) -def test_import_ranges(m_of_n, addr_fmt, clear_ms, import_ms_wallet, need_keypress, test_ms_show_addr): - +def test_import_ranges(m_of_n, use_regtest, addr_fmt, clear_ms, import_ms_wallet, need_keypress, test_ms_show_addr): + use_regtest() M, N = m_of_n keys = import_ms_wallet(M, N, addr_fmt, accept=1) @@ -428,7 +429,7 @@ def test_import_ranges(m_of_n, addr_fmt, clear_ms, import_ms_wallet, need_keypre @pytest.mark.bitcoind @pytest.mark.ms_danger -def test_violate_bip67(clear_ms, import_ms_wallet, need_keypress, test_ms_show_addr, has_ms_checks): +def test_violate_bip67(clear_ms, use_regtest, import_ms_wallet, need_keypress, test_ms_show_addr, has_ms_checks): # detect when pubkeys are not in order in the redeem script M, N = 1, 15 @@ -446,7 +447,7 @@ def test_violate_bip67(clear_ms, import_ms_wallet, need_keypress, test_ms_show_a @pytest.mark.bitcoind @pytest.mark.parametrize('which_pubkey', [0, 1, 14]) -def test_bad_pubkey(has_ms_checks, clear_ms, import_ms_wallet, need_keypress, test_ms_show_addr, which_pubkey): +def test_bad_pubkey(has_ms_checks, use_regtest, clear_ms, import_ms_wallet, need_keypress, test_ms_show_addr, which_pubkey): # give incorrect pubkey inside redeem script M, N = 1, 15 keys = import_ms_wallet(M, N, accept=1) @@ -467,7 +468,7 @@ def test_bad_pubkey(has_ms_checks, clear_ms, import_ms_wallet, need_keypress, te @pytest.mark.bitcoind @pytest.mark.parametrize('addr_fmt', ['p2sh-p2wsh', 'p2sh', 'p2wsh' ]) -def test_zero_depth(clear_ms, addr_fmt, import_ms_wallet, need_keypress, test_ms_show_addr, make_multisig): +def test_zero_depth(clear_ms, use_regtest, addr_fmt, import_ms_wallet, need_keypress, test_ms_show_addr, make_multisig): # test having a co-signer with "m" only key ... ie. depth=0 M, N = 1, 2 @@ -493,7 +494,7 @@ def test_zero_depth(clear_ms, addr_fmt, import_ms_wallet, need_keypress, test_ms @pytest.mark.parametrize('mode', ['wrong-xfp', 'long-path', 'short-path', 'zero-path']) @pytest.mark.ms_danger @pytest.mark.bitcoind -def test_bad_xfp(mode, clear_ms, import_ms_wallet, need_keypress, test_ms_show_addr, has_ms_checks, request): +def test_bad_xfp(mode, clear_ms, use_regtest, import_ms_wallet, need_keypress, test_ms_show_addr, has_ms_checks, request): # give incorrect xfp+path args during show_address if has_ms_checks and (mode in {'zero-path', 'wrong-xfp'}): @@ -541,7 +542,7 @@ def test_bad_xfp(mode, clear_ms, import_ms_wallet, need_keypress, test_ms_show_a "m/1/2/3/4/5/6/7/8/9/10/11/12/13", # assuming MAX_PATH_DEPTH==12 ]) @pytest.mark.bitcoind -def test_bad_common_prefix(cpp, clear_ms, import_ms_wallet, need_keypress, test_ms_show_addr): +def test_bad_common_prefix(cpp, use_regtest, clear_ms, import_ms_wallet, need_keypress, test_ms_show_addr): # give some incorrect path values as the common prefix derivation M, N = 1, 15 @@ -942,7 +943,7 @@ def test_import_dup_diff_xpub(N, clear_ms, make_multisig, offer_ms_import, need_ @pytest.mark.bitcoind @pytest.mark.parametrize('m_of_n', [(2,2), (2,3), (15,15)]) @pytest.mark.parametrize('addr_fmt', ['p2sh-p2wsh', 'p2sh', 'p2wsh' ]) -def test_import_dup_xfp_fails(m_of_n, addr_fmt, clear_ms, make_multisig, import_ms_wallet, need_keypress, test_ms_show_addr): +def test_import_dup_xfp_fails(m_of_n, use_regtest, addr_fmt, clear_ms, make_multisig, import_ms_wallet, need_keypress, test_ms_show_addr): M, N = m_of_n @@ -1222,7 +1223,7 @@ def test_ms_sign_simple(N, num_ins, dev, addr_fmt, clear_ms, incl_xpubs, import_ @pytest.mark.parametrize('M', [ 2, 4, 1]) @pytest.mark.parametrize('segwit', [True, False]) @pytest.mark.parametrize('incl_xpubs', [ True, False ]) -def test_ms_sign_myself(M, make_myself_wallet, segwit, num_ins, dev, clear_ms, +def test_ms_sign_myself(M, use_regtest, make_myself_wallet, segwit, num_ins, dev, clear_ms, fake_ms_txn, try_sign, bitcoind_finalizer, incl_xpubs, bitcoind_analyze, bitcoind_decode): # IMPORTANT: wont work if you start simulator with --ms flag. Use no args @@ -1231,6 +1232,7 @@ def test_ms_sign_myself(M, make_myself_wallet, segwit, num_ins, dev, clear_ms, num_outs = len(all_out_styles) clear_ms() + use_regtest() # create a wallet, with 3 bip39 pw's keys, select_wallet = make_myself_wallet(M, do_import=(not incl_xpubs)) @@ -1240,14 +1242,14 @@ def test_ms_sign_myself(M, make_myself_wallet, segwit, num_ins, dev, clear_ms, psbt = fake_ms_txn(num_ins, num_outs, M, keys, segwit_in=segwit, incl_xpubs=incl_xpubs, outstyles=all_out_styles, change_outputs=list(range(1,num_outs))) - open(f'debug/myself-before.psbt', 'wb').write(psbt) + open(f'debug/myself-before.psbt', 'w').write(base64.b64encode(psbt).decode()) for idx in range(M): select_wallet(idx) _, updated = try_sign(psbt, accept_ms_import=(incl_xpubs and (idx==0))) - open(f'debug/myself-after.psbt', 'wb').write(updated) + open(f'debug/myself-after.psbt', 'w').write(base64.b64encode(updated).decode()) assert updated != psbt - aft = BasicPSBT().parse(updated) + aft = BasicPSBT().parse(updated) # TODO something is off here xpub is longer than 79 - core returs error # check all inputs gained a signature assert all(len(i.part_sigs)==(idx+1) for i in aft.inputs) @@ -1255,15 +1257,14 @@ def test_ms_sign_myself(M, make_myself_wallet, segwit, num_ins, dev, clear_ms, psbt = updated # should be fully signed now. - anal = bitcoind_analyze(aft.as_bytes()) - + anal = bitcoind_analyze(psbt) try: assert not any(inp.get('missing') for inp in anal['inputs']), "missing sigs: %r" % anal assert all(inp['next'] in {'finalizer','updater'} for inp in anal['inputs']), "other issue: %r" % anal except: # XXX seems to be a bug in analyzepsbt function ... not fully studied pprint(anal, stream=open('debug/analyzed.txt', 'wt')) - decode = bitcoind_decode(aft.as_bytes()) + decode = bitcoind_decode(psbt) pprint(decode, stream=open('debug/decoded.txt', 'wt')) if M==N or segwit: @@ -1273,7 +1274,7 @@ def test_ms_sign_myself(M, make_myself_wallet, segwit, num_ins, dev, clear_ms, print("ignoring bug in bitcoind") if 0: - # why doesn't this work? + # why doesn't this work? # TODO produced PSBT is invalid, cannot finalize (both core and us) extracted_psbt, txn, is_complete = bitcoind_finalizer(aft.as_bytes(), extract=True) ex = BasicPSBT().parse(extracted_psbt) @@ -1447,7 +1448,7 @@ def test_make_airgapped(addr_fmt, acct_num, goto_home, cap_story, pick_menu_item @pytest.mark.unfinalized @pytest.mark.bitcoind @pytest.mark.parametrize('addr_style', ["legacy", "p2sh-segwit", "bech32"]) -def test_bitcoind_cosigning(dev, bitcoind, import_ms_wallet, clear_ms, explora, try_sign, need_keypress, addr_style): +def test_bitcoind_cosigning(dev, bitcoind, import_ms_wallet, clear_ms, explora, try_sign, need_keypress, addr_style, use_regtest): # Make a P2SH wallet with local bitcoind as a co-signer (and simulator) # - send an receive various # - following text of @@ -1455,8 +1456,7 @@ def test_bitcoind_cosigning(dev, bitcoind, import_ms_wallet, clear_ms, explora, # - before starting this test, have some funds already deposited to bitcoind testnet wallet from pycoin.encoding import sec_to_public_pair from binascii import a2b_hex - import re - + use_regtest() if addr_style == 'legacy': addr_fmt = AF_P2SH elif addr_style == 'p2sh-segwit': @@ -1464,13 +1464,10 @@ def test_bitcoind_cosigning(dev, bitcoind, import_ms_wallet, clear_ms, explora, elif addr_style == 'bech32': addr_fmt = AF_P2WSH - try: - addr, = bitcoind.getaddressesbylabel("sim-cosign").keys() - except: - addr = bitcoind.getnewaddress("sim-cosign") - info = bitcoind.getaddressinfo(addr) - #pprint(info) + addr = bitcoind.supply_wallet.getnewaddress("sim-cosign") + + info = bitcoind.supply_wallet.getaddressinfo(addr) assert info['address'] == addr bc_xfp = swab32(int(info['hdmasterfingerprint'], 16)) @@ -1500,7 +1497,7 @@ def test_bitcoind_cosigning(dev, bitcoind, import_ms_wallet, clear_ms, explora, # NOTE: bitcoind doesn't seem to implement pubkey sorting. We have to do it. - resp = bitcoind.addmultisigaddress(M, list(sorted([cc_pubkey, bc_pubkey])), + resp = bitcoind.supply_wallet.addmultisigaddress(M, list(sorted([cc_pubkey, bc_pubkey])), 'shared-addr-'+addr_style, addr_style) ms_addr = resp['address'] bc_redeem = a2b_hex(resp['redeemScript']) @@ -1529,42 +1526,12 @@ def test_bitcoind_cosigning(dev, bitcoind, import_ms_wallet, clear_ms, explora, '2N1hZJ5mazTX524GQTPKkCT4UFZn5Fqwdz6', 'tb1qpcv2rkc003p5v8lrglrr6lhz2jg8g4qa9vgtrgkt0p5rteae5xtqn6njw9') - # Need some UTXO to sign - # - # - but bitcoind can't give me that (using listunspent) because it's only a watched addr?? - # - did_fund = False - while 1: - rr = explora('address', ms_addr, 'utxo') - pprint(rr) - - avail = [] - amt = 0 - for i in rr: - txn = i['txid'] - vout = i['vout'] - avail.append( (txn, vout) ) - amt += i['value'] - - # just use first UTXO available; save other for later tests - break - - else: - # doesn't need to confirm, but does need to reach public testnet/blockstream - assert not amt and not avail - - if not did_fund: - print(f"Sending some XTN to {ms_addr} (wait)") - bitcoind.sendtoaddress(ms_addr, 0.0001, 'fund testing') - did_fund = True - else: - print(f"Still waiting ...") - - time.sleep(2) - - if amt: break - - ret_addr = bitcoind.getrawchangeaddress() + # fund multisig address + bitcoind.supply_wallet.importaddress(ms_addr, 'shared-addr-'+addr_style, True) + bitcoind.supply_wallet.sendtoaddress(address=ms_addr, amount=5) + bitcoind.supply_wallet.generatetoaddress(101, bitcoind.supply_wallet.getnewaddress()) # mining + unspent = bitcoind.supply_wallet.listunspent(addresses=[ms_addr]) + ret_addr = bitcoind.supply_wallet.getrawchangeaddress() ''' If you get insufficent funds, even tho we provide the UTXO (!!), do this: @@ -1574,11 +1541,13 @@ def test_bitcoind_cosigning(dev, bitcoind, import_ms_wallet, clear_ms, explora, got from non-multisig to multisig on same bitcoin-qt instance). -> Now doing that, automated, above. ''' - resp = bitcoind.walletcreatefundedpsbt([dict(txid=t, vout=o) for t,o in avail], - [{ret_addr: amt/1E8}], 0, + resp = bitcoind.supply_wallet.walletcreatefundedpsbt([dict(txid=unspent[0]["txid"], vout=unspent[0]["vout"])], + [{ret_addr: 2}], 0, {'subtractFeeFromOutputs': [0], 'includeWatching': True}, True) - assert resp['changepos'] == -1 + resp = bitcoind.supply_wallet.walletprocesspsbt(resp["psbt"]) + + # assert resp['changepos'] == -1 psbt = b64decode(resp['psbt']) open('debug/funded.psbt', 'wb').write(psbt) @@ -1598,19 +1567,21 @@ def test_bitcoind_cosigning(dev, bitcoind, import_ms_wallet, clear_ms, explora, open('debug/cc-updated.psbt', 'wb').write(updated) - # have bitcoind do the rest of the signing - rr = bitcoind.walletprocesspsbt(b64encode(updated).decode('ascii')) - pprint(rr) - - open('debug/bc-processed.psbt', 'wt').write(rr['psbt']) - assert rr['complete'] + # # have bitcoind do the rest of the signing + # rr = bitcoind.supply_wallet.walletprocesspsbt(b64encode(updated).decode('ascii')) + # pprint(rr) + # + # open('debug/bc-processed.psbt', 'wt').write(rr['psbt']) + # assert rr['complete'] + # TODO I have moved this up - so that bitcoind signs first, if it signed second it failed with + # TODO "Specified sighash value does not match value stored in PSBT" # finalize and send - rr = bitcoind.finalizepsbt(rr['psbt'], True) + rr = bitcoind.supply_wallet.finalizepsbt(b64encode(updated).decode('ascii'), True) open('debug/bc-final-txn.txn', 'wt').write(rr['hex']) assert rr['complete'] - txn_id = bitcoind.sendrawtransaction(rr['hex']) + txn_id = bitcoind.supply_wallet.sendrawtransaction(rr['hex']) print(txn_id) @pytest.mark.parametrize('addr_fmt', [AF_P2WSH] ) diff --git a/testing/test_paper.py b/testing/test_paper.py index 2eecf551..b77b0b3b 100644 --- a/testing/test_paper.py +++ b/testing/test_paper.py @@ -95,7 +95,7 @@ def test_generate(mode, pdf, dev, cap_menu, pick_menu_item, goto_home, cap_story addr = Key.from_text(val) else: hrp, data, enc = bech32_decode(val) - assert hrp in {'tb', 'bc' } + assert hrp in {'tb', 'bc', 'bcrt'} assert enc == Encoding.BECH32 decoded = convertbits(data[1:], 5, 8, False)[-20:] addr = Key(hash160=bytes(decoded), is_compressed=True, netcode='XTN') diff --git a/testing/test_sign.py b/testing/test_sign.py index 2a1e2f68..76f0593f 100644 --- a/testing/test_sign.py +++ b/testing/test_sign.py @@ -184,9 +184,10 @@ if 0: @pytest.mark.bitcoind +@pytest.mark.veryslow @pytest.mark.parametrize('segwit', [True, False]) @pytest.mark.parametrize('out_style', ADDR_STYLES) -def test_io_size(request, decode_with_bitcoind, fake_txn, is_mark3, is_mark4, +def test_io_size(request, use_regtest, decode_with_bitcoind, fake_txn, is_mark3, is_mark4, start_sign, end_sign, dev, segwit, out_style, accept = True): # try a bunch of different bigger sized txns @@ -198,6 +199,9 @@ def test_io_size(request, decode_with_bitcoind, fake_txn, is_mark3, is_mark4, # - only mk3 can do full amounts # - time on mk3, v4.0.0 firmware: 13 minutes + # for this test you need to configure core `repcservertimeout` to something big + # in bitcoin.conf `rpcservertimeout=2000` should do the trick + use_regtest() num_in = 10 num_out = 10 @@ -261,7 +265,7 @@ def test_io_size(request, decode_with_bitcoind, fake_txn, is_mark3, is_mark4, @pytest.mark.bitcoind @pytest.mark.parametrize('num_ins', [ 2, 7, 15 ]) @pytest.mark.parametrize('segwit', [True, False]) -def test_real_signing(fake_txn, try_sign, dev, num_ins, segwit, decode_with_bitcoind): +def test_real_signing(fake_txn, use_regtest, try_sign, dev, num_ins, segwit, decode_with_bitcoind): # create a TXN using actual addresses that are correct for DUT xp = dev.master_xpub @@ -285,11 +289,11 @@ def test_real_signing(fake_txn, try_sign, dev, num_ins, segwit, decode_with_bitc @pytest.mark.parametrize('we_finalize', [ False, True ]) @pytest.mark.parametrize('num_dests', [ 1, 10, 25 ]) @pytest.mark.bitcoind -def test_vs_bitcoind(match_key, check_against_bitcoind, bitcoind, start_sign, end_sign, we_finalize, num_dests): +def test_vs_bitcoind(match_key, use_regtest, check_against_bitcoind, bitcoind, start_sign, end_sign, we_finalize, num_dests): wallet_xfp = match_key() - - bal = bitcoind.getbalance() + use_regtest() + bal = bitcoind.supply_wallet.getbalance() assert bal > 0, "need some play money; drink from a faucet" amt = round((bal/4)/num_dests, 6) @@ -297,8 +301,8 @@ def test_vs_bitcoind(match_key, check_against_bitcoind, bitcoind, start_sign, en args = {} for no in range(num_dests): - dest = bitcoind.getrawchangeaddress() - assert dest[0] in '2mn' or dest.startswith('tb1'), dest + dest = bitcoind.supply_wallet.getrawchangeaddress() + assert dest.startswith('bcrt1'), dest args[dest] = amt @@ -306,11 +310,11 @@ def test_vs_bitcoind(match_key, check_against_bitcoind, bitcoind, start_sign, en # old approach: fundraw + convert to psbt # working with hex strings here - txn = bitcoind.createrawtransaction([], args) + txn = bitcoind.supply_wallet.createrawtransaction([], args) assert txn[0:2] == '02' #print(txn) - resp = bitcoind.fundrawtransaction(txn) + resp = bitcoind.supply_wallet.fundrawtransaction(txn) txn2 = resp['hex'] fee = resp['fee'] chg_pos = resp['changepos'] @@ -318,11 +322,11 @@ def test_vs_bitcoind(match_key, check_against_bitcoind, bitcoind, start_sign, en print("Sending %.8f XTN to %s (Change back in position: %d)" % (amt, dest, chg_pos)) - psbt = b64decode(bitcoind.converttopsbt(txn2, True)) + psbt = b64decode(bitcoind.supply_wallet.converttopsbt(txn2, True)) # use walletcreatefundedpsbt # - updated/validated against 0.17.1 - resp = bitcoind.walletcreatefundedpsbt([], args, 0, { + resp = bitcoind.supply_wallet.walletcreatefundedpsbt([], args, 0, { 'subtractFeeFromOutputs': list(range(num_dests)), 'feeRate': 0.00001500}, True) @@ -370,7 +374,7 @@ def test_vs_bitcoind(match_key, check_against_bitcoind, bitcoind, start_sign, en assert b4 != aft, "signing didn't change anything?" open('debug/signed.psbt', 'wb').write(signed) - resp = bitcoind.finalizepsbt(str(b64encode(signed), 'ascii'), True) + resp = bitcoind.supply_wallet.finalizepsbt(str(b64encode(signed), 'ascii'), True) #combined_psbt = b64decode(resp['psbt']) #open('debug/combined.psbt', 'wb').write(combined_psbt) @@ -384,7 +388,7 @@ def test_vs_bitcoind(match_key, check_against_bitcoind, bitcoind, start_sign, en open('debug/finalized-by-btcd.txn', 'wb').write(network) # try to send it - txed = bitcoind.sendrawtransaction(B2A(network)) + txed = bitcoind.supply_wallet.sendrawtransaction(B2A(network)) print("Final txn hash: %r" % txed) else: @@ -392,7 +396,7 @@ def test_vs_bitcoind(match_key, check_against_bitcoind, bitcoind, start_sign, en #print("Final txn: %s" % B2A(signed)) open('debug/finalized-by-cc.txn', 'wb').write(signed) - txed = bitcoind.sendrawtransaction(B2A(signed)) + txed = bitcoind.supply_wallet.sendrawtransaction(B2A(signed)) print("Final txn hash: %r" % txed) def test_sign_example(set_master_key, sim_execfile, start_sign, end_sign): @@ -422,7 +426,7 @@ def test_sign_example(set_master_key, sim_execfile, start_sign, end_sign): @pytest.mark.bitcoind @pytest.mark.unfinalized -def test_sign_p2sh_p2wpkh(match_key, start_sign, end_sign, bitcoind): +def test_sign_p2sh_p2wpkh(match_key, use_regtest, start_sign, end_sign, bitcoind): # Check we can finalize p2sh_p2wpkh inputs right. # TODO fix this @@ -447,7 +451,7 @@ def test_sign_p2sh_p2wpkh(match_key, start_sign, end_sign, bitcoind): # use bitcoind to combine open('debug/signed.psbt', 'wb').write(signed_psbt) - resp = bitcoind.finalizepsbt(str(b64encode(signed_psbt), 'ascii'), True) + resp = bitcoind.rpc.finalizepsbt(str(b64encode(signed_psbt), 'ascii'), True) assert resp['complete'] == True, "bitcoind wasn't able to finalize it" network = a2b_hex(resp['hex']) @@ -458,7 +462,7 @@ def test_sign_p2sh_p2wpkh(match_key, start_sign, end_sign, bitcoind): @pytest.mark.bitcoind @pytest.mark.unfinalized -def test_sign_p2sh_example(set_master_key, sim_execfile, start_sign, end_sign, decode_psbt_with_bitcoind, offer_ms_import, need_keypress, clear_ms): +def test_sign_p2sh_example(set_master_key, use_regtest, sim_execfile, start_sign, end_sign, decode_psbt_with_bitcoind, offer_ms_import, need_keypress, clear_ms): # Use the private key given in BIP 174 and do similar signing # as the examples. @@ -537,7 +541,7 @@ def test_sign_p2sh_example(set_master_key, sim_execfile, start_sign, end_sign, d @pytest.mark.bitcoind -def test_change_case(start_sign, end_sign, check_against_bitcoind, cap_story): +def test_change_case(start_sign, use_regtest, end_sign, check_against_bitcoind, cap_story): # is change shown/hidden at right times. no fraud checks # NOTE: out#1 is change: @@ -581,7 +585,7 @@ def test_change_case(start_sign, end_sign, check_against_bitcoind, cap_story): @pytest.mark.parametrize('case', [ 1, 2]) @pytest.mark.bitcoind -def test_change_fraud_path(start_sign, end_sign, case, check_against_bitcoind, cap_story): +def test_change_fraud_path(start_sign, use_regtest, end_sign, case, check_against_bitcoind, cap_story): # fraud: BIP-32 path of output doesn't lead to pubkey indicated # NOTE: out#1 is change: @@ -625,7 +629,7 @@ def test_change_fraud_path(start_sign, end_sign, case, check_against_bitcoind, c signed = end_sign(True) @pytest.mark.bitcoind -def test_change_fraud_addr(start_sign, end_sign, check_against_bitcoind, cap_story): +def test_change_fraud_addr(start_sign, end_sign, use_regtest, check_against_bitcoind, cap_story): # fraud: BIP-32 path of output doesn't match TXO address from pycoin.tx.Tx import Tx from pycoin.tx.TxOut import TxOut @@ -659,11 +663,10 @@ def test_change_fraud_addr(start_sign, end_sign, check_against_bitcoind, cap_sto @pytest.mark.parametrize('case', [ 'p2wpkh', 'p2sh']) @pytest.mark.bitcoind -def test_change_p2sh_p2wpkh(start_sign, end_sign, check_against_bitcoind, cap_story, case): +def test_change_p2sh_p2wpkh(start_sign, end_sign, check_against_bitcoind, use_regtest, cap_story, case): # not fraud: output address encoded in various equiv forms from pycoin.tx.Tx import Tx - from pycoin.tx.TxOut import TxOut - + use_regtest() # NOTE: out#1 is change: #chg_addr = 'mvBGHpVtTyjmcfSsy6f715nbTGvwgbgbwo' @@ -678,7 +681,7 @@ def test_change_p2sh_p2wpkh(start_sign, end_sign, check_against_bitcoind, cap_st t.txs_out[1].script = bytes([0, 20]) + bytes(pkh) from bech32 import encode - expect_addr = encode('tb', 0, pkh) + expect_addr = encode('bcrt', 0, pkh) elif case == 'p2sh': @@ -916,12 +919,13 @@ def KEEP_test_random_psbt(try_sign, sim_exec, fname="data/ .psbt"): @pytest.mark.bitcoind @pytest.mark.unfinalized @pytest.mark.parametrize('num_dests', [ 1, 10, 25 ]) -def test_finalization_vs_bitcoind(match_key, check_against_bitcoind, bitcoind, start_sign, end_sign, num_dests): +def test_finalization_vs_bitcoind(match_key, use_regtest, check_against_bitcoind, bitcoind, start_sign, end_sign, num_dests): # Compare how we finalize vs bitcoind ... should be exactly the same txn - wallet_xfp = match_key() + # has to be after match key + use_regtest() - bal = bitcoind.getbalance() + bal = bitcoind.supply_wallet.getbalance() assert bal > 0, "need some play money; drink from a faucet" amt = round((bal/4)/num_dests, 6) @@ -929,14 +933,14 @@ def test_finalization_vs_bitcoind(match_key, check_against_bitcoind, bitcoind, s args = {} for no in range(num_dests): - dest = bitcoind.getrawchangeaddress() - assert dest[0] in '2mn' or dest.startswith('tb1'), dest + dest = bitcoind.supply_wallet.getrawchangeaddress() + assert dest.startswith('bcrt1q'), dest args[dest] = amt # use walletcreatefundedpsbt # - updated/validated against 0.17.1 - resp = bitcoind.walletcreatefundedpsbt([], args, 0, { + resp = bitcoind.supply_wallet.walletcreatefundedpsbt([], args, 0, { 'subtractFeeFromOutputs': list(range(num_dests)), 'feeRate': 0.00001500}, True) @@ -958,9 +962,7 @@ def test_finalization_vs_bitcoind(match_key, check_against_bitcoind, bitcoind, s # pull out included txn txn2 = B2A(mine.txn) - start_sign(psbt, finalize=True) - # verify against how bitcoind reads it check_against_bitcoind(txn2, fee) @@ -975,7 +977,7 @@ def test_finalization_vs_bitcoind(match_key, check_against_bitcoind, bitcoind, s open('debug/vs-signed-unfin.psbt', 'wb').write(signed) # Use bitcoind to finalize it this time. - resp = bitcoind.finalizepsbt(str(b64encode(signed), 'ascii'), True) + resp = bitcoind.supply_wallet.finalizepsbt(str(b64encode(signed), 'ascii'), True) assert resp['complete'] == True, "bitcoind wasn't able to finalize it" network = a2b_hex(resp['hex']) @@ -987,7 +989,7 @@ def test_finalization_vs_bitcoind(match_key, check_against_bitcoind, bitcoind, s assert network == signed_final, "Finalized differently" # try to send it - txed = bitcoind.sendrawtransaction(B2A(network)) + txed = bitcoind.supply_wallet.sendrawtransaction(B2A(network)) print("Final txn hash: %r" % txed) From 8c36b6025a6c99e0a6ac334956efba771aabc04a Mon Sep 17 00:00:00 2001 From: avirgovi Date: Mon, 23 May 2022 16:02:31 +0200 Subject: [PATCH 21/29] remove explora --- testing/api.py | 28 ++-------------------------- testing/conftest.py | 2 +- testing/test_multisig.py | 2 +- 3 files changed, 4 insertions(+), 28 deletions(-) diff --git a/testing/api.py b/testing/api.py index ce2cdc10..00d7fb30 100644 --- a/testing/api.py +++ b/testing/api.py @@ -1,13 +1,6 @@ # (c) Copyright 2020 by Coinkite Inc. This file is covered by license found in COPYING-CC. # -# Access a local bitcoin-Qt/bitcoind on testnet (must be v22 or higher) -# -# Must have these lines in the bitcoin.conf file: -# -# testnet=1 -# server=1 -# rpcservertimeout=2000 # for test_sign.py::test_io_size -# +# needs local bitcoind in PATH import os import time @@ -20,16 +13,12 @@ import tempfile import subprocess from authproxy import AuthServiceProxy, JSONRPCException from base64 import b64encode, b64decode -from constants import simulator_fixed_words - -URL = '127.0.0.1:18332/wallet/' # stolen from HWI test suite and slightly modified class Bitcoind: - def __init__(self, bitcoind_path, signer="/home/more/PycharmProjects/HWI/venv/lib/python3.8/site-packages/hwi.py"): + def __init__(self, bitcoind_path): self.bitcoind_path = bitcoind_path - self.signer = signer self.datadir = tempfile.mkdtemp() self.rpc = None self.bitcoind_proc = None @@ -53,7 +42,6 @@ class Bitcoind: [ self.bitcoind_path, "-regtest", - f"-signer={self.signer}", f"-datadir={self.datadir}", "-noprinttoconsole", "-fallbackfee=0.0002", @@ -192,18 +180,6 @@ def bitcoind_decode(bitcoind): return doit -@pytest.fixture -def explora(): - def doit(*parts): - import urllib.request - import json - url = 'https://blockstream.info/testnet/api/' + '/'.join(parts) - with urllib.request.urlopen(url) as response: - return json.load(response) - - return doit - - @pytest.fixture def bitcoind_wallet(bitcoind): # Use bitcoind to create a temporary wallet file diff --git a/testing/conftest.py b/testing/conftest.py index b5f67596..fd0815cd 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -3,7 +3,7 @@ import pytest, time, sys, random, re, ndef from ckcc.protocol import CCProtocolPacker from helpers import B2A, U2SAT, prandom -from api import bitcoind, match_key, bitcoind_finalizer, bitcoind_analyze, bitcoind_decode, explora +from api import bitcoind, match_key, bitcoind_finalizer, bitcoind_analyze, bitcoind_decode from api import bitcoind_wallet, bitcoind_d_wallet, bitcoind_d_wallet_w_sk, bitcoind_d_sim from binascii import b2a_hex, a2b_hex from constants import * diff --git a/testing/test_multisig.py b/testing/test_multisig.py index 314eb0ac..ae2ab57a 100644 --- a/testing/test_multisig.py +++ b/testing/test_multisig.py @@ -1448,7 +1448,7 @@ def test_make_airgapped(addr_fmt, acct_num, goto_home, cap_story, pick_menu_item @pytest.mark.unfinalized @pytest.mark.bitcoind @pytest.mark.parametrize('addr_style', ["legacy", "p2sh-segwit", "bech32"]) -def test_bitcoind_cosigning(dev, bitcoind, import_ms_wallet, clear_ms, explora, try_sign, need_keypress, addr_style, use_regtest): +def test_bitcoind_cosigning(dev, bitcoind, import_ms_wallet, clear_ms, try_sign, need_keypress, addr_style, use_regtest): # Make a P2SH wallet with local bitcoind as a co-signer (and simulator) # - send an receive various # - following text of From 81634694c2cf4af87a91901291752a9075cc9d70 Mon Sep 17 00:00:00 2001 From: avirgovi Date: Tue, 24 May 2022 00:20:01 +0200 Subject: [PATCH 22/29] better font for ubuntu20; doc typo; regtest option for make ms address --- shared/multisig.py | 2 +- testing/conftest.py | 6 +++++- testing/test_multisig.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/shared/multisig.py b/shared/multisig.py index 04bd89ca..885adebc 100644 --- a/shared/multisig.py +++ b/shared/multisig.py @@ -259,7 +259,7 @@ class MultisigWallet: @classmethod def find_candidates(cls, xfp_paths, addr_fmt=None, M=None): # Return a list of matching wallets for various M values. - # - xpfs_paths hsould already be sorted + # - xpfs_paths should already be sorted # - returns set of matches, of any M value # we know N, but not M at this point. diff --git a/testing/conftest.py b/testing/conftest.py index fd0815cd..dd217feb 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -420,7 +420,11 @@ def qr_quality_check(): try: fnt = ImageFont.truetype('Courier', size=10) except: - fnt = ImageFont.load_default() + try: + fnt = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', size=10) + except: + fnt = ImageFont.load_default() + dr = ImageDraw.Draw(rv) mw = int((w*scale) / dr.textsize('M', fnt)[0]) diff --git a/testing/test_multisig.py b/testing/test_multisig.py index ae2ab57a..603de51d 100644 --- a/testing/test_multisig.py +++ b/testing/test_multisig.py @@ -343,7 +343,8 @@ def make_ms_address(M, keys, idx=0, is_change=0, addr_fmt=AF_P2SH, testnet=1, ** script, pubkeys, xfp_paths = make_redeem(M, keys, **make_redeem_args) if addr_fmt == AF_P2WSH: - hrp = ['bc', 'tb'][testnet] + # testnet=2 --> regtest + hrp = ['bc', 'tb', 'bcrt'][testnet] data = sha256(script).digest() addr = bech32.encode(hrp, 0, data) scriptPubKey = bytes([0x0, 0x20]) + data From 2e564978ddf4a2499737eeae02e66dedf20ef31d Mon Sep 17 00:00:00 2001 From: avirgovi Date: Tue, 24 May 2022 10:37:29 +0200 Subject: [PATCH 23/29] fix BasicPSBT parsing --- testing/psbt.py | 9 +++++---- testing/run_sim_tests.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/testing/psbt.py b/testing/psbt.py index fa5de250..831d88cb 100644 --- a/testing/psbt.py +++ b/testing/psbt.py @@ -231,7 +231,6 @@ class BasicPSBT: if raw[0:6] == b'cHNidP': raw = b64decode(raw) assert raw[0:5] == b'psbt\xff', "bad magic {}".format(raw[0:5]) - with io.BytesIO(raw[5:]) as fd: # globals @@ -254,7 +253,8 @@ class BasicPSBT: num_outs = len(t.txs_out) elif kt == PSBT_GLOBAL_XPUB: # key=(xpub) => val=(path) - self.xpubs.append( (key, val) ) + # ignore PSBT_GLOBAL_XPUB on 0th index (should not be part of parsed key) + self.xpubs.append((key[1:], val)) else: raise ValueError('unknown global key type: 0x%02x' % kt) @@ -271,8 +271,9 @@ class BasicPSBT: def serialize(self, fd): def wr(ktype, val, key=b''): - fd.write(ser_compact_size(1 + len(key))) - fd.write(bytes([ktype]) + key) + ktype_plus_key = bytes([ktype]) + key + fd.write(ser_compact_size(len(ktype_plus_key))) + fd.write(ktype_plus_key) fd.write(ser_compact_size(len(val))) fd.write(val) diff --git a/testing/run_sim_tests.py b/testing/run_sim_tests.py index 8871b95e..3c654f71 100644 --- a/testing/run_sim_tests.py +++ b/testing/run_sim_tests.py @@ -10,8 +10,8 @@ python run_sim_tests.py --onetime # run ONLY onetim python run_sim_tests.py --onetime --veryslow # run both onetime and very slow python run_sim_tests.py -m test_nfc.py # run only nfc tests python run_sim_tests.py -m test_nfc.py -m test_hsm.py # run nfc and hsm tests -python run_sim_tests.py -m all # run all tests but not onetime and not very slow -python run_sim_tests.py -m all --onetime --veryslow # run all test (most useful - grab coffee and wait) +python run_sim_tests.py -m all # run all tests but not onetime and not very slow (cca 90 minutes) +python run_sim_tests.py -m all --onetime --veryslow # run all tests (cca 235 minutes) Onetime/veryslow tests are completely separated form the rest of the test suite. From 149980ada84bb161d12ff794ef670d6ab5d71223 Mon Sep 17 00:00:00 2001 From: avirgovi Date: Tue, 24 May 2022 14:23:16 +0200 Subject: [PATCH 24/29] add comment to failing test --- testing/test_multisig.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/testing/test_multisig.py b/testing/test_multisig.py index 603de51d..59303821 100644 --- a/testing/test_multisig.py +++ b/testing/test_multisig.py @@ -1250,12 +1250,11 @@ def test_ms_sign_myself(M, use_regtest, make_myself_wallet, segwit, num_ins, dev open(f'debug/myself-after.psbt', 'w').write(base64.b64encode(updated).decode()) assert updated != psbt - aft = BasicPSBT().parse(updated) # TODO something is off here xpub is longer than 79 - core returs error - + aft = BasicPSBT().parse(updated) # check all inputs gained a signature assert all(len(i.part_sigs)==(idx+1) for i in aft.inputs) - psbt = updated + psbt = aft.as_bytes() # should be fully signed now. anal = bitcoind_analyze(psbt) @@ -1275,12 +1274,18 @@ def test_ms_sign_myself(M, use_regtest, make_myself_wallet, segwit, num_ins, dev print("ignoring bug in bitcoind") if 0: - # why doesn't this work? # TODO produced PSBT is invalid, cannot finalize (both core and us) - extracted_psbt, txn, is_complete = bitcoind_finalizer(aft.as_bytes(), extract=True) - - ex = BasicPSBT().parse(extracted_psbt) + # why doesn't this work? + # TODO this does NOT work only if parameter segwit is True + # TODO I have debuged bitcoin core to see why we're still in updater phase, not in desired finalizer + # relevant comment from core code: + # When we're taking our information from a witness UTXO, we can't verify it is actually data from + # the output being spent. This is safe in case a witness signature is produced (which includes this + # information directly in the hash), but not for non-witness signatures. Remember that we require + # a witness signature in this situation. + # + # In our case, witness signature was not produced (but was required) + _, txn, is_complete = bitcoind_finalizer(aft.as_bytes(), extract=True) assert is_complete - assert ex != aft @pytest.mark.parametrize('addr_fmt', ['p2wsh', 'p2sh-p2wsh']) @pytest.mark.parametrize('acct_num', [ 0, 99, 4321]) From 41bc984656aa0e3c7935c5c2201b87698280b2f5 Mon Sep 17 00:00:00 2001 From: avirgovi Date: Tue, 24 May 2022 22:50:04 +0200 Subject: [PATCH 25/29] review comments; importstyle; dissolve complex expression; ad test_ms_sign_simple to veryslow tests --- shared/multisig.py | 8 ++++++-- testing/api.py | 10 +--------- testing/pytest.ini | 2 +- testing/test_multisig.py | 3 ++- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/shared/multisig.py b/shared/multisig.py index 885adebc..ff755d0c 100644 --- a/shared/multisig.py +++ b/shared/multisig.py @@ -712,8 +712,12 @@ class MultisigWallet: assert node.privkey() == None # 'no privkeys plz' except ValueError: pass - # HACK but there is no difference extended_keys - just bech32 hrp - assert chain.ctype == expect_chain or expect_chain == "XRT" and chain.ctype == "XTN" # 'wrong chain' + + if expect_chain == "XRT": + # HACK but there is no difference extended_keys - just bech32 hrp + assert chain.ctype == "XTN" + else: + assert chain.ctype == expect_chain # 'wrong chain' depth = node.depth() diff --git a/testing/api.py b/testing/api.py index 00d7fb30..6f6646df 100644 --- a/testing/api.py +++ b/testing/api.py @@ -2,15 +2,7 @@ # # needs local bitcoind in PATH -import os -import time -import uuid -import atexit -import socket -import shutil -import pytest -import tempfile -import subprocess +import os, time, uuid, atexit, socket, shutil, pytest, tempfile, subprocess from authproxy import AuthServiceProxy, JSONRPCException from base64 import b64encode, b64decode diff --git a/testing/pytest.ini b/testing/pytest.ini index b7d654e3..5c80ecdd 100644 --- a/testing/pytest.ini +++ b/testing/pytest.ini @@ -8,7 +8,7 @@ markers = veryslow: test takes more than 30 minutes realtime qrcode: test uses or tests QR related features unfinalized: test cases produces an unfinalized PSBT - manual: test cannot be combined with nay others, check for "fully done" in repl (then it will hang - kill it) + manual: test cannot be combined with any others, check for "fully done" in repl (then it will hang - kill it) # DOES NOT WORK. see --disable-warnings instead filterwarnings = diff --git a/testing/test_multisig.py b/testing/test_multisig.py index 59303821..d9a1d498 100644 --- a/testing/test_multisig.py +++ b/testing/test_multisig.py @@ -1190,6 +1190,7 @@ def fake_ms_txn(): return doit +@pytest.mark.veryslow @pytest.mark.unfinalized @pytest.mark.parametrize('addr_fmt', [AF_P2SH, AF_P2WSH, AF_P2WSH_P2SH] ) @pytest.mark.parametrize('num_ins', [ 2, 15 ]) @@ -1726,7 +1727,7 @@ def test_iss6743(repeat, set_seed_words, sim_execfile, try_sign): tp = BasicPSBT().parse(psbt_b4) (hdr_xpub, hdr_path), = [(v,k) for v,k in tp.xpubs if k[0:4] == pack(' Date: Tue, 24 May 2022 22:57:00 +0200 Subject: [PATCH 26/29] remove ndef dependency -> ndeflib is part of nfcpy --- testing/requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/testing/requirements.txt b/testing/requirements.txt index fc7c5710..93b933a4 100644 --- a/testing/requirements.txt +++ b/testing/requirements.txt @@ -12,8 +12,6 @@ zbar-py==1.0.4 # NFC and NDEF handling nfcpy==1.0.3 -# commentt out below dependency 'ndef==0.2' if on debian based OS -#ndef==0.2 # optional, and only helpful if you have a desktop NFC-V capable reader pyscard==2.0.2 From dbffe10474de520083cdbee3ff8ed5aca767100f Mon Sep 17 00:00:00 2001 From: avirgovi Date: Thu, 26 May 2022 07:55:13 +0200 Subject: [PATCH 27/29] better comments --- testing/run_sim_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/run_sim_tests.py b/testing/run_sim_tests.py index 3c654f71..5a37f133 100644 --- a/testing/run_sim_tests.py +++ b/testing/run_sim_tests.py @@ -1,6 +1,6 @@ """ Run conveniently tests against simulator. Tests are run module after module. If any tests fail, -it will try to re-run those failed test with fresh simulator. has to be run from firmware/testing directory. +it will try to re-run those failed test with fresh simulator. Has to be run from firmware/testing directory. Do not forget to comment/uncomment line in pytest.ini. . ENV/bin/activate @@ -10,7 +10,7 @@ python run_sim_tests.py --onetime # run ONLY onetim python run_sim_tests.py --onetime --veryslow # run both onetime and very slow python run_sim_tests.py -m test_nfc.py # run only nfc tests python run_sim_tests.py -m test_nfc.py -m test_hsm.py # run nfc and hsm tests -python run_sim_tests.py -m all # run all tests but not onetime and not very slow (cca 90 minutes) +python run_sim_tests.py -m all # run all tests but not onetime and not very slow (cca 40 minutes) - most useful python run_sim_tests.py -m all --onetime --veryslow # run all tests (cca 235 minutes) From 3a86f6cc88b18161ecc829e13de5f9e35b2001ba Mon Sep 17 00:00:00 2001 From: avirgovi Date: Thu, 26 May 2022 16:27:31 +0200 Subject: [PATCH 28/29] merge imports --- testing/run_sim_tests.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/testing/run_sim_tests.py b/testing/run_sim_tests.py index 5a37f133..d8edba6c 100644 --- a/testing/run_sim_tests.py +++ b/testing/run_sim_tests.py @@ -25,16 +25,7 @@ python run_sim_tests.py --collect manual # just print all Make sure to run manual test if you want to state that your changes passed all the tests. """ -import os -import time -import glob -import json -import pytest -import atexit -import signal -import argparse -import subprocess -import contextlib +import os, time, glob, json, pytest, atexit, signal, argparse, subprocess, contextlib from typing import List from pytest import ExitCode From deabb6ea1e910e9f5aca214fca3e2122e46cc129 Mon Sep 17 00:00:00 2001 From: avirgovi Date: Thu, 26 May 2022 16:46:22 +0200 Subject: [PATCH 29/29] add COPYRIGHT --- testing/run_sim_tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testing/run_sim_tests.py b/testing/run_sim_tests.py index d8edba6c..0ba6b153 100644 --- a/testing/run_sim_tests.py +++ b/testing/run_sim_tests.py @@ -1,3 +1,5 @@ +# (c) Copyright 2022 by Coinkite Inc. This file is covered by license found in COPYING-CC. + """ Run conveniently tests against simulator. Tests are run module after module. If any tests fail, it will try to re-run those failed test with fresh simulator. Has to be run from firmware/testing directory.