diff --git a/testing/login_settings_tests.py b/testing/login_settings_tests.py index 71d6d890..ff915596 100644 --- a/testing/login_settings_tests.py +++ b/testing/login_settings_tests.py @@ -1,10 +1,15 @@ # (c) Copyright 2024 by Coinkite Inc. This file is covered by license found in COPYING-CC. # # to run it on both Mk4 and Q: -# python login_settings_tests.py; sleep 10; python --Q login_settings_tests.py +# pytest login_settings_tests.py; sleep 10; pytest --Q login_settings_tests.py +# +# or use test runner: +# python run_sim_tests --login +# +# python run_sim_tests --q1 --login -k countdown --pdb # import pytest, time, pdb -from charcodes import KEY_ENTER, KEY_DOWN, KEY_UP, KEY_HOME, KEY_DELETE +from charcodes import KEY_ENTER, KEY_DOWN, KEY_UP, KEY_HOME from ckcc_protocol.client import ColdcardDevice, CCProtocolPacker, CKCC_SIMULATOR_PATH from run_sim_tests import ColdcardSimulator, clean_sim_data @@ -215,6 +220,7 @@ def _login(device, pin, is_Q, scrambled=False, mk4_kbtn=None): _need_keypress(device, ch) _press_select(device, is_Q) + @pytest.mark.parametrize("nick", [100*"$", "$", 10*"20"+ " "+"8080"+ " " + "XX"+ " "+ "YY"]) def test_set_nickname(nick, request): is_Q = request.config.getoption('--Q') @@ -244,6 +250,7 @@ def test_set_nickname(nick, request): assert nick == target sim.stop() + def test_randomize_pin_keys(request): is_Q = request.config.getoption('--Q') clean_sim_data() # remove all from previous diff --git a/testing/run_sim_tests.py b/testing/run_sim_tests.py index f77ba04c..1471a51d 100644 --- a/testing/run_sim_tests.py +++ b/testing/run_sim_tests.py @@ -17,6 +17,7 @@ python run_sim_tests.py # same as with '- python run_sim_tests.py -m all --onetime --veryslow # run all tests (cca 252 minutes) python run_sim_tests.py -m test_multisig.py -k cosigning # run only tests that match expression from test_multisig.py python run_sim_tests.py -m test_export.py --pdb # run only export tests and attach debugger +python run_sim_tests.py -m test_attended.py --q1 -w 6 --login # run attended test + all login tests Onetime/veryslow tests are completely separated form the rest of the test suite. @@ -96,13 +97,11 @@ def is_ok(ec: ExitCode) -> bool: return False -def _run_tests_with_simulator(test_module: str, simulator_args: List[str], pytest_marks: str, - pytest_k: str, pdb: bool, failed_first: bool, psbt2=False) -> ExitCode: - sim = ColdcardSimulator(args=simulator_args) - sim.start() - time.sleep(1) +def _run_pytest_tests(test_module: str, pytest_marks: str, pytest_k: str, pdb: bool, + failed_first: bool, psbt2=False, is_Q=False) -> ExitCode: cmd_list = [ - "--cache-clear", "-m", pytest_marks, "--sim", test_module if test_module is not None else "" + "--cache-clear", "-m", pytest_marks, "--sim", + test_module if test_module is not None else "" ] if pytest_k: cmd_list += ["-k", pytest_k] @@ -112,28 +111,43 @@ def _run_tests_with_simulator(test_module: str, simulator_args: List[str], pytes cmd_list.append("--ff") if psbt2: cmd_list.append("--psbt2") + if is_Q: + cmd_list.insert(0, "--Q") # only changes behavior in login_settings_test - exit_code = pytest.main(cmd_list) - sim.stop() - time.sleep(1) - clean_sim_data() + return pytest.main(cmd_list) + +def _run_coldcard_tests(test_module: str, simulator_args: List[str], pytest_marks: str, + pytest_k: str, pdb: bool, failed_first: bool, psbt2=False, + is_Q=False) -> ExitCode: + if simulator_args: + sim = ColdcardSimulator(args=simulator_args) + sim.start() + time.sleep(1) + + exit_code = _run_pytest_tests(test_module, pytest_marks, pytest_k, pdb, + failed_first, psbt2, is_Q) + + if simulator_args: + sim.stop() + time.sleep(1) + clean_sim_data() return exit_code -def run_tests_with_simulator(test_module=None, simulator_args=None, pytest_k=None, pdb=False, - failed_first=False, psbt2=False, - pytest_marks="not onetime and not veryslow and not manual"): +def run_coldcard_tests(test_module=None, simulator_args=None, pytest_k=None, pdb=False, + failed_first=False, psbt2=False, is_Q=False, + pytest_marks="not onetime and not veryslow and not manual"): failed = [] - exit_code = _run_tests_with_simulator(test_module, simulator_args, pytest_marks, pytest_k, - pdb, failed_first, psbt2=psbt2) + exit_code = _run_coldcard_tests(test_module, simulator_args, pytest_marks, pytest_k, + pdb, failed_first, psbt2, is_Q) if not is_ok(exit_code): # no success, no nothing - give failed another try, each alone with its own simulator last_failed = get_last_failed() print("Running failed from last run", last_failed) exit_codes = [] for failed_test in last_failed: - exit_code_2 = _run_tests_with_simulator(failed_test, simulator_args, pytest_marks, - pytest_k, pdb, failed_first, psbt2=psbt2) + exit_code_2 = _run_coldcard_tests(failed_test, simulator_args, pytest_marks, + pytest_k, pdb, failed_first, psbt2, is_Q) exit_codes.append(exit_code_2) if not is_ok(exit_code_2): failed.append(failed_test) @@ -200,6 +214,8 @@ def main(): parser.add_argument("--onetime", action="store_true", default=False, help="run tests marked as 'onetime'") parser.add_argument("--veryslow", action="store_true", default=False, + help="run 'login_settings_tests.py'") + parser.add_argument("--login", action="store_true", default=False, help="run tests marked as 'veryslow'") parser.add_argument("--collect", type=str, metavar="MARK", help="Collect marked test and print them to stdout") @@ -216,7 +232,7 @@ def main(): print(collect_marked_tests(args.collect)) return - if args.module is None and (args.onetime is False and args.veryslow is False): + if args.module is None and (args.onetime is False and args.veryslow is False and args.login is False): args.module = ["all"] DEFAULT_SIMULATOR_ARGS = ["--eff", "--set", "nfc=1"] @@ -262,9 +278,9 @@ def main(): if args.q1 and '--q1' not in test_args: test_args.append('--q1') - ec, failed_tests = run_tests_with_simulator(test_module, simulator_args=test_args, - pytest_k=args.pytest_k, pdb=args.pdb, - failed_first=args.ff, psbt2=args.psbt2) + ec, failed_tests = run_coldcard_tests(test_module, simulator_args=test_args, + pytest_k=args.pytest_k, pdb=args.pdb, + failed_first=args.ff, psbt2=args.psbt2) result.append((test_module, ec, failed_tests)) print("Done", test_module) print(80 * "=") @@ -272,10 +288,10 @@ def main(): # run veryslow is specified if args.veryslow: print("started veryslow tests") - ec, failed_tests = run_tests_with_simulator(test_module=None, pytest_marks="veryslow", - pytest_k=args.pytest_k, pdb=args.pdb, - simulator_args=DEFAULT_SIMULATOR_ARGS, - failed_first=args.ff, psbt2=args.psbt2) + ec, failed_tests = run_coldcard_tests(test_module=None, pytest_marks="veryslow", + pytest_k=args.pytest_k, pdb=args.pdb, + simulator_args=DEFAULT_SIMULATOR_ARGS, + failed_first=args.ff, psbt2=args.psbt2) result.append(("veryslow", ec, failed_tests)) # run onetime is specified (each test against its own simulator) @@ -283,12 +299,19 @@ def main(): print("started onetime tests") onetime_tests = collect_marked_tests("onetime") for onetime_test in onetime_tests: - ec, failed_tests = run_tests_with_simulator(test_module=onetime_test, pdb=args.pdb, - failed_first=args.ff, pytest_marks="onetime", - simulator_args=DEFAULT_SIMULATOR_ARGS, - psbt2=args.psbt2) + ec, failed_tests = run_coldcard_tests(test_module=onetime_test, pdb=args.pdb, + failed_first=args.ff, pytest_marks="onetime", + simulator_args=DEFAULT_SIMULATOR_ARGS, + psbt2=args.psbt2) result.append((f"onetime: {onetime_test}", ec, failed_tests)) + if args.login: + print("start login settings tests") + ec, failed_tests = run_coldcard_tests(test_module="login_settings_tests.py", pdb=args.pdb, + failed_first=args.ff, pytest_k=args.pytest_k, + is_Q=True if args.q1 else False) + result.append((f"login_settings_tests", ec, failed_tests)) + print("All done") any_failed = False