From ef0ec76dabdf59314b30eba58a0375d0f42bc5f9 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho Date: Wed, 18 Nov 2026 11:44:44 -0300 Subject: [PATCH 1/5] Fix: run-tests rejects unknown options before the -- separator run-tests accepted any unknown option and forwarded it to the Python test runner as an extra test parameter, so a typo such as --prompt_timeout (instead of --prompt-timeout) only failed inside the SDK container, with the error visible only in the backend logs. Parse options before a literal -- strictly, so unknown options fail with "No such option" and a suggestion. Arguments after -- are still forwarded as test parameters. Closes #1215 --- tests/test_run_tests.py | 40 ++++++++++++++++++++++++++++++++++++ th_cli/commands/run_tests.py | 22 +++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/tests/test_run_tests.py b/tests/test_run_tests.py index e1dc7d9..7cfa4b2 100644 --- a/tests/test_run_tests.py +++ b/tests/test_run_tests.py @@ -20,6 +20,7 @@ from pathlib import Path from unittest.mock import AsyncMock, Mock, patch +import click import pytest from click.testing import CliRunner @@ -484,6 +485,45 @@ def test_run_tests_required_tests_list_parameter(self, cli_runner: CliRunner) -> assert result.exit_code != 0 assert "required" in result.output + def test_run_tests_rejects_unknown_option_before_separator(self, cli_runner: CliRunner) -> None: + """A typo like --prompt_timeout must fail up front instead of reaching the test runner.""" + # Act + result = cli_runner.invoke(run_tests, ["--tests-list", "TC-ACE-1.1", "--prompt_timeout", "5"]) + + # Assert + assert result.exit_code == 2 + assert "No such option '--prompt_timeout'" in result.output + assert "Did you mean '--prompt-timeout'?" in result.output + + def test_run_tests_rejects_unexpected_positional_argument(self, cli_runner: CliRunner) -> None: + """Stray positional arguments before -- are errors, not silently ignored.""" + # Act + result = cli_runner.invoke(run_tests, ["--tests-list", "TC-ACE-1.1", "stray"]) + + # Assert + assert result.exit_code == 2 + assert "unexpected extra argument" in result.output.lower() + + def test_run_tests_forwards_args_after_separator(self, cli_runner: CliRunner) -> None: + """Unknown options after -- are still captured as test parameters.""" + # Arrange + captured: dict[str, list[str]] = {} + + def fake_parse(args: list[str]) -> dict[str, str]: + captured["args"] = args + raise click.ClickException("stop after parsing") + + # Act + with patch("th_cli.commands.run_tests._parse_extra_args", side_effect=fake_parse): + result = cli_runner.invoke( + run_tests, + ["--tests-list", "TC-ACE-1.1", "--", "--int-arg", "PIXIT.X:1", "--prompt_timeout", "5"], + ) + + # Assert + assert captured["args"] == ["--int-arg", "PIXIT.X:1", "--prompt_timeout", "5"] + assert "stop after parsing" in result.output + @pytest.mark.parametrize( "test_list", [ diff --git a/th_cli/commands/run_tests.py b/th_cli/commands/run_tests.py index 1e40de4..cc0bf1c 100644 --- a/th_cli/commands/run_tests.py +++ b/th_cli/commands/run_tests.py @@ -87,11 +87,31 @@ def _complete_test_ids(ctx: click.Context, param: click.Parameter, incomplete: s return [] +class _RunTestsCommand(click.Command): + """Command that only treats arguments after a literal ``--`` as test parameters. + + Options before ``--`` are parsed strictly, so a typo such as ``--prompt_timeout`` + fails with "No such option" (plus a suggestion) instead of being forwarded to the + test runner as an extra test parameter. Everything after ``--`` is kept untouched + in ``ctx.args`` for ``_parse_extra_args``. + """ + + def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]: + if "--" in args: + separator = args.index("--") + cli_args, inline_args = args[:separator], args[separator + 1 :] + else: + cli_args, inline_args = args, [] + remaining = super().parse_args(ctx, cli_args) + ctx.args = inline_args + return remaining + + @click.command( + cls=_RunTestsCommand, no_args_is_help=True, short_help=colorize_help("CLI execution of a test run"), help=colorize_cmd_help("run_tests", "CLI execution of a test run from selected tests"), - context_settings={"ignore_unknown_options": True, "allow_extra_args": True}, ) @click.option( "--tests-list", From 829500c7abf7d3a08743379dd5a2fdb63c61ee7e Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho Date: Wed, 18 Nov 2026 20:08:56 -0300 Subject: [PATCH 2/5] Test: accept both Click wordings for the unknown-option error --- tests/test_run_tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_run_tests.py b/tests/test_run_tests.py index 7cfa4b2..b3e9195 100644 --- a/tests/test_run_tests.py +++ b/tests/test_run_tests.py @@ -492,8 +492,9 @@ def test_run_tests_rejects_unknown_option_before_separator(self, cli_runner: Cli # Assert assert result.exit_code == 2 - assert "No such option '--prompt_timeout'" in result.output - assert "Did you mean '--prompt-timeout'?" in result.output + # Wording differs across Click versions (quoted vs. unquoted option names) + assert re.search(r"No such option:? '?--prompt_timeout'?", result.output) + assert re.search(r"Did you mean '?--prompt-timeout'?\?", result.output) def test_run_tests_rejects_unexpected_positional_argument(self, cli_runner: CliRunner) -> None: """Stray positional arguments before -- are errors, not silently ignored.""" From 7d32b51f5dc9486d735629b8194b08142d33b7bf Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho Date: Wed, 18 Nov 2026 20:27:18 -0300 Subject: [PATCH 3/5] CLI: show suite-level errors reported by the backend Suite updates carry an errors list (e.g. a failed suite setup), but the CLI only printed the suite title and state, so the reason was visible only in the backend log. Print each suite error once. Related to #1215 --- tests/test_run/test_websocket_additional.py | 21 +++++++++++++++++++++ th_cli/test_run/websocket.py | 11 +++++++++++ 2 files changed, 32 insertions(+) diff --git a/tests/test_run/test_websocket_additional.py b/tests/test_run/test_websocket_additional.py index b45335f..86d1dff 100644 --- a/tests/test_run/test_websocket_additional.py +++ b/tests/test_run/test_websocket_additional.py @@ -527,6 +527,27 @@ async def test_routes_suite_update(self): await s._TestRunSocket__handle_test_update(update=update) mock_fn.assert_called_once_with(body) + def test_suite_update_prints_suite_errors_once(self, capsys): + """Suite errors (e.g. failed setup) are shown, and not repeated on later updates.""" + s = _make_socket(suites=[_make_suite()]) + error = "Error occurred during setup of test suite. Invalid test parameter(s): '--prompt_timeout'" + body = TestSuiteUpdate(state="error", test_suite_execution_index=0, errors=[error]) + + with patch("th_cli.test_run.websocket.get_log_stream_handler", return_value=None): + s._TestRunSocket__log_test_suite_update(body) + s._TestRunSocket__log_test_suite_update(body) + + assert capsys.readouterr().err.count(error) == 1 + + def test_suite_update_without_errors_prints_no_error(self, capsys): + s = _make_socket(suites=[_make_suite()]) + body = TestSuiteUpdate(state="executing", test_suite_execution_index=0) + + with patch("th_cli.test_run.websocket.get_log_stream_handler", return_value=None): + s._TestRunSocket__log_test_suite_update(body) + + assert capsys.readouterr().err == "" + @pytest.mark.asyncio async def test_routes_run_update_and_closes_socket_when_not_executing(self): s = _make_socket() diff --git a/th_cli/test_run/websocket.py b/th_cli/test_run/websocket.py index 3bd988e..6fb5e07 100644 --- a/th_cli/test_run/websocket.py +++ b/th_cli/test_run/websocket.py @@ -142,6 +142,9 @@ def __init__( # Track test step errors for logging # Key: (suite_index, case_index), Value: list of error strings from all steps self.test_case_step_errors: dict[tuple[int, int], list[str]] = {} + # Suite-level errors (e.g. a failed suite setup) already shown to the user. + # Key: (suite_index, error), so repeated updates don't print them again. + self._reported_suite_errors: set[tuple[int, str]] = set() # Track the final state of each test case for the end-of-run summary. # Key: (suite_index, case_index), Value: final TestStateEnum value. # A dict (rather than a running counter) so a case that is updated more @@ -398,6 +401,14 @@ def __log_test_suite_update(self, update: TestSuiteUpdate) -> None: colored_state = colorize_state(update.state.value) click.echo(f" - {colored_title} {colored_state}") + # Suite-level errors (e.g. a failed suite setup) would otherwise only be in + # the backend log. Show each one once. + for error in update.errors or []: + key = (update.test_suite_execution_index, error) + if key not in self._reported_suite_errors: + self._reported_suite_errors.add(key) + click.echo(colorize_error(f" {error}"), err=True) + handler = get_log_stream_handler() if handler: handler.update_tree_node( From b8b597790a9b146e2fcf72462b2b2b6ecd7e595e Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho Date: Wed, 18 Nov 2026 21:20:53 -0300 Subject: [PATCH 4/5] CLI: say where a run-tests option goes when it is placed after -- Everything after -- is passed to the test runner, so an option of run-tests itself there (e.g. -- --prompt-timeout 5, or the --prompt_timeout typo) was either rejected by the runner deep inside the run or silently ignored. Fail up front and tell the user to pass it before the separator. The option names come from the command definition, so they cannot drift. Related to #1215 --- tests/test_run_tests.py | 15 +++++++++++++-- th_cli/commands/run_tests.py | 21 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/tests/test_run_tests.py b/tests/test_run_tests.py index b3e9195..f19e245 100644 --- a/tests/test_run_tests.py +++ b/tests/test_run_tests.py @@ -518,13 +518,24 @@ def fake_parse(args: list[str]) -> dict[str, str]: with patch("th_cli.commands.run_tests._parse_extra_args", side_effect=fake_parse): result = cli_runner.invoke( run_tests, - ["--tests-list", "TC-ACE-1.1", "--", "--int-arg", "PIXIT.X:1", "--prompt_timeout", "5"], + ["--tests-list", "TC-ACE-1.1", "--", "--int-arg", "PIXIT.X:1", "--some-runner-flag", "5"], ) # Assert - assert captured["args"] == ["--int-arg", "PIXIT.X:1", "--prompt_timeout", "5"] + assert captured["args"] == ["--int-arg", "PIXIT.X:1", "--some-runner-flag", "5"] assert "stop after parsing" in result.output + @pytest.mark.parametrize("misplaced", ["--prompt-timeout", "--prompt_timeout", "--tests-list", "--project-id"]) + def test_run_tests_rejects_own_option_after_separator(self, cli_runner: CliRunner, misplaced: str) -> None: + """An option of run-tests after -- would only reach the test runner, so say where it goes.""" + # Act + result = cli_runner.invoke(run_tests, ["--tests-list", "TC-ACE-1.1", "--", misplaced, "5"]) + + # Assert + assert result.exit_code == 2 + assert f"'{misplaced}' is an option of run-tests" in result.output + assert "before the '--' separator" in result.output + @pytest.mark.parametrize( "test_list", [ diff --git a/th_cli/commands/run_tests.py b/th_cli/commands/run_tests.py index cc0bf1c..21f39c4 100644 --- a/th_cli/commands/run_tests.py +++ b/th_cli/commands/run_tests.py @@ -103,9 +103,30 @@ def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]: else: cli_args, inline_args = args, [] remaining = super().parse_args(ctx, cli_args) + self._reject_misplaced_options(ctx, inline_args) ctx.args = inline_args return remaining + def _reject_misplaced_options(self, ctx: click.Context, inline_args: list[str]) -> None: + """Fail if an option of this command was put after ``--``. + + After ``--`` everything goes to the test runner, which would reject (or + ignore) e.g. ``-- --prompt-timeout 5``. Dashes and underscores are treated + alike, so the common ``--prompt_timeout`` typo is caught too. + """ + own_options = { + opt.lstrip("-").replace("_", "-") + for param in self.params + for opt in (*param.opts, *param.secondary_opts) + if opt.startswith("--") + } + for arg in inline_args: + if arg.startswith("--") and arg[2:].replace("_", "-") in own_options: + raise click.UsageError( + f"'{arg}' is an option of run-tests, not a test parameter: " "pass it before the '--' separator.", + ctx, + ) + @click.command( cls=_RunTestsCommand, From 4c661d851ae05f4c4bd6685335ea078301a97da0 Mon Sep 17 00:00:00 2001 From: Romulo Quidute Filho Date: Wed, 18 Nov 2026 21:35:46 -0300 Subject: [PATCH 5/5] CLI: show multi-line API and suite errors on separate lines A long error glued to the header with ' - ' is hard to read. Put multi-line API error content below the header, and indent every line of a multi-line suite error. Related to #1215 --- tests/test_exceptions.py | 12 ++++++++++++ tests/test_run/test_websocket_additional.py | 13 +++++++++++++ th_cli/exceptions.py | 5 ++++- th_cli/test_run/websocket.py | 3 ++- 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index e08efff..40dc0b3 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -85,6 +85,18 @@ def test_with_status_code(self): assert "not found" in msg assert "404" in msg + def test_api_error_multiline_content_goes_below_header(self) -> None: + """Multi-line content (e.g. a list of problems) is not glued to the header.""" + content = "Invalid test parameter(s):\n '--int-args'\nNo test was started." + err = APIError("Failed to create", status_code=422, content=content) + + assert err.format_message() == f"Failed to create (Status: 422)\n{content}" + + def test_api_error_single_line_content_stays_inline(self) -> None: + err = APIError("Failed to create", status_code=500, content="boom") + + assert err.format_message() == "Failed to create (Status: 500) - boom" + def test_with_content(self): err = APIError("server error", content="Internal Server Error") msg = err.format_message() diff --git a/tests/test_run/test_websocket_additional.py b/tests/test_run/test_websocket_additional.py index 86d1dff..73d1dc9 100644 --- a/tests/test_run/test_websocket_additional.py +++ b/tests/test_run/test_websocket_additional.py @@ -539,6 +539,19 @@ def test_suite_update_prints_suite_errors_once(self, capsys): assert capsys.readouterr().err.count(error) == 1 + def test_suite_update_indents_every_line_of_multiline_error(self, capsys): + s = _make_socket(suites=[_make_suite()]) + error = "Invalid test parameter(s):\n '--int-args'\nNo test was started." + body = TestSuiteUpdate(state="error", test_suite_execution_index=0, errors=[error]) + + with patch("th_cli.test_run.websocket.get_log_stream_handler", return_value=None): + s._TestRunSocket__log_test_suite_update(body) + + lines = capsys.readouterr().err.splitlines() + assert any(line.endswith("Invalid test parameter(s):") for line in lines) + assert any(line.endswith(" '--int-args'") for line in lines) + assert all(line.startswith(" ") for line in lines if "No test was started" in line) + def test_suite_update_without_errors_prints_no_error(self, capsys): s = _make_socket(suites=[_make_suite()]) body = TestSuiteUpdate(state="executing", test_suite_execution_index=0) diff --git a/th_cli/exceptions.py b/th_cli/exceptions.py index 3797235..70ff5d1 100644 --- a/th_cli/exceptions.py +++ b/th_cli/exceptions.py @@ -50,7 +50,10 @@ def format_message(self) -> str: if self.status_code: msg += f" (Status: {self.status_code})" if self.content: - msg += f" - {self.content}" + # Multi-line content (e.g. a list of problems) reads better below the + # header than glued to it on one long line. + separator = "\n" if "\n" in str(self.content) else " - " + msg += f"{separator}{self.content}" return msg diff --git a/th_cli/test_run/websocket.py b/th_cli/test_run/websocket.py index 6fb5e07..2976b9a 100644 --- a/th_cli/test_run/websocket.py +++ b/th_cli/test_run/websocket.py @@ -407,7 +407,8 @@ def __log_test_suite_update(self, update: TestSuiteUpdate) -> None: key = (update.test_suite_execution_index, error) if key not in self._reported_suite_errors: self._reported_suite_errors.add(key) - click.echo(colorize_error(f" {error}"), err=True) + for line in error.splitlines() or [error]: + click.echo(colorize_error(f" {line}"), err=True) handler = get_log_stream_handler() if handler: