diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index e08efff..40dc0b3 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -85,6 +85,18 @@ def test_with_status_code(self): assert "not found" in msg assert "404" in msg + def test_api_error_multiline_content_goes_below_header(self) -> None: + """Multi-line content (e.g. a list of problems) is not glued to the header.""" + content = "Invalid test parameter(s):\n '--int-args'\nNo test was started." + err = APIError("Failed to create", status_code=422, content=content) + + assert err.format_message() == f"Failed to create (Status: 422)\n{content}" + + def test_api_error_single_line_content_stays_inline(self) -> None: + err = APIError("Failed to create", status_code=500, content="boom") + + assert err.format_message() == "Failed to create (Status: 500) - boom" + def test_with_content(self): err = APIError("server error", content="Internal Server Error") msg = err.format_message() diff --git a/tests/test_run/test_websocket_additional.py b/tests/test_run/test_websocket_additional.py index b45335f..73d1dc9 100644 --- a/tests/test_run/test_websocket_additional.py +++ b/tests/test_run/test_websocket_additional.py @@ -527,6 +527,40 @@ async def test_routes_suite_update(self): await s._TestRunSocket__handle_test_update(update=update) mock_fn.assert_called_once_with(body) + def test_suite_update_prints_suite_errors_once(self, capsys): + """Suite errors (e.g. failed setup) are shown, and not repeated on later updates.""" + s = _make_socket(suites=[_make_suite()]) + error = "Error occurred during setup of test suite. Invalid test parameter(s): '--prompt_timeout'" + body = TestSuiteUpdate(state="error", test_suite_execution_index=0, errors=[error]) + + with patch("th_cli.test_run.websocket.get_log_stream_handler", return_value=None): + s._TestRunSocket__log_test_suite_update(body) + s._TestRunSocket__log_test_suite_update(body) + + assert capsys.readouterr().err.count(error) == 1 + + def test_suite_update_indents_every_line_of_multiline_error(self, capsys): + s = _make_socket(suites=[_make_suite()]) + error = "Invalid test parameter(s):\n '--int-args'\nNo test was started." + body = TestSuiteUpdate(state="error", test_suite_execution_index=0, errors=[error]) + + with patch("th_cli.test_run.websocket.get_log_stream_handler", return_value=None): + s._TestRunSocket__log_test_suite_update(body) + + lines = capsys.readouterr().err.splitlines() + assert any(line.endswith("Invalid test parameter(s):") for line in lines) + assert any(line.endswith(" '--int-args'") for line in lines) + assert all(line.startswith(" ") for line in lines if "No test was started" in line) + + def test_suite_update_without_errors_prints_no_error(self, capsys): + s = _make_socket(suites=[_make_suite()]) + body = TestSuiteUpdate(state="executing", test_suite_execution_index=0) + + with patch("th_cli.test_run.websocket.get_log_stream_handler", return_value=None): + s._TestRunSocket__log_test_suite_update(body) + + assert capsys.readouterr().err == "" + @pytest.mark.asyncio async def test_routes_run_update_and_closes_socket_when_not_executing(self): s = _make_socket() diff --git a/tests/test_run_tests.py b/tests/test_run_tests.py index e1dc7d9..f19e245 100644 --- a/tests/test_run_tests.py +++ b/tests/test_run_tests.py @@ -20,6 +20,7 @@ from pathlib import Path from unittest.mock import AsyncMock, Mock, patch +import click import pytest from click.testing import CliRunner @@ -484,6 +485,57 @@ def test_run_tests_required_tests_list_parameter(self, cli_runner: CliRunner) -> assert result.exit_code != 0 assert "required" in result.output + def test_run_tests_rejects_unknown_option_before_separator(self, cli_runner: CliRunner) -> None: + """A typo like --prompt_timeout must fail up front instead of reaching the test runner.""" + # Act + result = cli_runner.invoke(run_tests, ["--tests-list", "TC-ACE-1.1", "--prompt_timeout", "5"]) + + # Assert + assert result.exit_code == 2 + # Wording differs across Click versions (quoted vs. unquoted option names) + assert re.search(r"No such option:? '?--prompt_timeout'?", result.output) + assert re.search(r"Did you mean '?--prompt-timeout'?\?", result.output) + + def test_run_tests_rejects_unexpected_positional_argument(self, cli_runner: CliRunner) -> None: + """Stray positional arguments before -- are errors, not silently ignored.""" + # Act + result = cli_runner.invoke(run_tests, ["--tests-list", "TC-ACE-1.1", "stray"]) + + # Assert + assert result.exit_code == 2 + assert "unexpected extra argument" in result.output.lower() + + def test_run_tests_forwards_args_after_separator(self, cli_runner: CliRunner) -> None: + """Unknown options after -- are still captured as test parameters.""" + # Arrange + captured: dict[str, list[str]] = {} + + def fake_parse(args: list[str]) -> dict[str, str]: + captured["args"] = args + raise click.ClickException("stop after parsing") + + # Act + with patch("th_cli.commands.run_tests._parse_extra_args", side_effect=fake_parse): + result = cli_runner.invoke( + run_tests, + ["--tests-list", "TC-ACE-1.1", "--", "--int-arg", "PIXIT.X:1", "--some-runner-flag", "5"], + ) + + # Assert + assert captured["args"] == ["--int-arg", "PIXIT.X:1", "--some-runner-flag", "5"] + assert "stop after parsing" in result.output + + @pytest.mark.parametrize("misplaced", ["--prompt-timeout", "--prompt_timeout", "--tests-list", "--project-id"]) + def test_run_tests_rejects_own_option_after_separator(self, cli_runner: CliRunner, misplaced: str) -> None: + """An option of run-tests after -- would only reach the test runner, so say where it goes.""" + # Act + result = cli_runner.invoke(run_tests, ["--tests-list", "TC-ACE-1.1", "--", misplaced, "5"]) + + # Assert + assert result.exit_code == 2 + assert f"'{misplaced}' is an option of run-tests" in result.output + assert "before the '--' separator" in result.output + @pytest.mark.parametrize( "test_list", [ diff --git a/th_cli/commands/run_tests.py b/th_cli/commands/run_tests.py index 1e40de4..21f39c4 100644 --- a/th_cli/commands/run_tests.py +++ b/th_cli/commands/run_tests.py @@ -87,11 +87,52 @@ def _complete_test_ids(ctx: click.Context, param: click.Parameter, incomplete: s return [] +class _RunTestsCommand(click.Command): + """Command that only treats arguments after a literal ``--`` as test parameters. + + Options before ``--`` are parsed strictly, so a typo such as ``--prompt_timeout`` + fails with "No such option" (plus a suggestion) instead of being forwarded to the + test runner as an extra test parameter. Everything after ``--`` is kept untouched + in ``ctx.args`` for ``_parse_extra_args``. + """ + + def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]: + if "--" in args: + separator = args.index("--") + cli_args, inline_args = args[:separator], args[separator + 1 :] + else: + cli_args, inline_args = args, [] + remaining = super().parse_args(ctx, cli_args) + self._reject_misplaced_options(ctx, inline_args) + ctx.args = inline_args + return remaining + + def _reject_misplaced_options(self, ctx: click.Context, inline_args: list[str]) -> None: + """Fail if an option of this command was put after ``--``. + + After ``--`` everything goes to the test runner, which would reject (or + ignore) e.g. ``-- --prompt-timeout 5``. Dashes and underscores are treated + alike, so the common ``--prompt_timeout`` typo is caught too. + """ + own_options = { + opt.lstrip("-").replace("_", "-") + for param in self.params + for opt in (*param.opts, *param.secondary_opts) + if opt.startswith("--") + } + for arg in inline_args: + if arg.startswith("--") and arg[2:].replace("_", "-") in own_options: + raise click.UsageError( + f"'{arg}' is an option of run-tests, not a test parameter: " "pass it before the '--' separator.", + ctx, + ) + + @click.command( + cls=_RunTestsCommand, no_args_is_help=True, short_help=colorize_help("CLI execution of a test run"), help=colorize_cmd_help("run_tests", "CLI execution of a test run from selected tests"), - context_settings={"ignore_unknown_options": True, "allow_extra_args": True}, ) @click.option( "--tests-list", diff --git a/th_cli/exceptions.py b/th_cli/exceptions.py index 3797235..70ff5d1 100644 --- a/th_cli/exceptions.py +++ b/th_cli/exceptions.py @@ -50,7 +50,10 @@ def format_message(self) -> str: if self.status_code: msg += f" (Status: {self.status_code})" if self.content: - msg += f" - {self.content}" + # Multi-line content (e.g. a list of problems) reads better below the + # header than glued to it on one long line. + separator = "\n" if "\n" in str(self.content) else " - " + msg += f"{separator}{self.content}" return msg diff --git a/th_cli/test_run/websocket.py b/th_cli/test_run/websocket.py index 3bd988e..2976b9a 100644 --- a/th_cli/test_run/websocket.py +++ b/th_cli/test_run/websocket.py @@ -142,6 +142,9 @@ def __init__( # Track test step errors for logging # Key: (suite_index, case_index), Value: list of error strings from all steps self.test_case_step_errors: dict[tuple[int, int], list[str]] = {} + # Suite-level errors (e.g. a failed suite setup) already shown to the user. + # Key: (suite_index, error), so repeated updates don't print them again. + self._reported_suite_errors: set[tuple[int, str]] = set() # Track the final state of each test case for the end-of-run summary. # Key: (suite_index, case_index), Value: final TestStateEnum value. # A dict (rather than a running counter) so a case that is updated more @@ -398,6 +401,15 @@ def __log_test_suite_update(self, update: TestSuiteUpdate) -> None: colored_state = colorize_state(update.state.value) click.echo(f" - {colored_title} {colored_state}") + # Suite-level errors (e.g. a failed suite setup) would otherwise only be in + # the backend log. Show each one once. + for error in update.errors or []: + key = (update.test_suite_execution_index, error) + if key not in self._reported_suite_errors: + self._reported_suite_errors.add(key) + for line in error.splitlines() or [error]: + click.echo(colorize_error(f" {line}"), err=True) + handler = get_log_stream_handler() if handler: handler.update_tree_node(