Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions tests/test_exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,18 @@ def test_with_status_code(self):
assert "not found" in msg
assert "404" in msg

def test_api_error_multiline_content_goes_below_header(self) -> None:
"""Multi-line content (e.g. a list of problems) is not glued to the header."""
content = "Invalid test parameter(s):\n '--int-args'\nNo test was started."
err = APIError("Failed to create", status_code=422, content=content)

assert err.format_message() == f"Failed to create (Status: 422)\n{content}"

def test_api_error_single_line_content_stays_inline(self) -> None:
err = APIError("Failed to create", status_code=500, content="boom")

assert err.format_message() == "Failed to create (Status: 500) - boom"

def test_with_content(self):
err = APIError("server error", content="Internal Server Error")
msg = err.format_message()
Expand Down
34 changes: 34 additions & 0 deletions tests/test_run/test_websocket_additional.py
Original file line number Diff line number Diff line change
Expand Up @@ -527,6 +527,40 @@ async def test_routes_suite_update(self):
await s._TestRunSocket__handle_test_update(update=update)
mock_fn.assert_called_once_with(body)

def test_suite_update_prints_suite_errors_once(self, capsys):
"""Suite errors (e.g. failed setup) are shown, and not repeated on later updates."""
s = _make_socket(suites=[_make_suite()])
error = "Error occurred during setup of test suite. Invalid test parameter(s): '--prompt_timeout'"
body = TestSuiteUpdate(state="error", test_suite_execution_index=0, errors=[error])

with patch("th_cli.test_run.websocket.get_log_stream_handler", return_value=None):
s._TestRunSocket__log_test_suite_update(body)
s._TestRunSocket__log_test_suite_update(body)

assert capsys.readouterr().err.count(error) == 1

def test_suite_update_indents_every_line_of_multiline_error(self, capsys):
s = _make_socket(suites=[_make_suite()])
error = "Invalid test parameter(s):\n '--int-args'\nNo test was started."
body = TestSuiteUpdate(state="error", test_suite_execution_index=0, errors=[error])

with patch("th_cli.test_run.websocket.get_log_stream_handler", return_value=None):
s._TestRunSocket__log_test_suite_update(body)

lines = capsys.readouterr().err.splitlines()
assert any(line.endswith("Invalid test parameter(s):") for line in lines)
assert any(line.endswith(" '--int-args'") for line in lines)
assert all(line.startswith(" ") for line in lines if "No test was started" in line)

def test_suite_update_without_errors_prints_no_error(self, capsys):
s = _make_socket(suites=[_make_suite()])
body = TestSuiteUpdate(state="executing", test_suite_execution_index=0)

with patch("th_cli.test_run.websocket.get_log_stream_handler", return_value=None):
s._TestRunSocket__log_test_suite_update(body)

assert capsys.readouterr().err == ""

@pytest.mark.asyncio
async def test_routes_run_update_and_closes_socket_when_not_executing(self):
s = _make_socket()
Expand Down
52 changes: 52 additions & 0 deletions tests/test_run_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from pathlib import Path
from unittest.mock import AsyncMock, Mock, patch

import click
import pytest
from click.testing import CliRunner

Expand Down Expand Up @@ -484,6 +485,57 @@ def test_run_tests_required_tests_list_parameter(self, cli_runner: CliRunner) ->
assert result.exit_code != 0
assert "required" in result.output

def test_run_tests_rejects_unknown_option_before_separator(self, cli_runner: CliRunner) -> None:
"""A typo like --prompt_timeout must fail up front instead of reaching the test runner."""
# Act
result = cli_runner.invoke(run_tests, ["--tests-list", "TC-ACE-1.1", "--prompt_timeout", "5"])

# Assert
assert result.exit_code == 2
# Wording differs across Click versions (quoted vs. unquoted option names)
assert re.search(r"No such option:? '?--prompt_timeout'?", result.output)
assert re.search(r"Did you mean '?--prompt-timeout'?\?", result.output)

def test_run_tests_rejects_unexpected_positional_argument(self, cli_runner: CliRunner) -> None:
"""Stray positional arguments before -- are errors, not silently ignored."""
# Act
result = cli_runner.invoke(run_tests, ["--tests-list", "TC-ACE-1.1", "stray"])

# Assert
assert result.exit_code == 2
assert "unexpected extra argument" in result.output.lower()

def test_run_tests_forwards_args_after_separator(self, cli_runner: CliRunner) -> None:
"""Unknown options after -- are still captured as test parameters."""
# Arrange
captured: dict[str, list[str]] = {}

def fake_parse(args: list[str]) -> dict[str, str]:
captured["args"] = args
raise click.ClickException("stop after parsing")

# Act
with patch("th_cli.commands.run_tests._parse_extra_args", side_effect=fake_parse):
result = cli_runner.invoke(
run_tests,
["--tests-list", "TC-ACE-1.1", "--", "--int-arg", "PIXIT.X:1", "--some-runner-flag", "5"],
)

# Assert
assert captured["args"] == ["--int-arg", "PIXIT.X:1", "--some-runner-flag", "5"]
assert "stop after parsing" in result.output

@pytest.mark.parametrize("misplaced", ["--prompt-timeout", "--prompt_timeout", "--tests-list", "--project-id"])
def test_run_tests_rejects_own_option_after_separator(self, cli_runner: CliRunner, misplaced: str) -> None:
"""An option of run-tests after -- would only reach the test runner, so say where it goes."""
# Act
result = cli_runner.invoke(run_tests, ["--tests-list", "TC-ACE-1.1", "--", misplaced, "5"])

# Assert
assert result.exit_code == 2
assert f"'{misplaced}' is an option of run-tests" in result.output
assert "before the '--' separator" in result.output

@pytest.mark.parametrize(
"test_list",
[
Expand Down
43 changes: 42 additions & 1 deletion th_cli/commands/run_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,11 +87,52 @@ def _complete_test_ids(ctx: click.Context, param: click.Parameter, incomplete: s
return []


class _RunTestsCommand(click.Command):
"""Command that only treats arguments after a literal ``--`` as test parameters.

Options before ``--`` are parsed strictly, so a typo such as ``--prompt_timeout``
fails with "No such option" (plus a suggestion) instead of being forwarded to the
test runner as an extra test parameter. Everything after ``--`` is kept untouched
in ``ctx.args`` for ``_parse_extra_args``.
"""

def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:
if "--" in args:
separator = args.index("--")
cli_args, inline_args = args[:separator], args[separator + 1 :]
else:
cli_args, inline_args = args, []
remaining = super().parse_args(ctx, cli_args)
self._reject_misplaced_options(ctx, inline_args)
ctx.args = inline_args
return remaining

def _reject_misplaced_options(self, ctx: click.Context, inline_args: list[str]) -> None:
"""Fail if an option of this command was put after ``--``.

After ``--`` everything goes to the test runner, which would reject (or
ignore) e.g. ``-- --prompt-timeout 5``. Dashes and underscores are treated
alike, so the common ``--prompt_timeout`` typo is caught too.
"""
own_options = {
opt.lstrip("-").replace("_", "-")
for param in self.params
for opt in (*param.opts, *param.secondary_opts)
if opt.startswith("--")
}
for arg in inline_args:
if arg.startswith("--") and arg[2:].replace("_", "-") in own_options:
raise click.UsageError(
f"'{arg}' is an option of run-tests, not a test parameter: " "pass it before the '--' separator.",
ctx,
)
Comment thread
greens marked this conversation as resolved.
Resolved


@click.command(
cls=_RunTestsCommand,
no_args_is_help=True,
short_help=colorize_help("CLI execution of a test run"),
help=colorize_cmd_help("run_tests", "CLI execution of a test run from selected tests"),
context_settings={"ignore_unknown_options": True, "allow_extra_args": True},
)
@click.option(
"--tests-list",
Expand Down
5 changes: 4 additions & 1 deletion th_cli/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,10 @@ def format_message(self) -> str:
if self.status_code:
msg += f" (Status: {self.status_code})"
if self.content:
msg += f" - {self.content}"
# Multi-line content (e.g. a list of problems) reads better below the
# header than glued to it on one long line.
separator = "\n" if "\n" in str(self.content) else " - "
msg += f"{separator}{self.content}"
return msg


Expand Down
12 changes: 12 additions & 0 deletions th_cli/test_run/websocket.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,9 @@ def __init__(
# Track test step errors for logging
# Key: (suite_index, case_index), Value: list of error strings from all steps
self.test_case_step_errors: dict[tuple[int, int], list[str]] = {}
# Suite-level errors (e.g. a failed suite setup) already shown to the user.
# Key: (suite_index, error), so repeated updates don't print them again.
self._reported_suite_errors: set[tuple[int, str]] = set()
# Track the final state of each test case for the end-of-run summary.
# Key: (suite_index, case_index), Value: final TestStateEnum value.
# A dict (rather than a running counter) so a case that is updated more
Expand Down Expand Up @@ -398,6 +401,15 @@ def __log_test_suite_update(self, update: TestSuiteUpdate) -> None:
colored_state = colorize_state(update.state.value)
click.echo(f" - {colored_title} {colored_state}")

# Suite-level errors (e.g. a failed suite setup) would otherwise only be in
# the backend log. Show each one once.
for error in update.errors or []:
key = (update.test_suite_execution_index, error)
if key not in self._reported_suite_errors:
self._reported_suite_errors.add(key)
for line in error.splitlines() or [error]:
click.echo(colorize_error(f" {line}"), err=True)

handler = get_log_stream_handler()
if handler:
handler.update_tree_node(
Expand Down
Loading