diff --git a/README.md b/README.md index 811e384..06fb823 100644 --- a/README.md +++ b/README.md @@ -111,10 +111,12 @@ parallel-cli │ ├── list # List available Parallel skills │ ├── reinstall # Cleanly reinstall managed Parallel skills │ └── uninstall # Remove only CLI-managed Parallel skills -└── memory # Saved Task, Monitor, and FindAll entries - ├── retrieve # Search Memory or list recent entries - ├── evict # Remove one entry from Memory - └── clear # Permanently clear selected Memory +├── memory # Saved Task, Monitor, and FindAll entries +│ ├── retrieve # Search Memory or list recent entries +│ ├── evict # Remove one entry from Memory +│ └── clear # Permanently clear selected Memory +└── feedback # Report problems to the Parallel team + └── send # Send a report and show what Parallel received ``` Run `parallel-cli skills install` to install or update agent skills. When using @@ -239,6 +241,26 @@ Omit `--scope-key` / `--memory-scope-key` to use personal Memory when the credential and account are eligible. Application credentials require a stable scope key containing only letters, digits, underscores, or hyphens. +### 7. Report a Problem to Parallel + +Report a problem with Parallel, with the request IDs attached: wrong, stale or +missing results, an answer the user says was wrong, a result you had to work +around, an error your request didn't cause, or a benchmark where Parallel +underperformed. The CLI sends the report and shows exactly what Parallel received. + +```bash +parallel-cli feedback send \ + -m "Eval runs return the wrong CEO for most companies." \ + --category results_quality --product task --severity blocking \ + --example "trun_abc123:Returned John Roe; expected Jane Doe" + +# Agents: --json prints the response, including what Parallel received. +parallel-cli feedback send ... --json +``` + +Feedback is open only to Parallel's own organizations for now, and never to +organizations on zero or partial data retention. + ## Non-Interactive Mode (for AI Agents & Scripts) All commands support `--json` output and can be fully controlled via CLI arguments. diff --git a/parallel_web_tools/cli/commands.py b/parallel_web_tools/cli/commands.py index c48e4e9..f4c7a3f 100644 --- a/parallel_web_tools/cli/commands.py +++ b/parallel_web_tools/cli/commands.py @@ -63,6 +63,14 @@ update_monitor, validate_memory_scope_key, ) +from parallel_web_tools.core.feedback import ( + FEEDBACK_CATEGORIES, + FEEDBACK_PRODUCTS, + FEEDBACK_SEVERITIES, + FeedbackInputError, + parse_example, + send_feedback, +) # Standalone CLI (PyInstaller) has limited features to reduce bundle size # YAML config and interactive planner require: pip install parallel-web-tools[cli] @@ -1084,6 +1092,110 @@ def memory_clear(memory_scope_key: str | None, confirm_clear: bool, output_json: console.print("[dim]Underlying Task, Monitor, and FindAll resources were not deleted.[/dim]") +# ============================================================================= +# Feedback Commands +# ============================================================================= + + +@main.group() +def feedback(): + """Report problems with Parallel to the Parallel team.""" + pass + + +def _example_option(_ctx: click.Context, _param: click.Parameter, values: tuple[str, ...]) -> list[dict[str, str]]: + try: + return [parse_example(value) for value in values] + except FeedbackInputError as e: + raise click.BadParameter(str(e)) from e + + +def _render_feedback_result(response: dict[str, Any]) -> None: + if response.get("duplicate"): + console.print(f"[yellow]Parallel already received this report as {response.get('feedback_id')}.[/yellow]") + else: + console.print(f"[bold green]Sent to Parallel as {response.get('feedback_id')}.[/bold green]") + report = response.get("report", {}) + context = report.get("context", {}) + console.print( + f" {report.get('category')} · {report.get('product')} · {report.get('severity')}", + markup=False, + ) + console.print(f" Message: {report.get('message', '')}", markup=False) + for label in ("expected", "actual"): + if report.get(label): + console.print(f" {label.capitalize()}: {report[label]}", markup=False) + for example in report.get("examples", []): + note = f" — {example['issue']}" if example.get("issue") else "" + console.print(f" • {example.get('id')}{note}", markup=False) + console.print(f" From: {context.get('sender', 'unknown')}", markup=False) + if report.get("redactions"): + console.print(f" [dim]{report['redactions']} secret(s) removed.[/dim]") + if report.get("recipient"): + console.print(f" {report['recipient']}", markup=False, style="dim") + + +@feedback.command(name="send") +@click.option("-m", "--message", required=True, help="What you were doing, what went wrong, and what you expected.") +@click.option("--category", type=click.Choice(list(FEEDBACK_CATEGORIES)), required=True, help="The kind of problem.") +@click.option( + "--product", type=click.Choice(list(FEEDBACK_PRODUCTS)), required=True, help="The Parallel product involved." +) +@click.option( + "--severity", + type=click.Choice(list(FEEDBACK_SEVERITIES)), + default="degraded", + show_default=True, + help="blocking, degraded (a workaround was needed) or minor (noticeable, but the work went on unchanged).", +) +@click.option( + "--example", + "examples", + multiple=True, + callback=_example_option, + help="A request ID, optionally with a note: 'trun_abc:Returned the wrong CEO'. Repeatable.", +) +@click.option("--expected", help="The overall expected behavior.") +@click.option("--actual", help="The overall observed behavior.") +@click.option("--json", "output_json", is_flag=True, help="Output JSON to stdout.") +def feedback_send( + message: str, + category: str, + product: str, + severity: str, + examples: list[dict[str, str]], + expected: str | None, + actual: str | None, + output_json: bool, +): + """Report a problem, such as wrong, stale or missing results for specific runs. + + Sends the report and shows exactly what Parallel received. + """ + try: + response = send_feedback( + message=message, + category=category, + product=product, + severity=severity, + examples=examples, + expected=expected, + actual=actual, + source="cli", + ) + except FeedbackInputError as e: + _handle_error(e, output_json=output_json, exit_code=EXIT_BAD_INPUT, prefix="Invalid feedback") + return + except Exception as e: + _handle_error(e, output_json=output_json, prefix="Feedback API error") + return + + if output_json: + print(json.dumps(response, indent=2)) + else: + _render_feedback_result(response) + + # ============================================================================= # Search Command # ============================================================================= diff --git a/parallel_web_tools/core/feedback.py b/parallel_web_tools/core/feedback.py new file mode 100644 index 0000000..5f9a1ca --- /dev/null +++ b/parallel_web_tools/core/feedback.py @@ -0,0 +1,107 @@ +"""Send feedback about Parallel to the Parallel team.""" + +from __future__ import annotations + +import re +from typing import Any, Literal + +from parallel_web_tools.core.auth import create_client +from parallel_web_tools.core.user_agent import ClientSource + +FeedbackCategory = Literal["results_quality", "error", "latency", "missing_capability", "docs", "billing", "other"] +FeedbackProduct = Literal[ + "search", "extract", "task", "findall", "monitor", "responses", "decisions", "mcp", "cli", "other" +] +FeedbackSeverity = Literal["blocking", "degraded", "minor"] + +FEEDBACK_CATEGORIES: tuple[FeedbackCategory, ...] = ( + "results_quality", + "error", + "latency", + "missing_capability", + "docs", + "billing", + "other", +) +FEEDBACK_PRODUCTS: tuple[FeedbackProduct, ...] = ( + "search", + "extract", + "task", + "findall", + "monitor", + "responses", + "decisions", + "mcp", + "cli", + "other", +) +FEEDBACK_SEVERITIES: tuple[FeedbackSeverity, ...] = ("blocking", "degraded", "minor") + +MIN_MESSAGE_CHARS = 20 +MAX_MESSAGE_CHARS = 4000 +MAX_EXAMPLES = 50 +MAX_NOTE_CHARS = 1000 +_EXAMPLE_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,128}$") + +_FEEDBACK_PATH = "/v1beta/feedback" + + +class FeedbackInputError(ValueError): + """Raised when a feedback request is invalid before it reaches the API.""" + + +def parse_example(raw: str) -> dict[str, str]: + """Parse ``ID`` or ``ID:note`` into an example; IDs never contain a colon.""" + example_id, sep, note = raw.partition(":") + example_id = example_id.strip() + if not _EXAMPLE_ID_RE.fullmatch(example_id): + raise FeedbackInputError( + f"example {raw!r} must start with a Parallel ID such as search_..., extract_..., or trun_..." + ) + note = note.strip() + if len(note) > MAX_NOTE_CHARS: + raise FeedbackInputError(f"example note must be at most {MAX_NOTE_CHARS} characters") + return {"id": example_id, "issue": note} if sep and note else {"id": example_id} + + +def _check_choice(value: str, choices: tuple[str, ...], field: str) -> None: + if value not in choices: + raise FeedbackInputError(f"{field} must be one of: {', '.join(choices)}") + + +def send_feedback( + message: str, + category: str, + product: str, + *, + severity: str = "degraded", + examples: list[dict[str, str]] | None = None, + expected: str | None = None, + actual: str | None = None, + api_key: str | None = None, + source: ClientSource = "python", +) -> dict[str, Any]: + """Send a feedback report; the response shows exactly what Parallel received.""" + if not MIN_MESSAGE_CHARS <= len(message) <= MAX_MESSAGE_CHARS: + raise FeedbackInputError(f"message must be {MIN_MESSAGE_CHARS}-{MAX_MESSAGE_CHARS} characters") + _check_choice(category, FEEDBACK_CATEGORIES, "category") + _check_choice(product, FEEDBACK_PRODUCTS, "product") + _check_choice(severity, FEEDBACK_SEVERITIES, "severity") + if examples is not None and len(examples) > MAX_EXAMPLES: + raise FeedbackInputError(f"examples must be at most {MAX_EXAMPLES}") + + body: dict[str, Any] = { + "message": message, + "category": category, + "product": product, + "severity": severity, + } + if examples: + body["examples"] = examples + if expected: + body["expected"] = expected + if actual: + body["actual"] = actual + + client = create_client(api_key, source) + return client.post(path=_FEEDBACK_PATH, body=body, cast_to=dict) diff --git a/tests/test_feedback.py b/tests/test_feedback.py new file mode 100644 index 0000000..21bf6dc --- /dev/null +++ b/tests/test_feedback.py @@ -0,0 +1,140 @@ +"""Tests for sending feedback to Parallel from the CLI.""" + +from __future__ import annotations + +import json +from typing import Any +from unittest import mock + +import pytest +from click.testing import CliRunner + +from parallel_web_tools.cli.commands import main +from parallel_web_tools.core.feedback import FeedbackInputError, parse_example, send_feedback + +_MESSAGE = "Search results for our eval queries are mostly irrelevant." +_RECEIVED = { + "status": "received", + "feedback_id": "feedback_1", + "duplicate": False, + "report": { + "message": _MESSAGE, + "category": "results_quality", + "product": "search", + "severity": "degraded", + "examples": [{"id": "search_abc", "issue": "Off topic", "expected": None}], + "expected": None, + "actual": None, + "context": {"org_id": "org", "sender": "user u1 (personal credential)", "surface": "cli"}, + "redactions": 0, + "recipient": "Parallel's team only.", + }, + "next_step": "Sent.", +} + + +@pytest.fixture +def runner() -> CliRunner: + return CliRunner() + + +@pytest.fixture +def client(): + client = mock.MagicMock() + with mock.patch("parallel_web_tools.core.feedback.create_client", return_value=client): + yield client + + +def _sent_bodies(client) -> list[dict]: + return [call.kwargs["body"] for call in client.post.call_args_list] + + +class TestValidation: + @pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"message": "too short"}, "message"), + ({"category": "praise"}, "category"), + ({"product": "spreadsheet"}, "product"), + ({"severity": "urgent"}, "severity"), + ({"examples": [{"id": f"trun_{n}"} for n in range(51)]}, "examples"), + ], + ) + def test_rejects_invalid_reports_before_calling_the_api(self, client, kwargs, message): + fields: dict[str, Any] = {"message": _MESSAGE, "category": "results_quality", "product": "search", **kwargs} + with pytest.raises(FeedbackInputError, match=message): + send_feedback(**fields) + client.post.assert_not_called() + + @pytest.mark.parametrize( + ("raw", "expected"), + [ + ("trun_abc", {"id": "trun_abc"}), + ("search_abc:Returned: nothing useful", {"id": "search_abc", "issue": "Returned: nothing useful"}), + ], + ) + def test_parses_examples(self, raw, expected): + assert parse_example(raw) == expected + + def test_rejects_an_example_that_is_not_an_id(self): + with pytest.raises(FeedbackInputError, match="example"): + parse_example("not an id:note") + + +class TestApi: + def test_send_posts_only_the_fields_given(self, client): + client.post.return_value = _RECEIVED + result = send_feedback( + message=_MESSAGE, + category="results_quality", + product="search", + examples=[{"id": "search_abc", "issue": "Off topic"}], + source="cli", + ) + assert result == _RECEIVED + client.post.assert_called_once_with( + path="/v1beta/feedback", + body={ + "message": _MESSAGE, + "category": "results_quality", + "product": "search", + "severity": "degraded", + "examples": [{"id": "search_abc", "issue": "Off topic"}], + }, + cast_to=dict, + ) + + +class TestSendCommand: + _ARGS = ["feedback", "send", "-m", _MESSAGE, "--category", "results_quality", "--product", "search"] + + def test_json_sends_once_and_prints_the_response(self, runner, client): + client.post.return_value = _RECEIVED + result = runner.invoke(main, [*self._ARGS, "--example", "search_abc:Off topic", "--json"]) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["feedback_id"] == "feedback_1" + assert len(_sent_bodies(client)) == 1 + + def test_shows_what_parallel_received(self, runner, client): + client.post.return_value = _RECEIVED + result = runner.invoke(main, self._ARGS) + assert result.exit_code == 0, result.output + assert len(_sent_bodies(client)) == 1 + assert "feedback_1" in result.output + assert _MESSAGE in result.output + assert "search_abc" in result.output + + def test_a_repeat_says_so(self, runner, client): + client.post.return_value = {**_RECEIVED, "duplicate": True} + result = runner.invoke(main, self._ARGS) + assert result.exit_code == 0, result.output + assert "already received" in result.output + + def test_a_bad_example_is_a_usage_error(self, runner, client): + result = runner.invoke(main, [*self._ARGS, "--example", "has spaces", "--json"]) + assert result.exit_code != 0 + client.post.assert_not_called() + + def test_there_is_no_consent_command(self, runner, client): + result = runner.invoke(main, ["feedback", "consent"]) + assert result.exit_code != 0