From 62238735acc16a71cd5658af478d9e28307525fd Mon Sep 17 00:00:00 2001 From: Manvesh Vyas Date: Thu, 22 Oct 2026 12:56:01 -0700 Subject: [PATCH 1/6] feat(cli): add feedback send and feedback consent commands parallel-cli feedback send previews a report via POST /v1beta/feedback, asks the user, and sends it with the approved preview_id. --json never prompts, so agents show the preview and rerun with --consent and --preview-id. feedback consent shows or sets "don't ask again". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0157KeK2RbX5ARs3g1ZcvmGt --- README.md | 35 ++++- parallel_web_tools/cli/commands.py | 194 ++++++++++++++++++++++++++++ parallel_web_tools/core/feedback.py | 139 ++++++++++++++++++++ tests/test_feedback.py | 192 +++++++++++++++++++++++++++ 4 files changed, 556 insertions(+), 4 deletions(-) create mode 100644 parallel_web_tools/core/feedback.py create mode 100644 tests/test_feedback.py diff --git a/README.md b/README.md index 811e384..928b4bc 100644 --- a/README.md +++ b/README.md @@ -111,10 +111,13 @@ parallel-cli │ ├── list # List available Parallel skills │ ├── reinstall # Cleanly reinstall managed Parallel skills │ └── uninstall # Remove only CLI-managed Parallel skills -└── memory # Saved Task, Monitor, and FindAll entries - ├── retrieve # Search Memory or list recent entries - ├── evict # Remove one entry from Memory - └── clear # Permanently clear selected Memory +├── memory # Saved Task, Monitor, and FindAll entries +│ ├── retrieve # Search Memory or list recent entries +│ ├── evict # Remove one entry from Memory +│ └── clear # Permanently clear selected Memory +└── feedback # Report problems to the Parallel team + ├── send # Preview a report, then send it with consent + └── consent # Show or set "don't ask again" ``` Run `parallel-cli skills install` to install or update agent skills. When using @@ -239,6 +242,30 @@ Omit `--scope-key` / `--memory-scope-key` to use personal Memory when the credential and account are eligible. Application credentials require a stable scope key containing only letters, digits, underscores, or hyphens. +### 7. Report a Problem to Parallel + +Report significant problems, such as wrong results for specific runs, with the +request IDs attached. Nothing is sent until the user agrees: the CLI shows +exactly what Parallel would receive and asks first. + +```bash +parallel-cli feedback send \ + -m "Eval runs return the wrong CEO for most companies." \ + --category results_quality --product task --severity blocking \ + --example "trun_abc123:Returned John Roe; expected Jane Doe" + +# Agents: --json prints the preview without sending. Show it to the user, then +# rerun with their answer and the preview's preview_id. +parallel-cli feedback send ... --json +parallel-cli feedback send ... --consent this_report --preview-id --json + +# Show or reset "don't ask again" (personal credentials only) +parallel-cli feedback consent +parallel-cli feedback consent ask +``` + +Feedback is not available to organizations on zero or partial data retention. + ## Non-Interactive Mode (for AI Agents & Scripts) All commands support `--json` output and can be fully controlled via CLI arguments. diff --git a/parallel_web_tools/cli/commands.py b/parallel_web_tools/cli/commands.py index c48e4e9..4fd6ef2 100644 --- a/parallel_web_tools/cli/commands.py +++ b/parallel_web_tools/cli/commands.py @@ -63,6 +63,17 @@ update_monitor, validate_memory_scope_key, ) +from parallel_web_tools.core.feedback import ( + CONSENT_PREFERENCES, + FEEDBACK_CATEGORIES, + FEEDBACK_PRODUCTS, + FEEDBACK_SEVERITIES, + FeedbackInputError, + get_feedback_consent, + parse_example, + send_feedback, + set_feedback_consent, +) # Standalone CLI (PyInstaller) has limited features to reduce bundle size # YAML config and interactive planner require: pip install parallel-web-tools[cli] @@ -1084,6 +1095,189 @@ def memory_clear(memory_scope_key: str | None, confirm_clear: bool, output_json: console.print("[dim]Underlying Task, Monitor, and FindAll resources were not deleted.[/dim]") +# ============================================================================= +# Feedback Commands +# ============================================================================= + + +@main.group() +def feedback(): + """Report significant problems with Parallel to the Parallel team.""" + pass + + +def _example_option(_ctx: click.Context, _param: click.Parameter, values: tuple[str, ...]) -> list[dict[str, str]]: + try: + return [parse_example(value) for value in values] + except FeedbackInputError as e: + raise click.BadParameter(str(e)) from e + + +def _render_feedback_preview(response: dict[str, Any]) -> None: + preview = response.get("preview", {}) + context = preview.get("context", {}) + console.print("[bold]Parallel would receive:[/bold]") + console.print( + f" {preview.get('category')} · {preview.get('product')} · {preview.get('severity')}", + markup=False, + ) + console.print(f" Message: {preview.get('message', '')}", markup=False) + for label in ("expected", "actual"): + if preview.get(label): + console.print(f" {label.capitalize()}: {preview[label]}", markup=False) + for example in preview.get("examples", []): + note = f" — {example['issue']}" if example.get("issue") else "" + console.print(f" • {example.get('id')}{note}", markup=False) + console.print(f" From: {context.get('sender', 'unknown')}", markup=False) + if preview.get("redactions"): + console.print(f" [dim]{preview['redactions']} secret(s) removed.[/dim]") + if preview.get("recipient"): + console.print(f" {preview['recipient']}", markup=False, style="dim") + + +def _render_feedback_result(response: dict[str, Any]) -> None: + if response.get("duplicate"): + console.print(f"[yellow]Parallel already received this report as {response.get('feedback_id')}.[/yellow]") + else: + console.print(f"[bold green]Sent to Parallel as {response.get('feedback_id')}.[/bold green]") + if response.get("consent_preference") == "always": + console.print("[dim]Later reports are sent without asking. Reset with: parallel-cli feedback consent ask[/dim]") + + +def _ask_feedback_consent(response: dict[str, Any]) -> str | None: + """Ask whether to send the previewed report; returns the consent, or None to stop.""" + can_remember = response.get("preview", {}).get("remember_scope") is not None + choices = ["s", "a", "n"] if can_remember else ["s", "n"] + console.print( + "Send it? [bold]s[/bold]end" + + (", send and don't ask [bold]a[/bold]gain" if can_remember else "") + + ", or [bold]n[/bold]o" + ) + answer = click.prompt("", type=click.Choice(choices), show_choices=False, prompt_suffix="> ") + return {"s": "this_report", "a": "always"}.get(answer) + + +@feedback.command(name="send") +@click.option("-m", "--message", required=True, help="What you were doing, what went wrong, and what you expected.") +@click.option("--category", type=click.Choice(list(FEEDBACK_CATEGORIES)), required=True, help="The kind of problem.") +@click.option( + "--product", type=click.Choice(list(FEEDBACK_PRODUCTS)), required=True, help="The Parallel product involved." +) +@click.option( + "--severity", + type=click.Choice(list(FEEDBACK_SEVERITIES)), + default="degraded", + show_default=True, + help="blocking, degraded (a workaround was needed) or minor.", +) +@click.option( + "--example", + "examples", + multiple=True, + callback=_example_option, + help="A request ID, optionally with a note: 'trun_abc:Returned the wrong CEO'. Repeatable.", +) +@click.option("--expected", help="The overall expected behavior.") +@click.option("--actual", help="The overall observed behavior.") +@click.option( + "--consent", + type=click.Choice(["this_report", "always"]), + help="The user approved sending this report (this_report) and, with always, later ones.", +) +@click.option("--preview-id", help="The preview_id of the preview the user approved.") +@click.option("--json", "output_json", is_flag=True, help="Output JSON to stdout; never prompts.") +def feedback_send( + message: str, + category: str, + product: str, + severity: str, + examples: list[dict[str, str]], + expected: str | None, + actual: str | None, + consent: str | None, + preview_id: str | None, + output_json: bool, +): + """Report a significant problem, such as wrong results for specific runs. + + Nothing is sent until the user agrees. Without --consent the CLI shows + exactly what Parallel would receive and asks; with --json it prints that + preview so an agent can ask the user, then rerun with --consent and + --preview-id. + """ + + def send(user_consent: str, approved_preview: str | None) -> dict[str, Any]: + return send_feedback( + message=message, + category=category, + product=product, + severity=severity, + examples=examples, + expected=expected, + actual=actual, + user_consent=user_consent, + preview_id=approved_preview, + source="cli", + ) + + try: + response = send(consent or "not_asked", preview_id) + if response.get("status") == "consent_required" and not output_json: + _render_feedback_preview(response) + approved = _ask_feedback_consent(response) + if approved is None: + console.print("Nothing was sent.") + return + response = send(approved, response.get("preview_id")) + except FeedbackInputError as e: + _handle_error(e, output_json=output_json, exit_code=EXIT_BAD_INPUT, prefix="Invalid feedback") + return + except Exception as e: + _handle_error(e, output_json=output_json, prefix="Feedback API error") + return + + if output_json: + print(json.dumps(response, indent=2)) + elif response.get("status") == "consent_required": + _render_feedback_preview(response) + console.print(response.get("next_step", "Nothing was sent."), markup=False, style="yellow") + else: + _render_feedback_result(response) + + +@feedback.command(name="consent") +@click.argument("preference", required=False, type=click.Choice(list(CONSENT_PREFERENCES))) +@click.option("--json", "output_json", is_flag=True, help="Output JSON to stdout.") +def feedback_consent(preference: str | None, output_json: bool): + """Show or set whether feedback reports are sent without asking. + + With no argument, shows the current preference. 'always' sends reports + without a preview; 'ask' previews each report first. Only personal + credentials can store a preference. + """ + try: + if preference is None: + result = get_feedback_consent(source="cli") + else: + result = set_feedback_consent(preference, source="cli") + except FeedbackInputError as e: + _handle_error(e, output_json=output_json, exit_code=EXIT_BAD_INPUT, prefix="Invalid feedback consent") + return + except Exception as e: + _handle_error(e, output_json=output_json, prefix="Feedback API error") + return + + if output_json: + print(json.dumps(result, indent=2)) + return + if result.get("preference") == "always": + console.print("Feedback reports are sent [bold]without asking[/bold].") + else: + console.print("Each feedback report is [bold]previewed[/bold] and needs approval.") + if result.get("scope"): + console.print(f"Applies to {result['scope']}.", markup=False, style="dim") + + # ============================================================================= # Search Command # ============================================================================= diff --git a/parallel_web_tools/core/feedback.py b/parallel_web_tools/core/feedback.py new file mode 100644 index 0000000..8e05446 --- /dev/null +++ b/parallel_web_tools/core/feedback.py @@ -0,0 +1,139 @@ +"""Send feedback about Parallel to the Parallel team. + +The API sends nothing until the user consents: a first call returns a preview of +exactly what Parallel would receive, and the user approves it for this report +(``this_report``) or for all later ones (``always``). +""" + +from __future__ import annotations + +import re +from typing import Any, Literal + +from parallel_web_tools.core.auth import create_client +from parallel_web_tools.core.user_agent import ClientSource + +FeedbackCategory = Literal["results_quality", "error", "latency", "missing_capability", "docs", "billing", "other"] +FeedbackProduct = Literal["search", "extract", "task", "findall", "monitor", "responses", "mcp", "cli", "other"] +FeedbackSeverity = Literal["blocking", "degraded", "minor"] +UserConsent = Literal["not_asked", "this_report", "always"] +ConsentPreference = Literal["ask", "always"] + +FEEDBACK_CATEGORIES: tuple[FeedbackCategory, ...] = ( + "results_quality", + "error", + "latency", + "missing_capability", + "docs", + "billing", + "other", +) +FEEDBACK_PRODUCTS: tuple[FeedbackProduct, ...] = ( + "search", + "extract", + "task", + "findall", + "monitor", + "responses", + "mcp", + "cli", + "other", +) +FEEDBACK_SEVERITIES: tuple[FeedbackSeverity, ...] = ("blocking", "degraded", "minor") +USER_CONSENTS: tuple[UserConsent, ...] = ("not_asked", "this_report", "always") +CONSENT_PREFERENCES: tuple[ConsentPreference, ...] = ("ask", "always") + +MIN_MESSAGE_CHARS = 20 +MAX_MESSAGE_CHARS = 4000 +MAX_EXAMPLES = 50 +MAX_NOTE_CHARS = 1000 +_EXAMPLE_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,128}$") + +_FEEDBACK_PATH = "/v1beta/feedback" +_CONSENT_PATH = "/v1beta/feedback/consent" + + +class FeedbackInputError(ValueError): + """Raised when a feedback request is invalid before it reaches the API.""" + + +def parse_example(raw: str) -> dict[str, str]: + """Parse ``ID`` or ``ID:note`` into an example; IDs never contain a colon.""" + example_id, sep, note = raw.partition(":") + example_id = example_id.strip() + if not _EXAMPLE_ID_RE.fullmatch(example_id): + raise FeedbackInputError( + f"example {raw!r} must start with a Parallel ID such as search_..., extract_..., or trun_..." + ) + note = note.strip() + if len(note) > MAX_NOTE_CHARS: + raise FeedbackInputError(f"example note must be at most {MAX_NOTE_CHARS} characters") + return {"id": example_id, "issue": note} if sep and note else {"id": example_id} + + +def _check_choice(value: str, choices: tuple[str, ...], field: str) -> None: + if value not in choices: + raise FeedbackInputError(f"{field} must be one of: {', '.join(choices)}") + + +def send_feedback( + message: str, + category: str, + product: str, + *, + severity: str = "degraded", + examples: list[dict[str, str]] | None = None, + expected: str | None = None, + actual: str | None = None, + user_consent: str = "not_asked", + preview_id: str | None = None, + api_key: str | None = None, + source: ClientSource = "python", +) -> dict[str, Any]: + """Preview or send a feedback report. + + With ``user_consent="not_asked"`` and no stored ``always`` preference, the + response has ``status="consent_required"`` and a preview; nothing is sent. + """ + if not MIN_MESSAGE_CHARS <= len(message) <= MAX_MESSAGE_CHARS: + raise FeedbackInputError(f"message must be {MIN_MESSAGE_CHARS}-{MAX_MESSAGE_CHARS} characters") + _check_choice(category, FEEDBACK_CATEGORIES, "category") + _check_choice(product, FEEDBACK_PRODUCTS, "product") + _check_choice(severity, FEEDBACK_SEVERITIES, "severity") + _check_choice(user_consent, USER_CONSENTS, "user_consent") + if examples is not None and len(examples) > MAX_EXAMPLES: + raise FeedbackInputError(f"examples must be at most {MAX_EXAMPLES}") + + body: dict[str, Any] = { + "message": message, + "category": category, + "product": product, + "severity": severity, + } + if examples: + body["examples"] = examples + if expected: + body["expected"] = expected + if actual: + body["actual"] = actual + body["user_consent"] = user_consent + if preview_id: + body["preview_id"] = preview_id + + client = create_client(api_key, source) + return client.post(path=_FEEDBACK_PATH, body=body, cast_to=dict) + + +def get_feedback_consent(*, api_key: str | None = None, source: ClientSource = "python") -> dict[str, Any]: + """Return whether reports are sent without asking (``always``) or previewed first (``ask``).""" + client = create_client(api_key, source) + return client.get(_CONSENT_PATH, cast_to=dict) + + +def set_feedback_consent( + preference: str, *, api_key: str | None = None, source: ClientSource = "python" +) -> dict[str, Any]: + """Store ``always`` or reset to ``ask``. Only personal credentials can store a preference.""" + _check_choice(preference, CONSENT_PREFERENCES, "preference") + client = create_client(api_key, source) + return client.put(path=_CONSENT_PATH, body={"preference": preference}, cast_to=dict) diff --git a/tests/test_feedback.py b/tests/test_feedback.py new file mode 100644 index 0000000..9794d2a --- /dev/null +++ b/tests/test_feedback.py @@ -0,0 +1,192 @@ +"""Tests for sending feedback to Parallel from the CLI.""" + +from __future__ import annotations + +import json +from typing import Any +from unittest import mock + +import pytest +from click.testing import CliRunner + +from parallel_web_tools.cli.commands import main +from parallel_web_tools.core.feedback import ( + FeedbackInputError, + get_feedback_consent, + parse_example, + send_feedback, + set_feedback_consent, +) + +_MESSAGE = "Search results for our eval queries are mostly irrelevant." +_PREVIEW = { + "status": "consent_required", + "feedback_id": None, + "preview_id": "p1", + "duplicate": False, + "consent_preference": "ask", + "preview": { + "message": _MESSAGE, + "category": "results_quality", + "product": "search", + "severity": "degraded", + "examples": [{"id": "search_abc", "link": "unchecked", "issue": "Off topic", "expected": None}], + "expected": None, + "actual": None, + "context": {"org_id": "org", "sender": "user u1 (personal credential)", "surface": "cli"}, + "redactions": 0, + "recipient": "Parallel's team only.", + "remember_scope": "Parallel user u1, on any of their personal credentials", + }, + "next_step": "Nothing was sent.", +} +_RECEIVED = {**_PREVIEW, "status": "received", "feedback_id": "feedback_1", "next_step": "Sent."} + + +@pytest.fixture +def runner() -> CliRunner: + return CliRunner() + + +@pytest.fixture +def client(): + client = mock.MagicMock() + with mock.patch("parallel_web_tools.core.feedback.create_client", return_value=client): + yield client + + +def _sent_bodies(client) -> list[dict]: + return [call.kwargs["body"] for call in client.post.call_args_list] + + +class TestValidation: + @pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"message": "too short"}, "message"), + ({"category": "praise"}, "category"), + ({"product": "spreadsheet"}, "product"), + ({"severity": "urgent"}, "severity"), + ({"examples": [{"id": f"trun_{n}"} for n in range(51)]}, "examples"), + ({"user_consent": "maybe"}, "user_consent"), + ], + ) + def test_rejects_invalid_reports_before_calling_the_api(self, client, kwargs, message): + fields: dict[str, Any] = {"message": _MESSAGE, "category": "results_quality", "product": "search", **kwargs} + with pytest.raises(FeedbackInputError, match=message): + send_feedback(**fields) + client.post.assert_not_called() + + @pytest.mark.parametrize( + ("raw", "expected"), + [ + ("trun_abc", {"id": "trun_abc"}), + ("search_abc:Returned: nothing useful", {"id": "search_abc", "issue": "Returned: nothing useful"}), + ], + ) + def test_parses_examples(self, raw, expected): + assert parse_example(raw) == expected + + def test_rejects_an_example_that_is_not_an_id(self): + with pytest.raises(FeedbackInputError, match="example"): + parse_example("not an id:note") + + def test_only_ask_or_always_can_be_stored(self, client): + with pytest.raises(FeedbackInputError, match="preference"): + set_feedback_consent("sometimes") + client.put.assert_not_called() + + +class TestApi: + def test_send_posts_only_the_fields_given(self, client): + client.post.return_value = _PREVIEW + result = send_feedback( + message=_MESSAGE, + category="results_quality", + product="search", + examples=[{"id": "search_abc", "issue": "Off topic"}], + source="cli", + ) + assert result == _PREVIEW + client.post.assert_called_once_with( + path="/v1beta/feedback", + body={ + "message": _MESSAGE, + "category": "results_quality", + "product": "search", + "severity": "degraded", + "examples": [{"id": "search_abc", "issue": "Off topic"}], + "user_consent": "not_asked", + }, + cast_to=dict, + ) + + def test_consent_round_trip(self, client): + client.get.return_value = {"preference": "ask", "scope": None} + client.put.return_value = {"preference": "always", "scope": "Parallel user u1"} + assert get_feedback_consent()["preference"] == "ask" + assert set_feedback_consent("always")["preference"] == "always" + client.put.assert_called_once_with(path="/v1beta/feedback/consent", body={"preference": "always"}, cast_to=dict) + + +class TestSendCommand: + _ARGS = ["feedback", "send", "-m", _MESSAGE, "--category", "results_quality", "--product", "search"] + + def test_json_without_consent_returns_the_preview_and_sends_nothing(self, runner, client): + client.post.return_value = _PREVIEW + result = runner.invoke(main, [*self._ARGS, "--example", "search_abc:Off topic", "--json"]) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["status"] == "consent_required" + assert [body["user_consent"] for body in _sent_bodies(client)] == ["not_asked"] + + def test_an_agent_sends_with_the_users_consent_and_preview_id(self, runner, client): + client.post.return_value = _RECEIVED + result = runner.invoke(main, [*self._ARGS, "--consent", "this_report", "--preview-id", "p1", "--json"]) + assert result.exit_code == 0, result.output + [body] = _sent_bodies(client) + assert body["user_consent"] == "this_report" + assert body["preview_id"] == "p1" + + @pytest.mark.parametrize(("answer", "consent"), [("s", "this_report"), ("a", "always")]) + def test_a_person_approves_the_preview_at_the_prompt(self, runner, client, answer, consent): + client.post.side_effect = [_PREVIEW, _RECEIVED] + result = runner.invoke(main, self._ARGS, input=f"{answer}\n") + assert result.exit_code == 0, result.output + assert _MESSAGE in result.output + assert "feedback_1" in result.output + second = _sent_bodies(client)[1] + assert second["user_consent"] == consent + assert second["preview_id"] == "p1" + + def test_declining_at_the_prompt_sends_nothing(self, runner, client): + client.post.return_value = _PREVIEW + result = runner.invoke(main, self._ARGS, input="n\n") + assert result.exit_code == 0, result.output + assert len(_sent_bodies(client)) == 1 + assert "Nothing was sent" in result.output + + def test_a_stored_always_sends_on_the_first_call(self, runner, client): + client.post.return_value = {**_RECEIVED, "consent_preference": "always"} + result = runner.invoke(main, self._ARGS) + assert result.exit_code == 0, result.output + assert len(_sent_bodies(client)) == 1 + assert "feedback_1" in result.output + + def test_a_bad_example_is_a_usage_error(self, runner, client): + result = runner.invoke(main, [*self._ARGS, "--example", "has spaces", "--json"]) + assert result.exit_code != 0 + client.post.assert_not_called() + + +class TestConsentCommand: + def test_shows_the_preference(self, runner, client): + client.get.return_value = {"preference": "always", "scope": "Parallel user u1"} + result = runner.invoke(main, ["feedback", "consent", "--json"]) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["preference"] == "always" + + def test_resets_to_ask(self, runner, client): + client.put.return_value = {"preference": "ask", "scope": "Parallel user u1"} + result = runner.invoke(main, ["feedback", "consent", "ask"]) + assert result.exit_code == 0, result.output + client.put.assert_called_once_with(path="/v1beta/feedback/consent", body={"preference": "ask"}, cast_to=dict) From dde9ff9880fa0c458d67a7e662bf6edd48c3961f Mon Sep 17 00:00:00 2001 From: Manvesh Vyas Date: Thu, 22 Oct 2026 16:27:47 -0700 Subject: [PATCH 2/6] docs(cli): define each feedback severity level Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0157KeK2RbX5ARs3g1ZcvmGt --- parallel_web_tools/cli/commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parallel_web_tools/cli/commands.py b/parallel_web_tools/cli/commands.py index 4fd6ef2..8545782 100644 --- a/parallel_web_tools/cli/commands.py +++ b/parallel_web_tools/cli/commands.py @@ -1168,7 +1168,8 @@ def _ask_feedback_consent(response: dict[str, Any]) -> str | None: type=click.Choice(list(FEEDBACK_SEVERITIES)), default="degraded", show_default=True, - help="blocking, degraded (a workaround was needed) or minor.", + help="blocking (the work could not continue), degraded (a workaround was needed) " + "or minor (noticeable, but the work went on unchanged).", ) @click.option( "--example", From 77f87e6429e32422d3b1254c431d825fe0b9e9df Mon Sep 17 00:00:00 2001 From: Manvesh Vyas Date: Thu, 22 Oct 2026 16:29:27 -0700 Subject: [PATCH 3/6] docs(cli): blocking needs no gloss Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0157KeK2RbX5ARs3g1ZcvmGt --- parallel_web_tools/cli/commands.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/parallel_web_tools/cli/commands.py b/parallel_web_tools/cli/commands.py index 8545782..c52da5b 100644 --- a/parallel_web_tools/cli/commands.py +++ b/parallel_web_tools/cli/commands.py @@ -1168,8 +1168,7 @@ def _ask_feedback_consent(response: dict[str, Any]) -> str | None: type=click.Choice(list(FEEDBACK_SEVERITIES)), default="degraded", show_default=True, - help="blocking (the work could not continue), degraded (a workaround was needed) " - "or minor (noticeable, but the work went on unchanged).", + help="blocking, degraded (a workaround was needed) or minor (noticeable, but the work went on unchanged).", ) @click.option( "--example", From 8972d7d81a4fcacd298127ce932b34093d8c40ad Mon Sep 17 00:00:00 2001 From: Manvesh Vyas Date: Mon, 26 Oct 2026 11:22:51 -0700 Subject: [PATCH 4/6] feat(feedback): accept the decisions product Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0157KeK2RbX5ARs3g1ZcvmGt --- parallel_web_tools/core/feedback.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/parallel_web_tools/core/feedback.py b/parallel_web_tools/core/feedback.py index 8e05446..6cd218d 100644 --- a/parallel_web_tools/core/feedback.py +++ b/parallel_web_tools/core/feedback.py @@ -14,7 +14,9 @@ from parallel_web_tools.core.user_agent import ClientSource FeedbackCategory = Literal["results_quality", "error", "latency", "missing_capability", "docs", "billing", "other"] -FeedbackProduct = Literal["search", "extract", "task", "findall", "monitor", "responses", "mcp", "cli", "other"] +FeedbackProduct = Literal[ + "search", "extract", "task", "findall", "monitor", "responses", "decisions", "mcp", "cli", "other" +] FeedbackSeverity = Literal["blocking", "degraded", "minor"] UserConsent = Literal["not_asked", "this_report", "always"] ConsentPreference = Literal["ask", "always"] @@ -35,6 +37,7 @@ "findall", "monitor", "responses", + "decisions", "mcp", "cli", "other", From b9732d5dc55882763b0b32166193adedda4ca7c6 Mon Sep 17 00:00:00 2001 From: Manvesh Vyas Date: Tue, 27 Oct 2026 11:29:45 -0700 Subject: [PATCH 5/6] feat(feedback): send reports without a consent step The API no longer previews reports or stores a consent preference, so feedback send posts once and shows what Parallel received, and the feedback consent command is gone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0157KeK2RbX5ARs3g1ZcvmGt --- README.md | 18 ++-- parallel_web_tools/cli/commands.py | 122 +++++----------------------- parallel_web_tools/core/feedback.py | 39 +-------- tests/test_feedback.py | 100 ++++++----------------- 4 files changed, 52 insertions(+), 227 deletions(-) diff --git a/README.md b/README.md index 928b4bc..78bffcc 100644 --- a/README.md +++ b/README.md @@ -116,8 +116,7 @@ parallel-cli │ ├── evict # Remove one entry from Memory │ └── clear # Permanently clear selected Memory └── feedback # Report problems to the Parallel team - ├── send # Preview a report, then send it with consent - └── consent # Show or set "don't ask again" + └── send # Send a report and show what Parallel received ``` Run `parallel-cli skills install` to install or update agent skills. When using @@ -245,8 +244,8 @@ scope key containing only letters, digits, underscores, or hyphens. ### 7. Report a Problem to Parallel Report significant problems, such as wrong results for specific runs, with the -request IDs attached. Nothing is sent until the user agrees: the CLI shows -exactly what Parallel would receive and asks first. +request IDs attached. The CLI sends the report and shows exactly what Parallel +received. ```bash parallel-cli feedback send \ @@ -254,17 +253,12 @@ parallel-cli feedback send \ --category results_quality --product task --severity blocking \ --example "trun_abc123:Returned John Roe; expected Jane Doe" -# Agents: --json prints the preview without sending. Show it to the user, then -# rerun with their answer and the preview's preview_id. +# Agents: --json prints the response, including what Parallel received. parallel-cli feedback send ... --json -parallel-cli feedback send ... --consent this_report --preview-id --json - -# Show or reset "don't ask again" (personal credentials only) -parallel-cli feedback consent -parallel-cli feedback consent ask ``` -Feedback is not available to organizations on zero or partial data retention. +Feedback is open only to Parallel's own organizations for now, and never to +organizations on zero or partial data retention. ## Non-Interactive Mode (for AI Agents & Scripts) diff --git a/parallel_web_tools/cli/commands.py b/parallel_web_tools/cli/commands.py index c52da5b..be299dd 100644 --- a/parallel_web_tools/cli/commands.py +++ b/parallel_web_tools/cli/commands.py @@ -64,15 +64,12 @@ validate_memory_scope_key, ) from parallel_web_tools.core.feedback import ( - CONSENT_PREFERENCES, FEEDBACK_CATEGORIES, FEEDBACK_PRODUCTS, FEEDBACK_SEVERITIES, FeedbackInputError, - get_feedback_consent, parse_example, send_feedback, - set_feedback_consent, ) # Standalone CLI (PyInstaller) has limited features to reduce bundle size @@ -1113,48 +1110,29 @@ def _example_option(_ctx: click.Context, _param: click.Parameter, values: tuple[ raise click.BadParameter(str(e)) from e -def _render_feedback_preview(response: dict[str, Any]) -> None: - preview = response.get("preview", {}) - context = preview.get("context", {}) - console.print("[bold]Parallel would receive:[/bold]") - console.print( - f" {preview.get('category')} · {preview.get('product')} · {preview.get('severity')}", - markup=False, - ) - console.print(f" Message: {preview.get('message', '')}", markup=False) - for label in ("expected", "actual"): - if preview.get(label): - console.print(f" {label.capitalize()}: {preview[label]}", markup=False) - for example in preview.get("examples", []): - note = f" — {example['issue']}" if example.get("issue") else "" - console.print(f" • {example.get('id')}{note}", markup=False) - console.print(f" From: {context.get('sender', 'unknown')}", markup=False) - if preview.get("redactions"): - console.print(f" [dim]{preview['redactions']} secret(s) removed.[/dim]") - if preview.get("recipient"): - console.print(f" {preview['recipient']}", markup=False, style="dim") - - def _render_feedback_result(response: dict[str, Any]) -> None: if response.get("duplicate"): console.print(f"[yellow]Parallel already received this report as {response.get('feedback_id')}.[/yellow]") else: console.print(f"[bold green]Sent to Parallel as {response.get('feedback_id')}.[/bold green]") - if response.get("consent_preference") == "always": - console.print("[dim]Later reports are sent without asking. Reset with: parallel-cli feedback consent ask[/dim]") - - -def _ask_feedback_consent(response: dict[str, Any]) -> str | None: - """Ask whether to send the previewed report; returns the consent, or None to stop.""" - can_remember = response.get("preview", {}).get("remember_scope") is not None - choices = ["s", "a", "n"] if can_remember else ["s", "n"] + report = response.get("report", {}) + context = report.get("context", {}) console.print( - "Send it? [bold]s[/bold]end" - + (", send and don't ask [bold]a[/bold]gain" if can_remember else "") - + ", or [bold]n[/bold]o" + f" {report.get('category')} · {report.get('product')} · {report.get('severity')}", + markup=False, ) - answer = click.prompt("", type=click.Choice(choices), show_choices=False, prompt_suffix="> ") - return {"s": "this_report", "a": "always"}.get(answer) + console.print(f" Message: {report.get('message', '')}", markup=False) + for label in ("expected", "actual"): + if report.get(label): + console.print(f" {label.capitalize()}: {report[label]}", markup=False) + for example in report.get("examples", []): + note = f" — {example['issue']}" if example.get("issue") else "" + console.print(f" • {example.get('id')}{note}", markup=False) + console.print(f" From: {context.get('sender', 'unknown')}", markup=False) + if report.get("redactions"): + console.print(f" [dim]{report['redactions']} secret(s) removed.[/dim]") + if report.get("recipient"): + console.print(f" {report['recipient']}", markup=False, style="dim") @feedback.command(name="send") @@ -1179,13 +1157,7 @@ def _ask_feedback_consent(response: dict[str, Any]) -> str | None: ) @click.option("--expected", help="The overall expected behavior.") @click.option("--actual", help="The overall observed behavior.") -@click.option( - "--consent", - type=click.Choice(["this_report", "always"]), - help="The user approved sending this report (this_report) and, with always, later ones.", -) -@click.option("--preview-id", help="The preview_id of the preview the user approved.") -@click.option("--json", "output_json", is_flag=True, help="Output JSON to stdout; never prompts.") +@click.option("--json", "output_json", is_flag=True, help="Output JSON to stdout.") def feedback_send( message: str, category: str, @@ -1194,20 +1166,14 @@ def feedback_send( examples: list[dict[str, str]], expected: str | None, actual: str | None, - consent: str | None, - preview_id: str | None, output_json: bool, ): """Report a significant problem, such as wrong results for specific runs. - Nothing is sent until the user agrees. Without --consent the CLI shows - exactly what Parallel would receive and asks; with --json it prints that - preview so an agent can ask the user, then rerun with --consent and - --preview-id. + Sends the report and shows exactly what Parallel received. """ - - def send(user_consent: str, approved_preview: str | None) -> dict[str, Any]: - return send_feedback( + try: + response = send_feedback( message=message, category=category, product=product, @@ -1215,20 +1181,8 @@ def send(user_consent: str, approved_preview: str | None) -> dict[str, Any]: examples=examples, expected=expected, actual=actual, - user_consent=user_consent, - preview_id=approved_preview, source="cli", ) - - try: - response = send(consent or "not_asked", preview_id) - if response.get("status") == "consent_required" and not output_json: - _render_feedback_preview(response) - approved = _ask_feedback_consent(response) - if approved is None: - console.print("Nothing was sent.") - return - response = send(approved, response.get("preview_id")) except FeedbackInputError as e: _handle_error(e, output_json=output_json, exit_code=EXIT_BAD_INPUT, prefix="Invalid feedback") return @@ -1238,46 +1192,10 @@ def send(user_consent: str, approved_preview: str | None) -> dict[str, Any]: if output_json: print(json.dumps(response, indent=2)) - elif response.get("status") == "consent_required": - _render_feedback_preview(response) - console.print(response.get("next_step", "Nothing was sent."), markup=False, style="yellow") else: _render_feedback_result(response) -@feedback.command(name="consent") -@click.argument("preference", required=False, type=click.Choice(list(CONSENT_PREFERENCES))) -@click.option("--json", "output_json", is_flag=True, help="Output JSON to stdout.") -def feedback_consent(preference: str | None, output_json: bool): - """Show or set whether feedback reports are sent without asking. - - With no argument, shows the current preference. 'always' sends reports - without a preview; 'ask' previews each report first. Only personal - credentials can store a preference. - """ - try: - if preference is None: - result = get_feedback_consent(source="cli") - else: - result = set_feedback_consent(preference, source="cli") - except FeedbackInputError as e: - _handle_error(e, output_json=output_json, exit_code=EXIT_BAD_INPUT, prefix="Invalid feedback consent") - return - except Exception as e: - _handle_error(e, output_json=output_json, prefix="Feedback API error") - return - - if output_json: - print(json.dumps(result, indent=2)) - return - if result.get("preference") == "always": - console.print("Feedback reports are sent [bold]without asking[/bold].") - else: - console.print("Each feedback report is [bold]previewed[/bold] and needs approval.") - if result.get("scope"): - console.print(f"Applies to {result['scope']}.", markup=False, style="dim") - - # ============================================================================= # Search Command # ============================================================================= diff --git a/parallel_web_tools/core/feedback.py b/parallel_web_tools/core/feedback.py index 6cd218d..5f9a1ca 100644 --- a/parallel_web_tools/core/feedback.py +++ b/parallel_web_tools/core/feedback.py @@ -1,9 +1,4 @@ -"""Send feedback about Parallel to the Parallel team. - -The API sends nothing until the user consents: a first call returns a preview of -exactly what Parallel would receive, and the user approves it for this report -(``this_report``) or for all later ones (``always``). -""" +"""Send feedback about Parallel to the Parallel team.""" from __future__ import annotations @@ -18,8 +13,6 @@ "search", "extract", "task", "findall", "monitor", "responses", "decisions", "mcp", "cli", "other" ] FeedbackSeverity = Literal["blocking", "degraded", "minor"] -UserConsent = Literal["not_asked", "this_report", "always"] -ConsentPreference = Literal["ask", "always"] FEEDBACK_CATEGORIES: tuple[FeedbackCategory, ...] = ( "results_quality", @@ -43,8 +36,6 @@ "other", ) FEEDBACK_SEVERITIES: tuple[FeedbackSeverity, ...] = ("blocking", "degraded", "minor") -USER_CONSENTS: tuple[UserConsent, ...] = ("not_asked", "this_report", "always") -CONSENT_PREFERENCES: tuple[ConsentPreference, ...] = ("ask", "always") MIN_MESSAGE_CHARS = 20 MAX_MESSAGE_CHARS = 4000 @@ -53,7 +44,6 @@ _EXAMPLE_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,128}$") _FEEDBACK_PATH = "/v1beta/feedback" -_CONSENT_PATH = "/v1beta/feedback/consent" class FeedbackInputError(ValueError): @@ -88,22 +78,15 @@ def send_feedback( examples: list[dict[str, str]] | None = None, expected: str | None = None, actual: str | None = None, - user_consent: str = "not_asked", - preview_id: str | None = None, api_key: str | None = None, source: ClientSource = "python", ) -> dict[str, Any]: - """Preview or send a feedback report. - - With ``user_consent="not_asked"`` and no stored ``always`` preference, the - response has ``status="consent_required"`` and a preview; nothing is sent. - """ + """Send a feedback report; the response shows exactly what Parallel received.""" if not MIN_MESSAGE_CHARS <= len(message) <= MAX_MESSAGE_CHARS: raise FeedbackInputError(f"message must be {MIN_MESSAGE_CHARS}-{MAX_MESSAGE_CHARS} characters") _check_choice(category, FEEDBACK_CATEGORIES, "category") _check_choice(product, FEEDBACK_PRODUCTS, "product") _check_choice(severity, FEEDBACK_SEVERITIES, "severity") - _check_choice(user_consent, USER_CONSENTS, "user_consent") if examples is not None and len(examples) > MAX_EXAMPLES: raise FeedbackInputError(f"examples must be at most {MAX_EXAMPLES}") @@ -119,24 +102,6 @@ def send_feedback( body["expected"] = expected if actual: body["actual"] = actual - body["user_consent"] = user_consent - if preview_id: - body["preview_id"] = preview_id client = create_client(api_key, source) return client.post(path=_FEEDBACK_PATH, body=body, cast_to=dict) - - -def get_feedback_consent(*, api_key: str | None = None, source: ClientSource = "python") -> dict[str, Any]: - """Return whether reports are sent without asking (``always``) or previewed first (``ask``).""" - client = create_client(api_key, source) - return client.get(_CONSENT_PATH, cast_to=dict) - - -def set_feedback_consent( - preference: str, *, api_key: str | None = None, source: ClientSource = "python" -) -> dict[str, Any]: - """Store ``always`` or reset to ``ask``. Only personal credentials can store a preference.""" - _check_choice(preference, CONSENT_PREFERENCES, "preference") - client = create_client(api_key, source) - return client.put(path=_CONSENT_PATH, body={"preference": preference}, cast_to=dict) diff --git a/tests/test_feedback.py b/tests/test_feedback.py index 9794d2a..21bf6dc 100644 --- a/tests/test_feedback.py +++ b/tests/test_feedback.py @@ -10,37 +10,27 @@ from click.testing import CliRunner from parallel_web_tools.cli.commands import main -from parallel_web_tools.core.feedback import ( - FeedbackInputError, - get_feedback_consent, - parse_example, - send_feedback, - set_feedback_consent, -) +from parallel_web_tools.core.feedback import FeedbackInputError, parse_example, send_feedback _MESSAGE = "Search results for our eval queries are mostly irrelevant." -_PREVIEW = { - "status": "consent_required", - "feedback_id": None, - "preview_id": "p1", +_RECEIVED = { + "status": "received", + "feedback_id": "feedback_1", "duplicate": False, - "consent_preference": "ask", - "preview": { + "report": { "message": _MESSAGE, "category": "results_quality", "product": "search", "severity": "degraded", - "examples": [{"id": "search_abc", "link": "unchecked", "issue": "Off topic", "expected": None}], + "examples": [{"id": "search_abc", "issue": "Off topic", "expected": None}], "expected": None, "actual": None, "context": {"org_id": "org", "sender": "user u1 (personal credential)", "surface": "cli"}, "redactions": 0, "recipient": "Parallel's team only.", - "remember_scope": "Parallel user u1, on any of their personal credentials", }, - "next_step": "Nothing was sent.", + "next_step": "Sent.", } -_RECEIVED = {**_PREVIEW, "status": "received", "feedback_id": "feedback_1", "next_step": "Sent."} @pytest.fixture @@ -68,7 +58,6 @@ class TestValidation: ({"product": "spreadsheet"}, "product"), ({"severity": "urgent"}, "severity"), ({"examples": [{"id": f"trun_{n}"} for n in range(51)]}, "examples"), - ({"user_consent": "maybe"}, "user_consent"), ], ) def test_rejects_invalid_reports_before_calling_the_api(self, client, kwargs, message): @@ -91,15 +80,10 @@ def test_rejects_an_example_that_is_not_an_id(self): with pytest.raises(FeedbackInputError, match="example"): parse_example("not an id:note") - def test_only_ask_or_always_can_be_stored(self, client): - with pytest.raises(FeedbackInputError, match="preference"): - set_feedback_consent("sometimes") - client.put.assert_not_called() - class TestApi: def test_send_posts_only_the_fields_given(self, client): - client.post.return_value = _PREVIEW + client.post.return_value = _RECEIVED result = send_feedback( message=_MESSAGE, category="results_quality", @@ -107,7 +91,7 @@ def test_send_posts_only_the_fields_given(self, client): examples=[{"id": "search_abc", "issue": "Off topic"}], source="cli", ) - assert result == _PREVIEW + assert result == _RECEIVED client.post.assert_called_once_with( path="/v1beta/feedback", body={ @@ -116,77 +100,41 @@ def test_send_posts_only_the_fields_given(self, client): "product": "search", "severity": "degraded", "examples": [{"id": "search_abc", "issue": "Off topic"}], - "user_consent": "not_asked", }, cast_to=dict, ) - def test_consent_round_trip(self, client): - client.get.return_value = {"preference": "ask", "scope": None} - client.put.return_value = {"preference": "always", "scope": "Parallel user u1"} - assert get_feedback_consent()["preference"] == "ask" - assert set_feedback_consent("always")["preference"] == "always" - client.put.assert_called_once_with(path="/v1beta/feedback/consent", body={"preference": "always"}, cast_to=dict) - class TestSendCommand: _ARGS = ["feedback", "send", "-m", _MESSAGE, "--category", "results_quality", "--product", "search"] - def test_json_without_consent_returns_the_preview_and_sends_nothing(self, runner, client): - client.post.return_value = _PREVIEW + def test_json_sends_once_and_prints_the_response(self, runner, client): + client.post.return_value = _RECEIVED result = runner.invoke(main, [*self._ARGS, "--example", "search_abc:Off topic", "--json"]) assert result.exit_code == 0, result.output - assert json.loads(result.output)["status"] == "consent_required" - assert [body["user_consent"] for body in _sent_bodies(client)] == ["not_asked"] + assert json.loads(result.output)["feedback_id"] == "feedback_1" + assert len(_sent_bodies(client)) == 1 - def test_an_agent_sends_with_the_users_consent_and_preview_id(self, runner, client): + def test_shows_what_parallel_received(self, runner, client): client.post.return_value = _RECEIVED - result = runner.invoke(main, [*self._ARGS, "--consent", "this_report", "--preview-id", "p1", "--json"]) - assert result.exit_code == 0, result.output - [body] = _sent_bodies(client) - assert body["user_consent"] == "this_report" - assert body["preview_id"] == "p1" - - @pytest.mark.parametrize(("answer", "consent"), [("s", "this_report"), ("a", "always")]) - def test_a_person_approves_the_preview_at_the_prompt(self, runner, client, answer, consent): - client.post.side_effect = [_PREVIEW, _RECEIVED] - result = runner.invoke(main, self._ARGS, input=f"{answer}\n") - assert result.exit_code == 0, result.output - assert _MESSAGE in result.output - assert "feedback_1" in result.output - second = _sent_bodies(client)[1] - assert second["user_consent"] == consent - assert second["preview_id"] == "p1" - - def test_declining_at_the_prompt_sends_nothing(self, runner, client): - client.post.return_value = _PREVIEW - result = runner.invoke(main, self._ARGS, input="n\n") + result = runner.invoke(main, self._ARGS) assert result.exit_code == 0, result.output assert len(_sent_bodies(client)) == 1 - assert "Nothing was sent" in result.output + assert "feedback_1" in result.output + assert _MESSAGE in result.output + assert "search_abc" in result.output - def test_a_stored_always_sends_on_the_first_call(self, runner, client): - client.post.return_value = {**_RECEIVED, "consent_preference": "always"} + def test_a_repeat_says_so(self, runner, client): + client.post.return_value = {**_RECEIVED, "duplicate": True} result = runner.invoke(main, self._ARGS) assert result.exit_code == 0, result.output - assert len(_sent_bodies(client)) == 1 - assert "feedback_1" in result.output + assert "already received" in result.output def test_a_bad_example_is_a_usage_error(self, runner, client): result = runner.invoke(main, [*self._ARGS, "--example", "has spaces", "--json"]) assert result.exit_code != 0 client.post.assert_not_called() - -class TestConsentCommand: - def test_shows_the_preference(self, runner, client): - client.get.return_value = {"preference": "always", "scope": "Parallel user u1"} - result = runner.invoke(main, ["feedback", "consent", "--json"]) - assert result.exit_code == 0, result.output - assert json.loads(result.output)["preference"] == "always" - - def test_resets_to_ask(self, runner, client): - client.put.return_value = {"preference": "ask", "scope": "Parallel user u1"} - result = runner.invoke(main, ["feedback", "consent", "ask"]) - assert result.exit_code == 0, result.output - client.put.assert_called_once_with(path="/v1beta/feedback/consent", body={"preference": "ask"}, cast_to=dict) + def test_there_is_no_consent_command(self, runner, client): + result = runner.invoke(main, ["feedback", "consent"]) + assert result.exit_code != 0 From 2514bb0bea12a9dd16a1ccb9b3048a79590c30be Mon Sep 17 00:00:00 2001 From: Manvesh Vyas Date: Fri, 30 Oct 2026 11:16:49 -0700 Subject: [PATCH 6/6] docs(feedback): say when to report, not just significant problems Matches the Universal MCP send_feedback description. In evals, agents told to report only significant problems noticed misses but rarely reported them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0157KeK2RbX5ARs3g1ZcvmGt --- README.md | 7 ++++--- parallel_web_tools/cli/commands.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 78bffcc..06fb823 100644 --- a/README.md +++ b/README.md @@ -243,9 +243,10 @@ scope key containing only letters, digits, underscores, or hyphens. ### 7. Report a Problem to Parallel -Report significant problems, such as wrong results for specific runs, with the -request IDs attached. The CLI sends the report and shows exactly what Parallel -received. +Report a problem with Parallel, with the request IDs attached: wrong, stale or +missing results, an answer the user says was wrong, a result you had to work +around, an error your request didn't cause, or a benchmark where Parallel +underperformed. The CLI sends the report and shows exactly what Parallel received. ```bash parallel-cli feedback send \ diff --git a/parallel_web_tools/cli/commands.py b/parallel_web_tools/cli/commands.py index be299dd..f4c7a3f 100644 --- a/parallel_web_tools/cli/commands.py +++ b/parallel_web_tools/cli/commands.py @@ -1099,7 +1099,7 @@ def memory_clear(memory_scope_key: str | None, confirm_clear: bool, output_json: @main.group() def feedback(): - """Report significant problems with Parallel to the Parallel team.""" + """Report problems with Parallel to the Parallel team.""" pass @@ -1168,7 +1168,7 @@ def feedback_send( actual: str | None, output_json: bool, ): - """Report a significant problem, such as wrong results for specific runs. + """Report a problem, such as wrong, stale or missing results for specific runs. Sends the report and shows exactly what Parallel received. """