From 8ae402f774671e70e6fad29994434f0e82a86468 Mon Sep 17 00:00:00 2001
From: Copilot <223556219+Copilot@users.noreply.github.com>
Date: Fri, 6 Nov 2026 11:52:52 -0500
Subject: [PATCH 1/4] FIX Make scorer batch evaluation eligibility explicit

Exclude the raw Decision scorer from automatic batch evaluation while keeping its threshold wrapper eligible. Preserve strict failures for unconfigured candidates and empty selections, and document the generic batch-only policy.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
---
 build_scripts/evaluate_scorers.py             |  30 +++-
 doc/code/scoring/4_scorer_metrics.ipynb       |  13 +-
 doc/code/scoring/4_scorer_metrics.py          |  13 +-
 doc/contributing/10_release_process.md        |   6 +
 pyrit/setup/initializers/scorers.py           |   5 +-
 .../build_scripts/test_evaluate_scorers.py    | 158 +++++++++++++++++-
 tests/unit/setup/test_scorer_initializer.py   |   5 +
 7 files changed, 212 insertions(+), 18 deletions(-)

diff --git a/build_scripts/evaluate_scorers.py b/build_scripts/evaluate_scorers.py
index 5ea646c659..5f302e942d 100644
--- a/build_scripts/evaluate_scorers.py
+++ b/build_scripts/evaluate_scorers.py
@@ -28,6 +28,7 @@
 from pyrit.registry import ScorerRegistry
 from pyrit.setup import IN_MEMORY, initialize_pyrit_async
 from pyrit.setup.initializers import ScorerInitializer, TargetInitializer
+from pyrit.setup.initializers.scorers import ScorerInitializerTags
 
 
 async def evaluate_scorers(tags: list[str] | None = None, max_concurrency: int = 5) -> None:
@@ -37,7 +38,7 @@ async def evaluate_scorers(tags: list[str] | None = None, max_concurrency: int =
     This will:
     1. Initialize PyRIT with in-memory database
     2. Register all scorers from ScorerInitializer into the ScorerRegistry
-    3. Iterate through registered scorers (optionally filtered by tags)
+    3. Select registered scorers by tags, excluding those tagged skip_batch_evaluation
     4. Run evaluate_async() on each scorer
     5. Save results to scorer_evals directory
 
@@ -46,13 +47,14 @@ async def evaluate_scorers(tags: list[str] | None = None, max_concurrency: int =
 
     Args:
         tags: Optional list of tags to filter which scorers to evaluate.
-            When provided, only scorers matching any of the tags are evaluated.
-            When None, all scorers are evaluated.
+            When provided, only eligible scorers matching any of the tags are evaluated.
+            When None, all eligible scorers are evaluated. Explicit batch-evaluation
+            exclusions apply in both cases.
         max_concurrency: Maximum number of concurrent scoring requests per scorer.
             Defaults to 5.
 
     Raises:
-        RuntimeError: If any scorer evaluation fails or returns no metrics.
+        RuntimeError: If no eligible scorers are selected, or any evaluation fails or returns no metrics.
     """
     print("Initializing PyRIT...")
     target_init = TargetInitializer()
@@ -76,11 +78,18 @@ async def evaluate_scorers(tags: list[str] | None = None, max_concurrency: int =
     else:
         scorer_names = registry.instances.get_names()
 
+    excluded_names = {
+        entry.name for entry in registry.instances.get_by_tag(tag=ScorerInitializerTags.SKIP_BATCH_EVALUATION)
+    }
+    for scorer_name in scorer_names:
+        if scorer_name in excluded_names:
+            print(f"Skipping {scorer_name}: tagged '{ScorerInitializerTags.SKIP_BATCH_EVALUATION.value}'.")
+    scorer_names = [name for name in scorer_names if name not in excluded_names]
+
     if not scorer_names:
-        print("No scorers registered. Check environment variable configuration.")
-        if tags:
-            print(f"  (filtered by tags: {tags})")
-        return
+        raise RuntimeError(
+            "No eligible scorers selected for evaluation. Check environment variable configuration and tag filters."
+        )
 
     print(f"\nEvaluating {len(scorer_names)} scorer(s)...\n")
 
@@ -164,7 +173,10 @@ def parse_args() -> argparse.Namespace:
         "--tags",
         type=str,
         default=None,
-        help="Comma-separated list of tags to filter which scorers to evaluate (e.g., --tags refusal,default)",
+        help=(
+            "Comma-separated list of tags to filter which scorers to evaluate (e.g., --tags refusal,default). "
+            "Scorers tagged skip_batch_evaluation remain excluded."
+        ),
     )
     parser.add_argument(
         "--max-concurrency",
diff --git a/doc/code/scoring/4_scorer_metrics.ipynb b/doc/code/scoring/4_scorer_metrics.ipynb
index 5150d26f3e..4d50b2a4a9 100644
--- a/doc/code/scoring/4_scorer_metrics.ipynb
+++ b/doc/code/scoring/4_scorer_metrics.ipynb
@@ -792,17 +792,24 @@
     "## Batch Evaluation with `evaluate_scorers.py`\n",
     "\n",
     "While `evaluate_async()` runs evaluations for a single scorer, the `evaluate_scorers.py` script\n",
-    "evaluates **all registered scorers** in bulk. This is useful for benchmarking after changing scorer\n",
+    "evaluates **eligible registered scorers** in bulk. This is useful for benchmarking after changing scorer\n",
     "prompts, adding new variants, or updating human-labeled datasets.\n",
     "\n",
     "The script initializes PyRIT with `ScorerInitializer` (which registers all configured scorers),\n",
-    "then runs `evaluate_async()` on each one. Results are saved to the JSONL registry files in\n",
+    "then runs `evaluate_async()` on each eligible entry. Results are saved to the JSONL registry files in\n",
     "`pyrit/datasets/scorer_evals/`.\n",
     "\n",
+    "Entries tagged `skip_batch_evaluation` are reported as skipped, even when matched by `--tags`.\n",
+    "The raw Decision probability scorer carries this tag; its 0.7-threshold wrapper remains eligible\n",
+    "for the objective benchmark. Both remain available for normal scoring. The tag only controls\n",
+    "batch selection; it does not change the public `evaluate_async()` API.\n",
+    "Missing evaluation mappings on other selected scorers remain errors. The batch finishes the\n",
+    "remaining evaluations before exiting nonzero on failure; selecting no eligible scorers also fails.\n",
+    "\n",
     "### Basic Usage\n",
     "\n",
     "```bash\n",
-    "# Evaluate all registered scorers (long-running — can take hours)\n",
+    "# Evaluate all eligible registered scorers (long-running — can take hours)\n",
     "python -m build_scripts.evaluate_scorers\n",
     "\n",
     "# Evaluate only scorers with specific tags\n",
diff --git a/doc/code/scoring/4_scorer_metrics.py b/doc/code/scoring/4_scorer_metrics.py
index d1c370bf2b..b6f0a8c5b3 100644
--- a/doc/code/scoring/4_scorer_metrics.py
+++ b/doc/code/scoring/4_scorer_metrics.py
@@ -372,17 +372,24 @@
 # ## Batch Evaluation with `evaluate_scorers.py`
 #
 # While `evaluate_async()` runs evaluations for a single scorer, the `evaluate_scorers.py` script
-# evaluates **all registered scorers** in bulk. This is useful for benchmarking after changing scorer
+# evaluates **eligible registered scorers** in bulk. This is useful for benchmarking after changing scorer
 # prompts, adding new variants, or updating human-labeled datasets.
 #
 # The script initializes PyRIT with `ScorerInitializer` (which registers all configured scorers),
-# then runs `evaluate_async()` on each one. Results are saved to the JSONL registry files in
+# then runs `evaluate_async()` on each eligible entry. Results are saved to the JSONL registry files in
 # `pyrit/datasets/scorer_evals/`.
 #
+# Entries tagged `skip_batch_evaluation` are reported as skipped, even when matched by `--tags`.
+# The raw Decision probability scorer carries this tag; its 0.7-threshold wrapper remains eligible
+# for the objective benchmark. Both remain available for normal scoring. The tag only controls
+# batch selection; it does not change the public `evaluate_async()` API.
+# Missing evaluation mappings on other selected scorers remain errors. The batch finishes the
+# remaining evaluations before exiting nonzero on failure; selecting no eligible scorers also fails.
+#
 # ### Basic Usage
 #
 # ```bash
-# # Evaluate all registered scorers (long-running — can take hours)
+# # Evaluate all eligible registered scorers (long-running — can take hours)
 # python -m build_scripts.evaluate_scorers
 #
 # # Evaluate only scorers with specific tags
diff --git a/doc/contributing/10_release_process.md b/doc/contributing/10_release_process.md
index 08d245bdeb..23e0e033ae 100644
--- a/doc/contributing/10_release_process.md
+++ b/doc/contributing/10_release_process.md
@@ -108,6 +108,12 @@ for private vulnerability reporting.
   - **Partner integration tests.** Ensure the partner integration tests are also passing. These tests validate that we are not breaking contracts with partner teams (e.g., Foundry). If any are failing, coordinate with the affected partner teams before proceeding with the release.
   - **Azure key-based auth is disabled in our tenant.** Our Azure subscription has API-key (local) auth turned off, so Azure target integration tests authenticate with Microsoft Entra ID. Tests that run notebooks requiring Azure API keys are deliberately skipped; otherwise they fail with HTTP 403 `AuthenticationTypeDisabled` ("Key based authentication is disabled for this resource"). Do not re-enable key auth for our tenant. When validating a release manually, authenticate Azure targets with Entra (`az login`) rather than API keys.
 - **Update scorer metrics.** Run `python -m build_scripts.evaluate_scorers` and commit the results so that scorer evaluation metrics are up to date.
+  Scorers explicitly tagged `skip_batch_evaluation` are reported as skipped, including
+  when selected through `--tags`. The raw `task_achieved_microsoft_decision` probability
+  scorer carries this tag; its `task_achieved_microsoft_decision_threshold_07` wrapper
+  remains eligible for the objective benchmark. Both remain registered for normal use.
+  Missing evaluation mappings on other selected scorers are errors, not implicit exclusions.
+  An empty eligible selection also exits nonzero.
   Completion may use cached metrics; it does not guarantee fresh scoring.
   Blocked judge requests produce undetermined judgments rather than aborting the evaluation.
   A blocked response being evaluated is different: existing scorer semantics still apply
diff --git a/pyrit/setup/initializers/scorers.py b/pyrit/setup/initializers/scorers.py
index 0d0d264672..c9212f52a6 100644
--- a/pyrit/setup/initializers/scorers.py
+++ b/pyrit/setup/initializers/scorers.py
@@ -69,6 +69,9 @@ class ScorerInitializerTags(str, Enum):
     OBJECTIVE_COMPOSITE = "objective_composite"
     MICROSOFT_DECISION = "microsoft_decision"
 
+    # Batch-evaluation exclusions do not affect availability for normal scoring.
+    SKIP_BATCH_EVALUATION = "skip_batch_evaluation"
+
     # Overall best objective scorer (used by scenarios)
     BEST_OBJECTIVE = "best_objective"
     DEFAULT_OBJECTIVE_SCORER = "default_objective_scorer"
@@ -482,7 +485,7 @@ def _build_decision_scorer() -> DecisionScorer:
             name="task_achieved_microsoft_decision",
             factory=_build_decision_scorer,
             required_targets=[target],
-            tags=decision_tags,
+            tags=[*decision_tags, ScorerInitializerTags.SKIP_BATCH_EVALUATION],
         )
         self._try_register(
             name="task_achieved_microsoft_decision_threshold_07",
diff --git a/tests/unit/build_scripts/test_evaluate_scorers.py b/tests/unit/build_scripts/test_evaluate_scorers.py
index 1bc84a2d49..c9b7b0ab46 100644
--- a/tests/unit/build_scripts/test_evaluate_scorers.py
+++ b/tests/unit/build_scripts/test_evaluate_scorers.py
@@ -16,8 +16,10 @@
 from pyrit.exceptions.exception_classes import handle_bad_request_exception
 from pyrit.memory import SQLiteMemory
 from pyrit.models import Message, MessagePiece
-from pyrit.registry import ScorerRegistry
+from pyrit.registry import ScorerRegistry, TargetRegistry
 from pyrit.score import (
+    DecisionScorer,
+    FloatScaleThresholdScorer,
     ScorerEvalDatasetFiles,
     ScorerMetrics,
     SelfAskRefusalScorer,
@@ -28,6 +30,7 @@
 )
 from pyrit.setup import IN_MEMORY
 from pyrit.setup.initializers import ScorerInitializer, TargetInitializer
+from pyrit.setup.initializers.scorers import ScorerInitializerTags
 from pyrit.setup.initializers.targets import TargetInitializerTags
 
 
@@ -41,6 +44,156 @@ def evaluation_path(tmp_path: Path) -> Path:
     return tmp_path
 
 
+@pytest.fixture
+def decision_registry(patch_central_database: None) -> tuple[ScorerRegistry, MockPromptTarget]:
+    registry = ScorerRegistry()
+    targets = TargetRegistry()
+    target = MockPromptTarget()
+    targets.instances.register(target, name="microsoft_decision")
+    with (
+        patch.object(ScorerRegistry, "get_registry_singleton", return_value=registry),
+        patch.object(TargetRegistry, "get_registry_singleton", return_value=targets),
+    ):
+        ScorerInitializer()._register_decision_scorers()
+    return registry, target
+
+
+@pytest.mark.parametrize("tags", [None, [ScorerInitializerTags.MICROSOFT_DECISION]])
+@pytest.mark.parametrize("evaluation_fails", [False, True])
+async def test_evaluate_scorers_evaluates_only_decision_wrapper_async(
+    *,
+    tags: list[str] | None,
+    evaluation_fails: bool,
+    decision_registry: tuple[ScorerRegistry, MockPromptTarget],
+    evaluation_path: Path,
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    registry, target = decision_registry
+    raw = registry.instances.get("task_achieved_microsoft_decision")
+    wrapped = registry.instances.get("task_achieved_microsoft_decision_threshold_07")
+    assert isinstance(raw, DecisionScorer)
+    assert isinstance(wrapped, FloatScaleThresholdScorer)
+    assert raw.evaluation_file_mapping is None
+    assert wrapped.evaluation_file_mapping is not None
+    wrapped.evaluation_file_mapping = ScorerEvalDatasetFiles(
+        human_labeled_datasets_files=["synthetic.csv"], result_file="metrics.jsonl"
+    )
+
+    async def send_async(*, normalized_conversation: list[Message]) -> list[Message]:
+        if evaluation_fails:
+            raise ValueError("Synthetic decision evaluation failure")
+        request = normalized_conversation[-1].message_pieces[0]
+        return [
+            MessagePiece(
+                role="assistant",
+                original_value='{"answers": {"objective_achieved": {"type": "noul", "noul": 0.83}}}',
+                conversation_id=request.conversation_id,
+            ).to_message()
+        ]
+
+    with (
+        patch.object(evaluate_scorers, "initialize_pyrit_async", new_callable=AsyncMock),
+        patch.object(ScorerRegistry, "get_registry_singleton", return_value=registry),
+        patch("pyrit.score.scorer_evaluation.scorer_evaluator.SCORER_EVALS_PATH", evaluation_path),
+        patch.object(raw, "evaluate_async", new_callable=AsyncMock) as raw_eval,
+        patch.object(target, "_send_prompt_to_target_async", new_callable=AsyncMock, side_effect=send_async) as send,
+    ):
+        if evaluation_fails:
+            with pytest.raises(
+                RuntimeError,
+                match=r"Evaluation failed for 1 scorer\(s\): task_achieved_microsoft_decision_threshold_07$",
+            ):
+                await evaluate_scorers.evaluate_scorers(tags=tags, max_concurrency=1)
+            send.assert_awaited()
+        else:
+            await evaluate_scorers.evaluate_scorers(tags=tags, max_concurrency=1)
+            assert send.await_count == 6
+        raw_eval.assert_not_awaited()
+
+    assert registry.instances.get("task_achieved_microsoft_decision") is raw
+    assert registry.instances.get("task_achieved_microsoft_decision_threshold_07") is wrapped
+    captured = capsys.readouterr()
+    output = captured.out
+    assert "Skipping task_achieved_microsoft_decision: tagged 'skip_batch_evaluation'." in output
+    assert "Evaluating 1 scorer(s)..." in output
+    result_path = evaluation_path / "metrics.jsonl"
+    if evaluation_fails:
+        assert "Synthetic decision evaluation failure" in captured.err
+        assert "Evaluation complete" not in output
+        assert not result_path.exists()
+    else:
+        record = json.loads(await asyncio.to_thread(result_path.read_text, encoding="utf-8"))
+        assert record["metrics"]["num_responses"] == 2
+        assert record["metrics"]["num_input_responses"] == 2
+        assert record["metrics"]["f1_score"] == 1.0
+        assert "Coverage: 2/2 responses evaluated; 0 excluded." in output
+
+
+async def test_evaluate_scorers_does_not_skip_unmarked_missing_mapping_async(
+    *,
+    decision_registry: tuple[ScorerRegistry, MockPromptTarget],
+    evaluation_path: Path,
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    decision_scorers, _ = decision_registry
+    raw = decision_scorers.instances.get("task_achieved_microsoft_decision")
+    assert isinstance(raw, DecisionScorer)
+    succeeding = SubStringScorer(substring="marker")
+    succeeding.evaluation_file_mapping = ScorerEvalDatasetFiles(
+        human_labeled_datasets_files=["synthetic.csv"], result_file="metrics.jsonl"
+    )
+    registry = ScorerRegistry()
+    registry.instances.register(raw, name="a_unconfigured")
+    registry.instances.register(succeeding, name="b_succeeding")
+    with (
+        patch.object(evaluate_scorers, "initialize_pyrit_async", new_callable=AsyncMock),
+        patch.object(ScorerRegistry, "get_registry_singleton", return_value=registry),
+        patch("pyrit.score.scorer_evaluation.scorer_evaluator.SCORER_EVALS_PATH", evaluation_path),
+    ):
+        with pytest.raises(RuntimeError, match=r"Evaluation failed for 1 scorer\(s\): a_unconfigured$"):
+            await evaluate_scorers.evaluate_scorers(max_concurrency=1)
+
+    output = capsys.readouterr().out
+    assert "no evaluation_file_mapping configured for DecisionScorer" in output
+    assert "Skipping" not in output
+    assert "Coverage: 2/2 responses evaluated; 0 excluded." in output
+    assert "Results location:" not in output
+    assert (evaluation_path / "metrics.jsonl").exists()
+
+
+@pytest.mark.usefixtures("patch_central_database")
+@pytest.mark.parametrize(
+    ("registered", "tags"),
+    [(False, None), (True, None), (True, ["missing"]), (True, ["skip_batch_evaluation"])],
+)
+def test_evaluate_scorers_exits_nonzero_without_eligible_scorers(
+    *, registered: bool, tags: list[str] | None, capsys: pytest.CaptureFixture[str]
+) -> None:
+    registry = ScorerRegistry()
+    scorer = SubStringScorer(substring="marker")
+    if registered:
+        registry.instances.register(scorer, name="excluded", tags=["skip_batch_evaluation"])
+    argv = ["evaluate_scorers"]
+    if tags:
+        argv.extend(["--tags", ",".join(tags)])
+
+    with (
+        patch.object(setup, "initialize_pyrit_async", new_callable=AsyncMock),
+        patch.object(ScorerRegistry, "get_registry_singleton", return_value=registry),
+        patch.object(scorer, "evaluate_async", new_callable=AsyncMock) as evaluate,
+        patch.object(sys, "argv", argv),
+    ):
+        with pytest.raises(SystemExit) as exit_info:
+            runpy.run_path(evaluate_scorers.__file__, run_name="__main__")
+
+    assert exit_info.value.code == 1
+    evaluate.assert_not_awaited()
+    output = capsys.readouterr().out
+    assert "No eligible scorers selected for evaluation" in output
+    assert "Evaluation complete" not in output
+    assert ("Skipping excluded:" in output) == (registered and tags != ["missing"])
+
+
 @pytest.mark.usefixtures("patch_central_database")
 @pytest.mark.parametrize(
     ("substring", "blocked_count", "blocked_trials", "excluded_count"),
@@ -219,7 +372,8 @@ async def test_evaluate_scorers_preserves_initializer_defaults_async() -> None:
         patch.object(evaluate_scorers, "initialize_pyrit_async", new_callable=AsyncMock) as initialize_mock,
         patch.object(ScorerRegistry, "get_registry_singleton", return_value=ScorerRegistry()),
     ):
-        await evaluate_scorers.evaluate_scorers()
+        with pytest.raises(RuntimeError, match="No eligible scorers selected for evaluation"):
+            await evaluate_scorers.evaluate_scorers()
 
     initialize_mock.assert_awaited_once()
     assert initialize_mock.await_args is not None
diff --git a/tests/unit/setup/test_scorer_initializer.py b/tests/unit/setup/test_scorer_initializer.py
index b97e45e613..d559b37405 100644
--- a/tests/unit/setup/test_scorer_initializer.py
+++ b/tests/unit/setup/test_scorer_initializer.py
@@ -166,6 +166,11 @@ async def test_decision_scorers_register_without_defaults_async(self) -> None:
         assert wrapped._scorer.get_chat_target() is target
         assert wrapped._scorer.get_identifier() == raw.get_identifier()
         assert wrapped._threshold == 0.7
+        assert raw.evaluation_file_mapping is None
+        assert wrapped.evaluation_file_mapping is not None
+        assert [entry.name for entry in registry.instances.get_by_tag(tag="skip_batch_evaluation")] == [
+            "task_achieved_microsoft_decision"
+        ]
         assert len(registry.instances) == 2
         assert len(registry.instances.get_by_tag(tag=ScorerInitializerTags.MICROSOFT_DECISION)) == 2
         for tag in (

From f5c8ea0e5b1bc2277436e977b55f568cdd4961aa Mon Sep 17 00:00:00 2001
From: Copilot <223556219+Copilot@users.noreply.github.com>
Date: Fri, 6 Nov 2026 12:35:47 -0500
Subject: [PATCH 2/4] FIX Require complete coverage for objective scorer
 ranking

Rank F1 only with known positive complete coverage. Preserve configured fallback choices and tags with explicit diagnostics, and document the selection boundary without changing persisted metrics.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
---
 doc/code/scoring/4_scorer_metrics.ipynb     |  30 ++--
 doc/code/scoring/4_scorer_metrics.py        |  30 ++--
 doc/contributing/10_release_process.md      |   5 +
 pyrit/setup/initializers/scorers.py         |  40 ++++--
 tests/unit/setup/test_scorer_initializer.py | 152 ++++++++++++++++----
 5 files changed, 193 insertions(+), 64 deletions(-)

diff --git a/doc/code/scoring/4_scorer_metrics.ipynb b/doc/code/scoring/4_scorer_metrics.ipynb
index 4d50b2a4a9..2602090386 100644
--- a/doc/code/scoring/4_scorer_metrics.ipynb
+++ b/doc/code/scoring/4_scorer_metrics.ipynb
@@ -825,10 +825,19 @@
     "`ScorerInitializer` applies tags to scorers during registration. These tags let you target\n",
     "specific subsets for evaluation:\n",
     "\n",
-    "- `refusal` — The 4 standalone refusal scorer variants\n",
-    "- `default` — All scorers registered by default\n",
-    "- `best_refusal_f1` — The refusal variant with the highest F1 (set dynamically from metrics)\n",
-    "- `best_objective_f1` — The objective scorer with the highest F1\n",
+    "- `refusal` — Registered standalone refusal scorer variants\n",
+    "- `best_refusal` — The configured preferred refusal variant, or the first available variant\n",
+    "- `best_objective` / `default_objective_scorer` — The objective scorer selected below\n",
+    "\n",
+    "Automatic best-objective selection ranks F1 only among otherwise eligible candidates with\n",
+    "known, positive `num_input_responses` and `num_responses == num_input_responses`.\n",
+    "Partial, unknown, zero, or inconsistent coverage counts cannot qualify for this ranking.\n",
+    "If no metrics qualify, the initializer keeps the configured fallback: `scale_and_refusal`,\n",
+    "then the first available objective composite. It retains both public tags and warns that the\n",
+    "choice is a configured fallback, not an F1-backed winner. With no composite, neither tag is added.\n",
+    "Partial results remain computed, stored, reported, and cached normally; users can still select\n",
+    "those scorers explicitly. Complete coverage does not itself establish that different records\n",
+    "evaluated identical examples.\n",
     "\n",
     "### Recommended Workflow: Refusal → Dependent Scorers\n",
     "\n",
@@ -840,9 +849,9 @@
     "python -m build_scripts.evaluate_scorers --tags refusal\n",
     "```\n",
     "\n",
-    "This evaluates only the 4 refusal variants and writes results to\n",
-    "`refusal_scorer/refusal_metrics.jsonl`. After this step, `ScorerInitializer` can determine which\n",
-    "refusal variant has the best F1 and tag it as `best_refusal_f1`.\n",
+    "This evaluates the registered refusal variants and writes results to\n",
+    "`refusal_scorer/refusal_metrics.jsonl`. The `best_refusal` choice remains configuration-based,\n",
+    "rather than being reranked from these metrics.\n",
     "\n",
     "**Step 2: Re-evaluate all scorers**\n",
     "\n",
@@ -850,10 +859,9 @@
     "python -m build_scripts.evaluate_scorers\n",
     "```\n",
     "\n",
-    "On the next full run, `ScorerInitializer` reads the refusal metrics from Step 1, picks the best\n",
-    "refusal variant, and uses it to build dependent scorers (e.g., `TrueFalseInverterScorer` wrapping\n",
-    "the best refusal scorer). This ensures objective scorers that depend on refusal detection use the\n",
-    "best-performing refusal prompt.\n",
+    "On the next full run, `ScorerInitializer` builds dependent scorers (e.g.,\n",
+    "`TrueFalseInverterScorer`) using the configured `best_refusal` variant.\n",
+    "Overall best-objective selection uses the complete-coverage rule above.\n",
     "\n",
     "Scorers whose metrics are already up-to-date (same dataset version, sufficient trials) are\n",
     "automatically skipped, so re-running the full script is efficient.\n",
diff --git a/doc/code/scoring/4_scorer_metrics.py b/doc/code/scoring/4_scorer_metrics.py
index b6f0a8c5b3..d2fb85680d 100644
--- a/doc/code/scoring/4_scorer_metrics.py
+++ b/doc/code/scoring/4_scorer_metrics.py
@@ -405,10 +405,19 @@
 # `ScorerInitializer` applies tags to scorers during registration. These tags let you target
 # specific subsets for evaluation:
 #
-# - `refusal` — The 4 standalone refusal scorer variants
-# - `default` — All scorers registered by default
-# - `best_refusal_f1` — The refusal variant with the highest F1 (set dynamically from metrics)
-# - `best_objective_f1` — The objective scorer with the highest F1
+# - `refusal` — Registered standalone refusal scorer variants
+# - `best_refusal` — The configured preferred refusal variant, or the first available variant
+# - `best_objective` / `default_objective_scorer` — The objective scorer selected below
+#
+# Automatic best-objective selection ranks F1 only among otherwise eligible candidates with
+# known, positive `num_input_responses` and `num_responses == num_input_responses`.
+# Partial, unknown, zero, or inconsistent coverage counts cannot qualify for this ranking.
+# If no metrics qualify, the initializer keeps the configured fallback: `scale_and_refusal`,
+# then the first available objective composite. It retains both public tags and warns that the
+# choice is a configured fallback, not an F1-backed winner. With no composite, neither tag is added.
+# Partial results remain computed, stored, reported, and cached normally; users can still select
+# those scorers explicitly. Complete coverage does not itself establish that different records
+# evaluated identical examples.
 #
 # ### Recommended Workflow: Refusal → Dependent Scorers
 #
@@ -420,9 +429,9 @@
 # python -m build_scripts.evaluate_scorers --tags refusal
 # ```
 #
-# This evaluates only the 4 refusal variants and writes results to
-# `refusal_scorer/refusal_metrics.jsonl`. After this step, `ScorerInitializer` can determine which
-# refusal variant has the best F1 and tag it as `best_refusal_f1`.
+# This evaluates the registered refusal variants and writes results to
+# `refusal_scorer/refusal_metrics.jsonl`. The `best_refusal` choice remains configuration-based,
+# rather than being reranked from these metrics.
 #
 # **Step 2: Re-evaluate all scorers**
 #
@@ -430,10 +439,9 @@
 # python -m build_scripts.evaluate_scorers
 # ```
 #
-# On the next full run, `ScorerInitializer` reads the refusal metrics from Step 1, picks the best
-# refusal variant, and uses it to build dependent scorers (e.g., `TrueFalseInverterScorer` wrapping
-# the best refusal scorer). This ensures objective scorers that depend on refusal detection use the
-# best-performing refusal prompt.
+# On the next full run, `ScorerInitializer` builds dependent scorers (e.g.,
+# `TrueFalseInverterScorer`) using the configured `best_refusal` variant.
+# Overall best-objective selection uses the complete-coverage rule above.
 #
 # Scorers whose metrics are already up-to-date (same dataset version, sufficient trials) are
 # automatically skipped, so re-running the full script is efficient.
diff --git a/doc/contributing/10_release_process.md b/doc/contributing/10_release_process.md
index 23e0e033ae..f2575bc3f8 100644
--- a/doc/contributing/10_release_process.md
+++ b/doc/contributing/10_release_process.md
@@ -124,6 +124,11 @@ for private vulnerability reporting.
   interpreting metrics as full-dataset results. Legacy cached metrics without input counts
   have unknown coverage. Evaluations with no determined results or other errors still fail:
   the script finishes the remaining scorers, then exits nonzero without reporting overall success.
+  Automatic best-objective selection ranks F1 only when `num_input_responses > 0`
+  and `num_responses == num_input_responses`. Partial and unknown-coverage results
+  remain stored and reusable, but cannot win that ranking. If none qualify, the
+  initializer preserves its configured composite fallback and warns that it is not
+  an F1-backed winner. Explicit scorer selection remains available.
 
 ## 2. Decide the Next Version
 
diff --git a/pyrit/setup/initializers/scorers.py b/pyrit/setup/initializers/scorers.py
index c9212f52a6..93ebe819f0 100644
--- a/pyrit/setup/initializers/scorers.py
+++ b/pyrit/setup/initializers/scorers.py
@@ -598,8 +598,8 @@ def _tag_best_objective(self) -> None:
         """
         Tag the overall best objective scorer.
 
-        Uses metrics-based selection (F1 score) when evaluation data is available,
-        falling back to a hardcoded default composite scorer.
+        Ranks F1 only when a positive input count confirms complete coverage.
+        Otherwise uses a configured composite fallback, not an F1-backed winner.
         """
         scorer_registry = self._get_scorer_registry()
         best_name: str | None = None
@@ -612,7 +612,14 @@ def _tag_best_objective(self) -> None:
             if not eval_hash:
                 continue
             metrics = find_objective_metrics_by_eval_hash(eval_hash=eval_hash)
-            if metrics is not None and metrics.f1_score > best_f1:
+            if (
+                metrics is None
+                or metrics.num_input_responses is None
+                or metrics.num_input_responses <= 0
+                or metrics.num_responses != metrics.num_input_responses
+            ):
+                continue
+            if metrics.f1_score > best_f1:
                 best_f1 = metrics.f1_score
                 best_name = entry.name
 
@@ -621,21 +628,32 @@ def _tag_best_objective(self) -> None:
                 name=best_name,
                 tags=[ScorerInitializerTags.BEST_OBJECTIVE, ScorerInitializerTags.DEFAULT_OBJECTIVE_SCORER],
             )
-            logger.info(f"Tagged {best_name} as {ScorerInitializerTags.BEST_OBJECTIVE} with F1={best_f1:.4f}")
+            logger.info(
+                f"Tagged {best_name} as {ScorerInitializerTags.BEST_OBJECTIVE} "
+                f"with F1={best_f1:.4f} on complete-coverage metrics"
+            )
             return
 
         # Fall back: prefer scale_and_refusal, then first composite
         best_tags: list[str] = [ScorerInitializerTags.BEST_OBJECTIVE, ScorerInitializerTags.DEFAULT_OBJECTIVE_SCORER]
         if scorer_registry.instances.get_entry(self.SCALE_AND_REFUSAL):
-            scorer_registry.instances.add_tags(name=self.SCALE_AND_REFUSAL, tags=best_tags)
-            logger.info(f"Tagged {self.SCALE_AND_REFUSAL} as {ScorerInitializerTags.BEST_OBJECTIVE} (default)")
+            fallback_name = self.SCALE_AND_REFUSAL
         else:
             composites = scorer_registry.instances.get_by_tag(tag=ScorerInitializerTags.OBJECTIVE_COMPOSITE)
-            if composites:
-                scorer_registry.instances.add_tags(name=composites[0].name, tags=best_tags)
-                logger.info(f"Tagged {composites[0].name} as {ScorerInitializerTags.BEST_OBJECTIVE} (fallback)")
-            else:
-                logger.warning("No composite scorers available; skipping best objective tagging.")
+            fallback_name = composites[0].name if composites else None
+        if fallback_name is not None:
+            scorer_registry.instances.add_tags(name=fallback_name, tags=best_tags)
+            logger.warning(
+                "No complete-coverage metrics qualified; tagged %s as %s using the "
+                "configured fallback, not an F1-backed winner.",
+                fallback_name,
+                ScorerInitializerTags.BEST_OBJECTIVE,
+            )
+        else:
+            logger.warning(
+                "No complete-coverage metrics qualified and no composite scorers available; "
+                "skipping best objective tagging."
+            )
 
     # ---------------------------------------------------------------------------
     # Helpers
diff --git a/tests/unit/setup/test_scorer_initializer.py b/tests/unit/setup/test_scorer_initializer.py
index d559b37405..67b7a13f8c 100644
--- a/tests/unit/setup/test_scorer_initializer.py
+++ b/tests/unit/setup/test_scorer_initializer.py
@@ -1,6 +1,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
+import logging
 import os
 from threading import get_ident
 from unittest.mock import MagicMock, patch
@@ -9,7 +10,13 @@
 
 from pyrit.prompt_target import LiteLLMDecisionTarget, OpenAIChatTarget
 from pyrit.registry import ScorerRegistry, TargetRegistry
-from pyrit.score import DecisionScorer, FloatScaleThresholdScorer, LikertScalePaths
+from pyrit.score import (
+    DecisionScorer,
+    FloatScaleThresholdScorer,
+    LikertScalePaths,
+    ObjectiveScorerMetrics,
+    SubStringScorer,
+)
 from pyrit.setup.initializers import ScorerInitializer
 from pyrit.setup.initializers.scorers import (
     GPT4O_TARGET,
@@ -24,6 +31,21 @@
 )
 
 
+def _objective_metrics(
+    *, f1: float, num_responses: int = 100, num_input_responses: int | None = None
+) -> ObjectiveScorerMetrics:
+    return ObjectiveScorerMetrics(
+        num_responses=num_responses,
+        num_input_responses=num_input_responses,
+        num_human_raters=1,
+        accuracy=f1,
+        accuracy_standard_error=0.0,
+        f1_score=f1,
+        precision=f1,
+        recall=f1,
+    )
+
+
 async def test_scorer_registration_runs_off_loop_async() -> None:
     initializer = ScorerInitializer()
     backend_thread = get_ident()
@@ -344,13 +366,11 @@ def _register_mock_target(self, *, name: str, underlying_model: str = "gpt-4o")
         return target
 
     @patch("pyrit.setup.initializers.scorers.find_objective_metrics_by_eval_hash")
-    async def test_best_objective_tags_best_scorer(self, mock_find_metrics) -> None:
+    async def test_best_objective_tags_best_scorer(self, mock_find_metrics: MagicMock) -> None:
         """Test that _tag_best_objective tags the scorer with highest F1."""
         self._register_mock_target(name=GPT4O_TARGET)
 
-        mock_metrics = MagicMock()
-        mock_metrics.f1_score = 0.85
-        mock_find_metrics.return_value = mock_metrics
+        mock_find_metrics.return_value = _objective_metrics(f1=0.85, num_input_responses=100)
 
         init = ScorerInitializer()
         await init.initialize_async()
@@ -378,38 +398,108 @@ async def test_best_objective_no_metrics_falls_back_to_category(self, mock_find_
         else:
             assert len(results) == 0
 
-    @patch("pyrit.setup.initializers.scorers.find_objective_metrics_by_eval_hash")
-    async def test_best_objective_picks_highest_f1(self, mock_find_metrics) -> None:
-        """Test that the scorer with the highest F1 score gets tagged."""
-        self._register_mock_target(name=GPT4O_TARGET)
-        self._register_mock_target(name=GPT4O_TEMP9_TARGET)
-
-        def mock_metrics_by_hash(*, eval_hash: str, file_path=None) -> MagicMock | None:
-            metrics = MagicMock()
-            if "refusal" in eval_hash.lower() if eval_hash else False:
-                metrics.f1_score = 0.5
-                return metrics
-            metrics.f1_score = 0.9
-            return metrics
-
-        mock_find_metrics.side_effect = mock_metrics_by_hash
-
-        init = ScorerInitializer()
-        await init.initialize_async()
-
+    @pytest.mark.parametrize("names", [("lower", "higher"), ("higher", "lower")])
+    def test_best_objective_picks_highest_f1(self, *, names: tuple[str, str], caplog: pytest.LogCaptureFixture) -> None:
         registry = ScorerRegistry.get_registry_singleton()
-        results = registry.instances.get_by_tag(tag=ScorerInitializerTags.BEST_OBJECTIVE)
-        assert len(results) == 1
-        assert ScorerInitializerTags.DEFAULT_OBJECTIVE_SCORER in results[0].tags
+        metrics_by_hash = {}
+        for name in names:
+            scorer = SubStringScorer(substring=name)
+            registry.instances.register(scorer, name=name)
+            metrics_by_hash[scorer.get_identifier().eval_hash] = _objective_metrics(
+                f1=0.9 if name == "higher" else 0.8, num_input_responses=100
+            )
+
+        with (
+            patch(
+                "pyrit.setup.initializers.scorers.find_objective_metrics_by_eval_hash",
+                side_effect=lambda *, eval_hash: metrics_by_hash[eval_hash],
+            ),
+            caplog.at_level(logging.INFO, logger="pyrit.setup.initializers.scorers"),
+        ):
+            ScorerInitializer()._tag_best_objective()
+
+        for tag in (ScorerInitializerTags.BEST_OBJECTIVE, ScorerInitializerTags.DEFAULT_OBJECTIVE_SCORER):
+            assert [entry.name for entry in registry.instances.get_by_tag(tag=tag)] == ["higher"]
+        assert "F1=0.9000" in caplog.text
+        assert "configured fallback" not in caplog.text
+
+    @pytest.mark.parametrize(
+        ("num_responses", "num_input_responses"),
+        [(20, 100), (100, None), (0, 0), (-1, -1), (101, 100), (-1, 100), (0, 100), (100, -1)],
+        ids=["partial", "unknown", "zero", "negative", "overfull", "negative-answered", "unanswered", "negative-input"],
+    )
+    def test_best_objective_rejects_incomplete_coverage(
+        self, *, num_responses: int, num_input_responses: int | None
+    ) -> None:
+        registry = ScorerRegistry.get_registry_singleton()
+        incomplete = SubStringScorer(substring="incomplete")
+        complete = SubStringScorer(substring="complete")
+        registry.instances.register(incomplete, name="incomplete")
+        registry.instances.register(complete, name="complete")
+        metrics = _objective_metrics(f1=1.0, num_responses=num_responses, num_input_responses=num_input_responses)
+        metrics_by_hash = {
+            incomplete.get_identifier().eval_hash: metrics,
+            complete.get_identifier().eval_hash: _objective_metrics(f1=0.9, num_input_responses=100),
+        }
+
+        with patch(
+            "pyrit.setup.initializers.scorers.find_objective_metrics_by_eval_hash",
+            side_effect=lambda *, eval_hash: metrics_by_hash[eval_hash],
+        ):
+            ScorerInitializer()._tag_best_objective()
+
+        for tag in (ScorerInitializerTags.BEST_OBJECTIVE, ScorerInitializerTags.DEFAULT_OBJECTIVE_SCORER):
+            assert [entry.name for entry in registry.instances.get_by_tag(tag=tag)] == ["complete"]
+        assert registry.instances.get("incomplete") is incomplete
+        assert metrics.num_responses == num_responses
+        assert metrics.num_input_responses == num_input_responses
+        assert metrics.f1_score == 1.0
+
+    @pytest.mark.parametrize("fallback", [ScorerInitializer.SCALE_AND_REFUSAL, "other_composite", None])
+    @pytest.mark.parametrize("coverage", ["missing", "unknown", "partial", "zero"])
+    def test_best_objective_without_complete_metrics_uses_configured_fallback(
+        self, *, fallback: str | None, coverage: str, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        registry = ScorerRegistry.get_registry_singleton()
+        registry.instances.register(SubStringScorer(substring="candidate"), name="candidate")
+        if fallback is not None:
+            registry.instances.register(
+                SubStringScorer(substring="other"),
+                name="other_composite",
+                tags=[ScorerInitializerTags.OBJECTIVE_COMPOSITE],
+            )
+            if fallback == ScorerInitializer.SCALE_AND_REFUSAL:
+                registry.instances.register(
+                    SubStringScorer(substring="preferred"),
+                    name=fallback,
+                    tags=[ScorerInitializerTags.OBJECTIVE_COMPOSITE],
+                )
+        metrics = {
+            "missing": None,
+            "unknown": _objective_metrics(f1=1.0),
+            "partial": _objective_metrics(f1=1.0, num_responses=20, num_input_responses=100),
+            "zero": _objective_metrics(f1=1.0, num_responses=0, num_input_responses=0),
+        }[coverage]
+        with patch("pyrit.setup.initializers.scorers.find_objective_metrics_by_eval_hash", return_value=metrics):
+            ScorerInitializer()._tag_best_objective()
+
+        for tag in (ScorerInitializerTags.BEST_OBJECTIVE, ScorerInitializerTags.DEFAULT_OBJECTIVE_SCORER):
+            assert [entry.name for entry in registry.instances.get_by_tag(tag=tag)] == (
+                [fallback] if fallback is not None else []
+            )
+        assert "No complete-coverage metrics qualified" in caplog.text
+        if fallback is not None:
+            assert fallback in caplog.text
+            assert "configured fallback, not an F1-backed winner" in caplog.text
+        else:
+            assert "no composite scorers available" in caplog.text
 
     @patch("pyrit.setup.initializers.scorers.find_objective_metrics_by_eval_hash")
-    async def test_best_objective_does_not_add_extra_entry(self, mock_find_metrics) -> None:
+    async def test_best_objective_does_not_add_extra_entry(self, mock_find_metrics: MagicMock) -> None:
         """Test that tagging best objective doesn't increase registry count."""
         self._register_mock_target(name=GPT4O_TARGET)
 
-        mock_metrics = MagicMock()
-        mock_metrics.f1_score = 0.85
-        mock_find_metrics.return_value = mock_metrics
+        mock_find_metrics.return_value = _objective_metrics(f1=0.85, num_input_responses=100)
 
         init = ScorerInitializer()
         await init.initialize_async()

From bde0eca76bd42b2425860b8cbcb089028a40caae Mon Sep 17 00:00:00 2001
From: Copilot <223556219+Copilot@users.noreply.github.com>
Date: Fri, 6 Nov 2026 12:41:03 -0500
Subject: [PATCH 3/4] DOC Shorten scorer evaluation selection guidance

Keep batch exclusion, complete-coverage ranking, and configured fallback guidance concise in paired notebook sources. Preserve code cells and saved outputs.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
---
 doc/code/scoring/4_scorer_metrics.ipynb | 38 +++++++------------------
 doc/code/scoring/4_scorer_metrics.py    | 38 +++++++------------------
 2 files changed, 20 insertions(+), 56 deletions(-)

diff --git a/doc/code/scoring/4_scorer_metrics.ipynb b/doc/code/scoring/4_scorer_metrics.ipynb
index 2602090386..66780b17a6 100644
--- a/doc/code/scoring/4_scorer_metrics.ipynb
+++ b/doc/code/scoring/4_scorer_metrics.ipynb
@@ -791,20 +791,12 @@
    "source": [
     "## Batch Evaluation with `evaluate_scorers.py`\n",
     "\n",
-    "While `evaluate_async()` runs evaluations for a single scorer, the `evaluate_scorers.py` script\n",
-    "evaluates **eligible registered scorers** in bulk. This is useful for benchmarking after changing scorer\n",
-    "prompts, adding new variants, or updating human-labeled datasets.\n",
+    "`evaluate_scorers.py` initializes registered scorers and benchmarks eligible entries in bulk.\n",
+    "Results are saved under `pyrit/datasets/scorer_evals/`.\n",
     "\n",
-    "The script initializes PyRIT with `ScorerInitializer` (which registers all configured scorers),\n",
-    "then runs `evaluate_async()` on each eligible entry. Results are saved to the JSONL registry files in\n",
-    "`pyrit/datasets/scorer_evals/`.\n",
-    "\n",
-    "Entries tagged `skip_batch_evaluation` are reported as skipped, even when matched by `--tags`.\n",
-    "The raw Decision probability scorer carries this tag; its 0.7-threshold wrapper remains eligible\n",
-    "for the objective benchmark. Both remain available for normal scoring. The tag only controls\n",
-    "batch selection; it does not change the public `evaluate_async()` API.\n",
-    "Missing evaluation mappings on other selected scorers remain errors. The batch finishes the\n",
-    "remaining evaluations before exiting nonzero on failure; selecting no eligible scorers also fails.\n",
+    "`skip_batch_evaluation` excludes an entry even when matched by `--tags`. This skips raw Decision,\n",
+    "not its 0.7-threshold wrapper; both remain usable for scoring.\n",
+    "Missing evaluation configuration, evaluation failures, and empty eligible selections still fail the batch.\n",
     "\n",
     "### Basic Usage\n",
     "\n",
@@ -829,15 +821,9 @@
     "- `best_refusal` — The configured preferred refusal variant, or the first available variant\n",
     "- `best_objective` / `default_objective_scorer` — The objective scorer selected below\n",
     "\n",
-    "Automatic best-objective selection ranks F1 only among otherwise eligible candidates with\n",
-    "known, positive `num_input_responses` and `num_responses == num_input_responses`.\n",
-    "Partial, unknown, zero, or inconsistent coverage counts cannot qualify for this ranking.\n",
-    "If no metrics qualify, the initializer keeps the configured fallback: `scale_and_refusal`,\n",
-    "then the first available objective composite. It retains both public tags and warns that the\n",
-    "choice is a configured fallback, not an F1-backed winner. With no composite, neither tag is added.\n",
-    "Partial results remain computed, stored, reported, and cached normally; users can still select\n",
-    "those scorers explicitly. Complete coverage does not itself establish that different records\n",
-    "evaluated identical examples.\n",
+    "F1 ranking requires a known positive input count with every response evaluated.\n",
+    "If none qualify, selection warns and uses the configured fallback (`scale_and_refusal`,\n",
+    "then the first available composite), not an F1-backed winner.\n",
     "\n",
     "### Recommended Workflow: Refusal → Dependent Scorers\n",
     "\n",
@@ -849,9 +835,7 @@
     "python -m build_scripts.evaluate_scorers --tags refusal\n",
     "```\n",
     "\n",
-    "This evaluates the registered refusal variants and writes results to\n",
-    "`refusal_scorer/refusal_metrics.jsonl`. The `best_refusal` choice remains configuration-based,\n",
-    "rather than being reranked from these metrics.\n",
+    "Results are saved to `refusal_scorer/refusal_metrics.jsonl`.\n",
     "\n",
     "**Step 2: Re-evaluate all scorers**\n",
     "\n",
@@ -859,9 +843,7 @@
     "python -m build_scripts.evaluate_scorers\n",
     "```\n",
     "\n",
-    "On the next full run, `ScorerInitializer` builds dependent scorers (e.g.,\n",
-    "`TrueFalseInverterScorer`) using the configured `best_refusal` variant.\n",
-    "Overall best-objective selection uses the complete-coverage rule above.\n",
+    "Dependent scorers use the configured `best_refusal` variant.\n",
     "\n",
     "Scorers whose metrics are already up-to-date (same dataset version, sufficient trials) are\n",
     "automatically skipped, so re-running the full script is efficient.\n",
diff --git a/doc/code/scoring/4_scorer_metrics.py b/doc/code/scoring/4_scorer_metrics.py
index d2fb85680d..89892b354d 100644
--- a/doc/code/scoring/4_scorer_metrics.py
+++ b/doc/code/scoring/4_scorer_metrics.py
@@ -371,20 +371,12 @@
 # %% [markdown]
 # ## Batch Evaluation with `evaluate_scorers.py`
 #
-# While `evaluate_async()` runs evaluations for a single scorer, the `evaluate_scorers.py` script
-# evaluates **eligible registered scorers** in bulk. This is useful for benchmarking after changing scorer
-# prompts, adding new variants, or updating human-labeled datasets.
+# `evaluate_scorers.py` initializes registered scorers and benchmarks eligible entries in bulk.
+# Results are saved under `pyrit/datasets/scorer_evals/`.
 #
-# The script initializes PyRIT with `ScorerInitializer` (which registers all configured scorers),
-# then runs `evaluate_async()` on each eligible entry. Results are saved to the JSONL registry files in
-# `pyrit/datasets/scorer_evals/`.
-#
-# Entries tagged `skip_batch_evaluation` are reported as skipped, even when matched by `--tags`.
-# The raw Decision probability scorer carries this tag; its 0.7-threshold wrapper remains eligible
-# for the objective benchmark. Both remain available for normal scoring. The tag only controls
-# batch selection; it does not change the public `evaluate_async()` API.
-# Missing evaluation mappings on other selected scorers remain errors. The batch finishes the
-# remaining evaluations before exiting nonzero on failure; selecting no eligible scorers also fails.
+# `skip_batch_evaluation` excludes an entry even when matched by `--tags`. This skips raw Decision,
+# not its 0.7-threshold wrapper; both remain usable for scoring.
+# Missing evaluation configuration, evaluation failures, and empty eligible selections still fail the batch.
 #
 # ### Basic Usage
 #
@@ -409,15 +401,9 @@
 # - `best_refusal` — The configured preferred refusal variant, or the first available variant
 # - `best_objective` / `default_objective_scorer` — The objective scorer selected below
 #
-# Automatic best-objective selection ranks F1 only among otherwise eligible candidates with
-# known, positive `num_input_responses` and `num_responses == num_input_responses`.
-# Partial, unknown, zero, or inconsistent coverage counts cannot qualify for this ranking.
-# If no metrics qualify, the initializer keeps the configured fallback: `scale_and_refusal`,
-# then the first available objective composite. It retains both public tags and warns that the
-# choice is a configured fallback, not an F1-backed winner. With no composite, neither tag is added.
-# Partial results remain computed, stored, reported, and cached normally; users can still select
-# those scorers explicitly. Complete coverage does not itself establish that different records
-# evaluated identical examples.
+# F1 ranking requires a known positive input count with every response evaluated.
+# If none qualify, selection warns and uses the configured fallback (`scale_and_refusal`,
+# then the first available composite), not an F1-backed winner.
 #
 # ### Recommended Workflow: Refusal → Dependent Scorers
 #
@@ -429,9 +415,7 @@
 # python -m build_scripts.evaluate_scorers --tags refusal
 # ```
 #
-# This evaluates the registered refusal variants and writes results to
-# `refusal_scorer/refusal_metrics.jsonl`. The `best_refusal` choice remains configuration-based,
-# rather than being reranked from these metrics.
+# Results are saved to `refusal_scorer/refusal_metrics.jsonl`.
 #
 # **Step 2: Re-evaluate all scorers**
 #
@@ -439,9 +423,7 @@
 # python -m build_scripts.evaluate_scorers
 # ```
 #
-# On the next full run, `ScorerInitializer` builds dependent scorers (e.g.,
-# `TrueFalseInverterScorer`) using the configured `best_refusal` variant.
-# Overall best-objective selection uses the complete-coverage rule above.
+# Dependent scorers use the configured `best_refusal` variant.
 #
 # Scorers whose metrics are already up-to-date (same dataset version, sufficient trials) are
 # automatically skipped, so re-running the full script is efficient.

From 8a71727fb4793d8aea72e20538fd935ae7b1bf0c Mon Sep 17 00:00:00 2001
From: Copilot <223556219+Copilot@users.noreply.github.com>
Date: Sat, 7 Nov 2026 12:22:39 -0500
Subject: [PATCH 4/4] MAINT Export scorer initializer tags from the package

Shorten the batch evaluator import via the lazy initializer package export and cover its resolution and caching contract.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
---
 build_scripts/evaluate_scorers.py                 | 3 +--
 pyrit/setup/initializers/__init__.py              | 3 ++-
 tests/unit/build_scripts/test_evaluate_scorers.py | 3 +--
 tests/unit/common/test_lazy_package_imports.py    | 6 ++++++
 tests/unit/setup/test_scorer_initializer.py       | 3 +--
 5 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/build_scripts/evaluate_scorers.py b/build_scripts/evaluate_scorers.py
index 5f302e942d..13b895f443 100644
--- a/build_scripts/evaluate_scorers.py
+++ b/build_scripts/evaluate_scorers.py
@@ -27,8 +27,7 @@
 from pyrit.exceptions.exception_context import ComponentRole, execution_context
 from pyrit.registry import ScorerRegistry
 from pyrit.setup import IN_MEMORY, initialize_pyrit_async
-from pyrit.setup.initializers import ScorerInitializer, TargetInitializer
-from pyrit.setup.initializers.scorers import ScorerInitializerTags
+from pyrit.setup.initializers import ScorerInitializer, ScorerInitializerTags, TargetInitializer
 
 
 async def evaluate_scorers(tags: list[str] | None = None, max_concurrency: int = 5) -> None:
diff --git a/pyrit/setup/initializers/__init__.py b/pyrit/setup/initializers/__init__.py
index d5693e3c80..a57c937604 100644
--- a/pyrit/setup/initializers/__init__.py
+++ b/pyrit/setup/initializers/__init__.py
@@ -14,7 +14,7 @@
     from pyrit.setup.initializers.load_default_datasets import LoadDefaultDatasets
     from pyrit.setup.initializers.preload_scenario_metadata import PreloadScenarioMetadata
     from pyrit.setup.initializers.refresh_datasets import RefreshDatasets
-    from pyrit.setup.initializers.scorers import ScorerInitializer
+    from pyrit.setup.initializers.scorers import ScorerInitializer, ScorerInitializerTags
     from pyrit.setup.initializers.targets import TargetInitializer
     from pyrit.setup.initializers.techniques import TechniqueInitializer
     from pyrit.setup.initializers.tools import ToolsInitializer
@@ -27,6 +27,7 @@
     "TechniqueInitializer": "pyrit.setup.initializers.techniques",
     "ToolsInitializer": "pyrit.setup.initializers.tools",
     "ScorerInitializer": "pyrit.setup.initializers.scorers",
+    "ScorerInitializerTags": "pyrit.setup.initializers.scorers",
     "TargetInitializer": "pyrit.setup.initializers.targets",
     "LoadDefaultDatasets": "pyrit.setup.initializers.load_default_datasets",
     "PreloadScenarioMetadata": "pyrit.setup.initializers.preload_scenario_metadata",
diff --git a/tests/unit/build_scripts/test_evaluate_scorers.py b/tests/unit/build_scripts/test_evaluate_scorers.py
index c9b7b0ab46..9b4f28f639 100644
--- a/tests/unit/build_scripts/test_evaluate_scorers.py
+++ b/tests/unit/build_scripts/test_evaluate_scorers.py
@@ -29,8 +29,7 @@
     TrueFalseScoreAggregator,
 )
 from pyrit.setup import IN_MEMORY
-from pyrit.setup.initializers import ScorerInitializer, TargetInitializer
-from pyrit.setup.initializers.scorers import ScorerInitializerTags
+from pyrit.setup.initializers import ScorerInitializer, ScorerInitializerTags, TargetInitializer
 from pyrit.setup.initializers.targets import TargetInitializerTags
 
 
diff --git a/tests/unit/common/test_lazy_package_imports.py b/tests/unit/common/test_lazy_package_imports.py
index 655f73bed9..4288921e8e 100644
--- a/tests/unit/common/test_lazy_package_imports.py
+++ b/tests/unit/common/test_lazy_package_imports.py
@@ -196,6 +196,12 @@
         "pyrit.score.observation.otel_trace_source",
         "pyrit.score.observation.otel_span_exporter",
     ),
+    (
+        "pyrit.setup.initializers",
+        "ScorerInitializerTags",
+        "pyrit.setup.initializers.scorers",
+        "pyrit.setup.initializers.tools",
+    ),
 ]
 
 
diff --git a/tests/unit/setup/test_scorer_initializer.py b/tests/unit/setup/test_scorer_initializer.py
index 67b7a13f8c..7a5d7f3ffc 100644
--- a/tests/unit/setup/test_scorer_initializer.py
+++ b/tests/unit/setup/test_scorer_initializer.py
@@ -17,7 +17,7 @@
     ObjectiveScorerMetrics,
     SubStringScorer,
 )
-from pyrit.setup.initializers import ScorerInitializer
+from pyrit.setup.initializers import ScorerInitializer, ScorerInitializerTags
 from pyrit.setup.initializers.scorers import (
     GPT4O_TARGET,
     GPT4O_TEMP0_TARGET,
@@ -27,7 +27,6 @@
     GPT4O_UNSAFE_TEMP9_TARGET,
     GPT5_1_TARGET,
     GPT5_4_TARGET,
-    ScorerInitializerTags,
 )