FIX Make scorer batch evaluation eligibility explicit

build_scripts/evaluate_scorers.py

@@ -27,7 +27,7 @@
 from pyrit.exceptions import ComponentRole, execution_context
 from pyrit.registry import ScorerRegistry
 from pyrit.setup import IN_MEMORY, initialize_pyrit_async
-from pyrit.setup.initializers import ScorerInitializer, TargetInitializer
+from pyrit.setup.initializers import ScorerInitializer, ScorerInitializerTags, TargetInitializer
 
 
 async def evaluate_scorers(tags: list[str] | None = None, max_concurrency: int = 5) -> None:
@@ -37,7 +37,7 @@ async def evaluate_scorers(tags: list[str] | None = None, max_concurrency: int =
     This will:
     1. Initialize PyRIT with in-memory database
     2. Register all scorers from ScorerInitializer into the ScorerRegistry
-    3. Iterate through registered scorers (optionally filtered by tags)
+    3. Select registered scorers by tags, excluding those tagged skip_batch_evaluation
     4. Run evaluate_async() on each scorer
     5. Save results to scorer_evals directory
 
@@ -46,13 +46,14 @@ async def evaluate_scorers(tags: list[str] | None = None, max_concurrency: int =
 
     Args:
         tags: Optional list of tags to filter which scorers to evaluate.
-            When provided, only scorers matching any of the tags are evaluated.
-            When None, all scorers are evaluated.
+            When provided, only eligible scorers matching any of the tags are evaluated.
+            When None, all eligible scorers are evaluated. Explicit batch-evaluation
+            exclusions apply in both cases.
         max_concurrency: Maximum number of concurrent scoring requests per scorer.
             Defaults to 5.
 
     Raises:
-        RuntimeError: If any scorer evaluation fails or returns no metrics.
+        RuntimeError: If no eligible scorers are selected, or any evaluation fails or returns no metrics.
     """
     print("Initializing PyRIT...")
     target_init = TargetInitializer()
@@ -76,11 +77,18 @@ async def evaluate_scorers(tags: list[str] | None = None, max_concurrency: int =
     else:
         scorer_names = registry.instances.get_names()
 
+    excluded_names = {
+        entry.name for entry in registry.instances.get_by_tag(tag=ScorerInitializerTags.SKIP_BATCH_EVALUATION)
+    }
+    for scorer_name in scorer_names:
+        if scorer_name in excluded_names:
+            print(f"Skipping {scorer_name}: tagged '{ScorerInitializerTags.SKIP_BATCH_EVALUATION.value}'.")
+    scorer_names = [name for name in scorer_names if name not in excluded_names]
+
     if not scorer_names:
-        print("No scorers registered. Check environment variable configuration.")
-        if tags:
-            print(f"  (filtered by tags: {tags})")
-        return
+        raise RuntimeError(
+            "No eligible scorers selected for evaluation. Check environment variable configuration and tag filters."
+        )
 
     print(f"\nEvaluating {len(scorer_names)} scorer(s)...\n")
 
@@ -164,7 +172,10 @@ def parse_args() -> argparse.Namespace:
         "--tags",
         type=str,
         default=None,
-        help="Comma-separated list of tags to filter which scorers to evaluate (e.g., --tags refusal,default)",
+        help=(
+            "Comma-separated list of tags to filter which scorers to evaluate (e.g., --tags refusal,default). "
+            "Scorers tagged skip_batch_evaluation remain excluded."
+        ),
     )
     parser.add_argument(
         "--max-concurrency",

doc/code/scoring/4_scorer_metrics.ipynb

@@ -795,18 +795,17 @@
    "source": [
     "## Batch Evaluation with `evaluate_scorers.py`\n",
     "\n",
-    "While `evaluate_async()` runs evaluations for a single scorer, the `evaluate_scorers.py` script\n",
-    "evaluates **all registered scorers** in bulk. This is useful for benchmarking after changing scorer\n",
-    "prompts, adding new variants, or updating human-labeled datasets.\n",
+    "`evaluate_scorers.py` initializes registered scorers and benchmarks eligible entries in bulk.\n",
+    "Results are saved under `pyrit/datasets/scorer_evals/`.\n",
     "\n",
-    "The script initializes PyRIT with `ScorerInitializer` (which registers all configured scorers),\n",
-    "then runs `evaluate_async()` on each one. Results are saved to the JSONL registry files in\n",
-    "`pyrit/datasets/scorer_evals/`.\n",
+    "`skip_batch_evaluation` excludes an entry even when matched by `--tags`. This skips raw Decision,\n",
+    "not its 0.7-threshold wrapper; both remain usable for scoring.\n",
+    "Missing evaluation configuration, evaluation failures, and empty eligible selections still fail the batch.\n",
     "\n",
     "### Basic Usage\n",
     "\n",
     "```bash\n",
-    "# Evaluate all registered scorers (long-running — can take hours)\n",
+    "# Evaluate all eligible registered scorers (long-running — can take hours)\n",
     "python -m build_scripts.evaluate_scorers\n",
     "\n",
     "# Evaluate only scorers with specific tags\n",
@@ -822,10 +821,13 @@
     "`ScorerInitializer` applies tags to scorers during registration. These tags let you target\n",
     "specific subsets for evaluation:\n",
     "\n",
-    "- `refusal` — The 4 standalone refusal scorer variants\n",
-    "- `default` — All scorers registered by default\n",
-    "- `best_refusal_f1` — The refusal variant with the highest F1 (set dynamically from metrics)\n",
-    "- `best_objective_f1` — The objective scorer with the highest F1\n",
+    "- `refusal` — Registered standalone refusal scorer variants\n",
+    "- `best_refusal` — The configured preferred refusal variant, or the first available variant\n",
+    "- `best_objective` / `default_objective_scorer` — The objective scorer selected below\n",
+    "\n",
+    "F1 ranking requires a known positive input count with every response evaluated.\n",
+    "If none qualify, selection warns and uses the configured fallback (`scale_and_refusal`,\n",
+    "then the first available composite), not an F1-backed winner.\n",
     "\n",
     "### Recommended Workflow: Refusal → Dependent Scorers\n",
     "\n",
@@ -837,20 +839,15 @@
     "python -m build_scripts.evaluate_scorers --tags refusal\n",
     "```\n",
     "\n",
-    "This evaluates only the 4 refusal variants and writes results to\n",
-    "`refusal_scorer/refusal_metrics.jsonl`. After this step, `ScorerInitializer` can determine which\n",
-    "refusal variant has the best F1 and tag it as `best_refusal_f1`.\n",
+    "Results are saved to `refusal_scorer/refusal_metrics.jsonl`.\n",
     "\n",
     "**Step 2: Re-evaluate all scorers**\n",
     "\n",
     "```bash\n",
     "python -m build_scripts.evaluate_scorers\n",
     "```\n",
     "\n",
-    "On the next full run, `ScorerInitializer` reads the refusal metrics from Step 1, picks the best\n",
-    "refusal variant, and uses it to build dependent scorers (e.g., `TrueFalseInverterScorer` wrapping\n",
-    "the best refusal scorer). This ensures objective scorers that depend on refusal detection use the\n",
-    "best-performing refusal prompt.\n",
+    "Dependent scorers use the configured `best_refusal` variant.\n",
     "\n",
     "Scorers whose metrics are already up-to-date (same dataset version, sufficient trials) are\n",
     "automatically skipped, so re-running the full script is efficient.\n",

doc/code/scoring/4_scorer_metrics.py

@@ -375,18 +375,17 @@
 # %% [markdown]
 # ## Batch Evaluation with `evaluate_scorers.py`
 #
-# While `evaluate_async()` runs evaluations for a single scorer, the `evaluate_scorers.py` script
-# evaluates **all registered scorers** in bulk. This is useful for benchmarking after changing scorer
-# prompts, adding new variants, or updating human-labeled datasets.
+# `evaluate_scorers.py` initializes registered scorers and benchmarks eligible entries in bulk.
+# Results are saved under `pyrit/datasets/scorer_evals/`.
 #
-# The script initializes PyRIT with `ScorerInitializer` (which registers all configured scorers),
-# then runs `evaluate_async()` on each one. Results are saved to the JSONL registry files in
-# `pyrit/datasets/scorer_evals/`.
+# `skip_batch_evaluation` excludes an entry even when matched by `--tags`. This skips raw Decision,
+# not its 0.7-threshold wrapper; both remain usable for scoring.
+# Missing evaluation configuration, evaluation failures, and empty eligible selections still fail the batch.
 #
 # ### Basic Usage
 #
 # ```bash
-# # Evaluate all registered scorers (long-running — can take hours)
+# # Evaluate all eligible registered scorers (long-running — can take hours)
 # python -m build_scripts.evaluate_scorers
 #
 # # Evaluate only scorers with specific tags
@@ -402,10 +401,13 @@
 # `ScorerInitializer` applies tags to scorers during registration. These tags let you target
 # specific subsets for evaluation:
 #
-# - `refusal` — The 4 standalone refusal scorer variants
-# - `default` — All scorers registered by default
-# - `best_refusal_f1` — The refusal variant with the highest F1 (set dynamically from metrics)
-# - `best_objective_f1` — The objective scorer with the highest F1
+# - `refusal` — Registered standalone refusal scorer variants
+# - `best_refusal` — The configured preferred refusal variant, or the first available variant
+# - `best_objective` / `default_objective_scorer` — The objective scorer selected below
+#
+# F1 ranking requires a known positive input count with every response evaluated.
+# If none qualify, selection warns and uses the configured fallback (`scale_and_refusal`,
+# then the first available composite), not an F1-backed winner.
 #
 # ### Recommended Workflow: Refusal → Dependent Scorers
 #
@@ -417,20 +419,15 @@
 # python -m build_scripts.evaluate_scorers --tags refusal
 # ```
 #
-# This evaluates only the 4 refusal variants and writes results to
-# `refusal_scorer/refusal_metrics.jsonl`. After this step, `ScorerInitializer` can determine which
-# refusal variant has the best F1 and tag it as `best_refusal_f1`.
+# Results are saved to `refusal_scorer/refusal_metrics.jsonl`.
 #
 # **Step 2: Re-evaluate all scorers**
 #
 # ```bash
 # python -m build_scripts.evaluate_scorers
 # ```
 #
-# On the next full run, `ScorerInitializer` reads the refusal metrics from Step 1, picks the best
-# refusal variant, and uses it to build dependent scorers (e.g., `TrueFalseInverterScorer` wrapping
-# the best refusal scorer). This ensures objective scorers that depend on refusal detection use the
-# best-performing refusal prompt.
+# Dependent scorers use the configured `best_refusal` variant.
 #
 # Scorers whose metrics are already up-to-date (same dataset version, sufficient trials) are
 # automatically skipped, so re-running the full script is efficient.

doc/contributing/10_release_process.md

@@ -108,6 +108,12 @@ for private vulnerability reporting.
   - **Partner integration tests.** Ensure the partner integration tests are also passing. These tests validate that we are not breaking contracts with partner teams (e.g., Foundry). If any are failing, coordinate with the affected partner teams before proceeding with the release.
   - **Azure key-based auth is disabled in our tenant.** Our Azure subscription has API-key (local) auth turned off, so Azure target integration tests authenticate with Microsoft Entra ID. Tests that run notebooks requiring Azure API keys are deliberately skipped; otherwise they fail with HTTP 403 `AuthenticationTypeDisabled` ("Key based authentication is disabled for this resource"). Do not re-enable key auth for our tenant. When validating a release manually, authenticate Azure targets with Entra (`az login`) rather than API keys.
 - **Update scorer metrics.** Run `python -m build_scripts.evaluate_scorers` and commit the results so that scorer evaluation metrics are up to date.
+  Scorers explicitly tagged `skip_batch_evaluation` are reported as skipped, including
+  when selected through `--tags`. The raw `task_achieved_microsoft_decision` probability
+  scorer carries this tag; its `task_achieved_microsoft_decision_threshold_07` wrapper
+  remains eligible for the objective benchmark. Both remain registered for normal use.
+  Missing evaluation mappings on other selected scorers are errors, not implicit exclusions.
+  An empty eligible selection also exits nonzero.
   Completion may use cached metrics; it does not guarantee fresh scoring.
   Blocked judge requests produce undetermined judgments rather than aborting the evaluation.
   A blocked response being evaluated is different: existing scorer semantics still apply
@@ -118,6 +124,11 @@ for private vulnerability reporting.
   interpreting metrics as full-dataset results. Legacy cached metrics without input counts
   have unknown coverage. Evaluations with no determined results or other errors still fail:
   the script finishes the remaining scorers, then exits nonzero without reporting overall success.
+  Automatic best-objective selection ranks F1 only when `num_input_responses > 0`
+  and `num_responses == num_input_responses`. Partial and unknown-coverage results
+  remain stored and reusable, but cannot win that ranking. If none qualify, the
+  initializer preserves its configured composite fallback and warns that it is not
+  an F1-backed winner. Explicit scorer selection remains available.
 
 ## 2. Decide the Next Version
 

pyrit/setup/initializers/__init__.py

@@ -14,7 +14,7 @@
     from pyrit.setup.initializers.load_default_datasets import LoadDefaultDatasets
     from pyrit.setup.initializers.preload_scenario_metadata import PreloadScenarioMetadata
     from pyrit.setup.initializers.refresh_datasets import RefreshDatasets
-    from pyrit.setup.initializers.scorers import ScorerInitializer
+    from pyrit.setup.initializers.scorers import ScorerInitializer, ScorerInitializerTags
     from pyrit.setup.initializers.targets import TargetInitializer
     from pyrit.setup.initializers.techniques import TechniqueInitializer
     from pyrit.setup.initializers.tools import ToolsInitializer
@@ -27,6 +27,7 @@
     "TechniqueInitializer": "pyrit.setup.initializers.techniques",
     "ToolsInitializer": "pyrit.setup.initializers.tools",
     "ScorerInitializer": "pyrit.setup.initializers.scorers",
+    "ScorerInitializerTags": "pyrit.setup.initializers.scorers",
     "TargetInitializer": "pyrit.setup.initializers.targets",
     "LoadDefaultDatasets": "pyrit.setup.initializers.load_default_datasets",
     "PreloadScenarioMetadata": "pyrit.setup.initializers.preload_scenario_metadata",

pyrit/setup/initializers/scorers.py

@@ -68,6 +68,9 @@ class ScorerInitializerTags(str, Enum):
     OBJECTIVE_COMPOSITE = "objective_composite"
     MICROSOFT_DECISION = "microsoft_decision"
 
+    # Batch-evaluation exclusions do not affect availability for normal scoring.
+    SKIP_BATCH_EVALUATION = "skip_batch_evaluation"
+
     # Overall best objective scorer (used by scenarios)
     BEST_OBJECTIVE = "best_objective"
     DEFAULT_OBJECTIVE_SCORER = "default_objective_scorer"
@@ -481,7 +484,7 @@ def _build_decision_scorer() -> DecisionScorer:
             name="task_achieved_microsoft_decision",
             factory=_build_decision_scorer,
             required_targets=[target],
-            tags=decision_tags,
+            tags=[*decision_tags, ScorerInitializerTags.SKIP_BATCH_EVALUATION],
         )
         self._try_register(
             name="task_achieved_microsoft_decision_threshold_07",
@@ -594,8 +597,8 @@ def _tag_best_objective(self) -> None:
         """
         Tag the overall best objective scorer.
 
-        Uses metrics-based selection (F1 score) when evaluation data is available,
-        falling back to a hardcoded default composite scorer.
+        Ranks F1 only when a positive input count confirms complete coverage.
+        Otherwise uses a configured composite fallback, not an F1-backed winner.
         """
         scorer_registry = self._get_scorer_registry()
         best_name: str | None = None
@@ -608,7 +611,14 @@ def _tag_best_objective(self) -> None:
             if not eval_hash:
                 continue
             metrics = find_objective_metrics_by_eval_hash(eval_hash=eval_hash)
-            if metrics is not None and metrics.f1_score > best_f1:
+            if (
+                metrics is None
+                or metrics.num_input_responses is None
+                or metrics.num_input_responses <= 0
+                or metrics.num_responses != metrics.num_input_responses
+            ):
+                continue
+            if metrics.f1_score > best_f1:
                 best_f1 = metrics.f1_score
                 best_name = entry.name
 
@@ -617,21 +627,32 @@ def _tag_best_objective(self) -> None:
                 name=best_name,
                 tags=[ScorerInitializerTags.BEST_OBJECTIVE, ScorerInitializerTags.DEFAULT_OBJECTIVE_SCORER],
             )
-            logger.info(f"Tagged {best_name} as {ScorerInitializerTags.BEST_OBJECTIVE} with F1={best_f1:.4f}")
+            logger.info(
+                f"Tagged {best_name} as {ScorerInitializerTags.BEST_OBJECTIVE} "
+                f"with F1={best_f1:.4f} on complete-coverage metrics"
+            )
             return
 
         # Fall back: prefer scale_and_refusal, then first composite
         best_tags: list[str] = [ScorerInitializerTags.BEST_OBJECTIVE, ScorerInitializerTags.DEFAULT_OBJECTIVE_SCORER]
         if scorer_registry.instances.get_entry(self.SCALE_AND_REFUSAL):
-            scorer_registry.instances.add_tags(name=self.SCALE_AND_REFUSAL, tags=best_tags)
-            logger.info(f"Tagged {self.SCALE_AND_REFUSAL} as {ScorerInitializerTags.BEST_OBJECTIVE} (default)")
+            fallback_name = self.SCALE_AND_REFUSAL
         else:
             composites = scorer_registry.instances.get_by_tag(tag=ScorerInitializerTags.OBJECTIVE_COMPOSITE)
-            if composites:
-                scorer_registry.instances.add_tags(name=composites[0].name, tags=best_tags)
-                logger.info(f"Tagged {composites[0].name} as {ScorerInitializerTags.BEST_OBJECTIVE} (fallback)")
-            else:
-                logger.warning("No composite scorers available; skipping best objective tagging.")
+            fallback_name = composites[0].name if composites else None
+        if fallback_name is not None:
+            scorer_registry.instances.add_tags(name=fallback_name, tags=best_tags)
+            logger.warning(
+                "No complete-coverage metrics qualified; tagged %s as %s using the "
+                "configured fallback, not an F1-backed winner.",
+                fallback_name,
+                ScorerInitializerTags.BEST_OBJECTIVE,
+            )
+        else:
+            logger.warning(
+                "No complete-coverage metrics qualified and no composite scorers available; "
+                "skipping best objective tagging."
+            )
 
     # ---------------------------------------------------------------------------
     # Helpers