Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ verify-benchmark-contracts: banner venv
PYTHONPATH=$(BENCH_PYTHONPATH) "$(PYTHON)" scripts/test_benchmark_contracts.py
PYTHONPATH=$(BENCH_PYTHONPATH) "$(PYTHON)" scripts/test_thread_study.py
PYTHONPATH=$(BENCH_PYTHONPATH) "$(PYTHON)" scripts/test_solver_output_contracts.py
PYTHONPATH=scripts "$(PYTHON)" scripts/test_verify_stock_solverforge_guardrails.py
PYTHONPATH=. "$(PYTHON)" scripts/test_reaudit_legacy_publication.py
PYTHONPATH=src "$(PYTHON)" scripts/test_git_provenance.py
PYTHONPATH=$(BENCH_PYTHONPATH) "$(PYTHON)" scripts/test_reference_catalog.py
Expand Down
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,10 @@ Use `make verify-stock-solverforge-guardrails` after upstream SolverForge
solver architecture changes. It builds the active native adapters, runs the
stock JSSP quick group plus a fixed JSPLIB canonical subset, and requires
SolverForge JSSP, CVRP, and employee smoke rows to stay hard-feasible with
valid fair-start witnesses. Add `--require-jssp-win` through `GUARDRAIL_ARGS`
valid fair-start witnesses. A reference solver that ends its time limit without
a schedule (`no_solution` or `no_incumbent`) is recorded as its result; any
other reference error, an invalid fair start, or a watchdog kill fails the
guardrail. Add `--require-jssp-win` through `GUARDRAIL_ARGS`
when checking the final JSSP win condition.

## SolverForge-Py Adapter Test Gate
Expand Down
2 changes: 2 additions & 0 deletions WIREFRAME.md
Original file line number Diff line number Diff line change
Expand Up @@ -511,6 +511,8 @@ Contract gates:
strongest-policy hash so jointly weakening both copies also fails.
- `make verify-stock-solverforge-guardrails` builds the active native adapters,
runs stock SolverForge guardrail benchmarks, and parses the resulting CSVs.
SolverForge rows fail on any error; reference rows fail on any error except
ending the time limit without a schedule (`no_solution`, `no_incumbent`).
Pass `--require-jssp-win` through `GUARDRAIL_ARGS` to enforce that
SolverForge ties or beats the best feasible JSSP solver row.
- `make verify-solverforge-py-smoke` builds the native SolverForge adapters,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@
solver_result,
witness_from_native_output,
)
from solverforge_bench.model import SolverResult
from solverforge_bench.model import NoSolutionFoundError, SolverResult

_JAR_PATH = Path(__file__).parent / "timefold" / "target" / "timefold-jssp.jar"
# Main.NO_SOLUTION_EXIT_CODE: no complete schedule within the time limit.
_NO_SOLUTION_EXIT_CODE = 3


def solve_with_timefold(instance: JobShopInstance, time_limit: int) -> SolverResult:
Expand All @@ -37,6 +39,11 @@ def solve_with_timefold(instance: JobShopInstance, time_limit: int) -> SolverRes
capture_output=True,
)
stderr = result.stderr.decode()
if result.returncode == _NO_SOLUTION_EXIT_CODE:
raise NoSolutionFoundError(
f"timefold returned no complete JSSP schedule within the time limit:\n{stderr}",
termination_status="no_incumbent",
)
if result.returncode != 0:
raise RuntimeError(
f"timefold solver failed (exit {result.returncode}):\n" f"{stderr}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
import java.util.Objects;

public class Main {
/** Exit status for "no complete schedule within the time limit"; the Python adapter maps it to NoSolutionFoundError. */
static final int NO_SOLUTION_EXIT_CODE = 3;

static class OperationInput {
@JsonProperty("job_id")
Expand Down Expand Up @@ -147,9 +149,9 @@ public static void main(String[] args) throws Exception {

JsspScheduleEvaluator.Evaluation evaluation = JsspScheduleEvaluator.evaluate(solution);
if (evaluation.hardPenalty() > 0) {
throw new IllegalStateException(
"Timefold returned an incomplete or invalid JSSP schedule "
+ "(hard penalty: " + evaluation.hardPenalty() + ")");
System.err.println("Timefold returned no complete JSSP schedule within the time limit "
+ "(hard penalty: " + evaluation.hardPenalty() + ")");
System.exit(NO_SOLUTION_EXIT_CODE);
}
for (OperationAssignment operation : solution.getOperations()) {
operation.setStart(evaluation.startFor(operation));
Expand Down
41 changes: 41 additions & 0 deletions scripts/test_solver_output_contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,47 @@ def test_unclassified_native_failure_stays_an_adapter_error(self) -> None:
self.assertEqual(raised.exception.termination_status, "adapter_error")


class TimefoldJsspNoScheduleTests(unittest.TestCase):
"""The Timefold JSSP wrapper exits 3 when its time limit ends before the
schedule is complete; that is a missing incumbent, not an adapter crash."""

@staticmethod
def _instance() -> JobShopInstance:
return JobShopInstance(
name="no-schedule",
family="regression",
num_jobs=1,
num_machines=1,
operations_by_job=(
(Operation(job_id=0, op_index=0, machine_id=0, duration=1),),
),
)

def _solve(self, returncode: int, stderr: bytes):
from job_shop_bench.solver import timefold

completed = SimpleNamespace(returncode=returncode, stderr=stderr, stdout=b"")
with patch.object(
timefold.subprocess, "run", return_value=completed
), patch.object(timefold, "emit_fair_start_witness"):
return timefold.solve_with_timefold(self._instance(), 1)

def test_incomplete_schedule_is_a_missing_incumbent(self) -> None:
with self.assertRaises(NoSolutionFoundError) as raised:
self._solve(
3, b"Timefold returned no complete JSSP schedule (hard penalty: 306)"
)

self.assertEqual(raised.exception.termination_status, "no_incumbent")
self.assertIn("hard penalty: 306", str(raised.exception))

def test_other_nonzero_exits_stay_adapter_failures(self) -> None:
with self.assertRaises(RuntimeError) as raised:
self._solve(1, b"Exception in thread main")

self.assertNotIsInstance(raised.exception, NoSolutionFoundError)


class PyvrpRouteTranslationTests(unittest.TestCase):
"""pyvrp 0.14.0 restructured routes and location registration.

Expand Down
45 changes: 45 additions & 0 deletions scripts/test_verify_stock_solverforge_guardrails.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/usr/bin/env python3.14
"""Regression tests for the stock SolverForge guardrail row checks."""

from __future__ import annotations

import unittest

import verify_stock_solverforge_guardrails as guardrails


def row(
solver: str, run_error: str = "", termination_status: str = "solution_returned"
):
return {
"instance": "swv20",
"solver": solver,
"time_limit_seconds": "1",
"fair_start_valid": "true",
"run_error": run_error,
"termination_status": termination_status,
"watchdog_killed": "false",
}


class CommonRowTests(unittest.TestCase):
def test_reference_without_schedule_is_a_result(self) -> None:
for solver, status in (("highs", "no_incumbent"), ("timefold", "no_solution")):
rows = [row(solver, "NoSolutionFoundError: none in time", status)]
self.assertEqual(guardrails.validate_common_rows(rows, "JSSP"), [], solver)

def test_reference_crash_still_fails(self) -> None:
for status in ("adapter_error", "proved_infeasible", "model_invalid"):
rows = [row("timefold", "RuntimeError: timefold solver failed", status)]
failures = guardrails.validate_common_rows(rows, "JSSP")
self.assertEqual(len(failures), 1, status)

def test_solverforge_without_schedule_still_fails(self) -> None:
for solver in ("solverforge", "solverforge-py"):
rows = [row(solver, "NoSolutionFoundError: none in time", "no_solution")]
failures = guardrails.validate_common_rows(rows, "JSSP")
self.assertEqual(len(failures), 1, solver)


if __name__ == "__main__":
unittest.main()
18 changes: 17 additions & 1 deletion scripts/verify_stock_solverforge_guardrails.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
REPO_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_JSSP_SUBSET = ("ft10", "swv20", "yn3", "ta01")
DEFAULT_TIME_LIMITS = ("1", "10")
SOLVERFORGE_SOLVERS = frozenset({"solverforge", "solverforge-py"})
REFERENCE_NO_SCHEDULE_STATUSES = frozenset({"no_solution", "no_incumbent"})


def main() -> None:
Expand Down Expand Up @@ -204,13 +206,27 @@ def validate_common_rows(rows: list[dict[str, str]], label: str) -> list[str]:
)
if not is_true(row["fair_start_valid"]):
failures.append(f"{row_label} fair-start witness invalid")
if row["run_error"]:
if row["run_error"] and not is_reference_without_schedule(row):
failures.append(f"{row_label} run_error={row['run_error']}")
if is_true(row["watchdog_killed"]):
failures.append(f"{row_label} watchdog killed solver")
return failures


def is_reference_without_schedule(row: dict[str, str]) -> bool:
"""A reference solver that ends its time limit without a schedule.

That is a benchmark result for the reference solver, not a defect in the
harness or in SolverForge. SolverForge rows, and any reference row that
crashed, proved a valid instance infeasible, or rejected the model, still
fail the guardrail.
"""
return (
row["solver"] not in SOLVERFORGE_SOLVERS
and row["termination_status"] in REFERENCE_NO_SCHEDULE_STATUSES
)


def read_rows(path: Path) -> list[dict[str, str]]:
with path.open(newline="", encoding="utf-8") as handle:
return list(csv.DictReader(handle))
Expand Down
Loading