From 769a1bfe02038d576f0007a2358ea02af3748c15 Mon Sep 17 00:00:00 2001 From: Vittorio Distefano Date: Mon, 9 Nov 2026 20:45:21 +0100 Subject: [PATCH 1/2] fix(jssp): report Timefold's incomplete schedule as a missing incumbent When Timefold's time limit ends before construction completes a JSSP schedule, the Java wrapper threw IllegalStateException and the Python adapter turned the exit into a RuntimeError, so the row read as an adapter crash. OR-Tools and HiGHS already report the same outcome as NoSolutionFoundError. The wrapper now exits with status 3 for an incomplete schedule and the adapter raises NoSolutionFoundError with termination_status no_incumbent, keeping the hard-penalty detail. Every other non-zero exit stays an adapter error. --- .../src/job_shop_bench/solver/timefold.py | 9 +++- .../java/com/solverforgebench/jssp/Main.java | 8 ++-- scripts/test_solver_output_contracts.py | 41 +++++++++++++++++++ 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/scalar-variable/job-shop-scheduling/src/job_shop_bench/solver/timefold.py b/scalar-variable/job-shop-scheduling/src/job_shop_bench/solver/timefold.py index 28f2c30..fc71fb0 100644 --- a/scalar-variable/job-shop-scheduling/src/job_shop_bench/solver/timefold.py +++ b/scalar-variable/job-shop-scheduling/src/job_shop_bench/solver/timefold.py @@ -17,9 +17,11 @@ solver_result, witness_from_native_output, ) -from solverforge_bench.model import SolverResult +from solverforge_bench.model import NoSolutionFoundError, SolverResult _JAR_PATH = Path(__file__).parent / "timefold" / "target" / "timefold-jssp.jar" +# Main.NO_SOLUTION_EXIT_CODE: no complete schedule within the time limit. +_NO_SOLUTION_EXIT_CODE = 3 def solve_with_timefold(instance: JobShopInstance, time_limit: int) -> SolverResult: @@ -37,6 +39,11 @@ def solve_with_timefold(instance: JobShopInstance, time_limit: int) -> SolverRes capture_output=True, ) stderr = result.stderr.decode() + if result.returncode == _NO_SOLUTION_EXIT_CODE: + raise NoSolutionFoundError( + f"timefold returned no complete JSSP schedule within the time limit:\n{stderr}", + termination_status="no_incumbent", + ) if result.returncode != 0: raise RuntimeError( f"timefold solver failed (exit {result.returncode}):\n" f"{stderr}" diff --git a/scalar-variable/job-shop-scheduling/src/job_shop_bench/solver/timefold/src/main/java/com/solverforgebench/jssp/Main.java b/scalar-variable/job-shop-scheduling/src/job_shop_bench/solver/timefold/src/main/java/com/solverforgebench/jssp/Main.java index 859fcb8..3d44ead 100644 --- a/scalar-variable/job-shop-scheduling/src/job_shop_bench/solver/timefold/src/main/java/com/solverforgebench/jssp/Main.java +++ b/scalar-variable/job-shop-scheduling/src/job_shop_bench/solver/timefold/src/main/java/com/solverforgebench/jssp/Main.java @@ -18,6 +18,8 @@ import java.util.Objects; public class Main { + /** Exit status for "no complete schedule within the time limit"; the Python adapter maps it to NoSolutionFoundError. */ + static final int NO_SOLUTION_EXIT_CODE = 3; static class OperationInput { @JsonProperty("job_id") @@ -147,9 +149,9 @@ public static void main(String[] args) throws Exception { JsspScheduleEvaluator.Evaluation evaluation = JsspScheduleEvaluator.evaluate(solution); if (evaluation.hardPenalty() > 0) { - throw new IllegalStateException( - "Timefold returned an incomplete or invalid JSSP schedule " - + "(hard penalty: " + evaluation.hardPenalty() + ")"); + System.err.println("Timefold returned no complete JSSP schedule within the time limit " + + "(hard penalty: " + evaluation.hardPenalty() + ")"); + System.exit(NO_SOLUTION_EXIT_CODE); } for (OperationAssignment operation : solution.getOperations()) { operation.setStart(evaluation.startFor(operation)); diff --git a/scripts/test_solver_output_contracts.py b/scripts/test_solver_output_contracts.py index 235ecdc..07e71df 100644 --- a/scripts/test_solver_output_contracts.py +++ b/scripts/test_solver_output_contracts.py @@ -332,6 +332,47 @@ def test_unclassified_native_failure_stays_an_adapter_error(self) -> None: self.assertEqual(raised.exception.termination_status, "adapter_error") +class TimefoldJsspNoScheduleTests(unittest.TestCase): + """The Timefold JSSP wrapper exits 3 when its time limit ends before the + schedule is complete; that is a missing incumbent, not an adapter crash.""" + + @staticmethod + def _instance() -> JobShopInstance: + return JobShopInstance( + name="no-schedule", + family="regression", + num_jobs=1, + num_machines=1, + operations_by_job=( + (Operation(job_id=0, op_index=0, machine_id=0, duration=1),), + ), + ) + + def _solve(self, returncode: int, stderr: bytes): + from job_shop_bench.solver import timefold + + completed = SimpleNamespace(returncode=returncode, stderr=stderr, stdout=b"") + with patch.object( + timefold.subprocess, "run", return_value=completed + ), patch.object(timefold, "emit_fair_start_witness"): + return timefold.solve_with_timefold(self._instance(), 1) + + def test_incomplete_schedule_is_a_missing_incumbent(self) -> None: + with self.assertRaises(NoSolutionFoundError) as raised: + self._solve( + 3, b"Timefold returned no complete JSSP schedule (hard penalty: 306)" + ) + + self.assertEqual(raised.exception.termination_status, "no_incumbent") + self.assertIn("hard penalty: 306", str(raised.exception)) + + def test_other_nonzero_exits_stay_adapter_failures(self) -> None: + with self.assertRaises(RuntimeError) as raised: + self._solve(1, b"Exception in thread main") + + self.assertNotIsInstance(raised.exception, NoSolutionFoundError) + + class PyvrpRouteTranslationTests(unittest.TestCase): """pyvrp 0.14.0 restructured routes and location registration. From 9111abd9db6f480235621e8b2c472374beb31af5 Mon Sep 17 00:00:00 2001 From: Vittorio Distefano Date: Mon, 9 Nov 2026 20:46:02 +0100 Subject: [PATCH 2/2] fix(guardrails): treat a reference solver without a schedule as a result The stock SolverForge guardrail failed on any run_error, so HiGHS and Timefold ending a 1s or 10s limit on swv20, yn3 or ta01 without a schedule failed the SolverForge guardrail on every run, with every SolverForge row hard-feasible. Reference rows whose termination_status is no_solution or no_incumbent are now accepted as results. Reference crashes, proved infeasibility of a valid instance, rejected models, invalid fair starts and watchdog kills still fail, and SolverForge and solverforge-py rows still fail on any error. Closes #3 --- Makefile | 1 + README.md | 5 ++- WIREFRAME.md | 2 + ...est_verify_stock_solverforge_guardrails.py | 45 +++++++++++++++++++ .../verify_stock_solverforge_guardrails.py | 18 +++++++- 5 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 scripts/test_verify_stock_solverforge_guardrails.py diff --git a/Makefile b/Makefile index 9fdd804..92671c5 100644 --- a/Makefile +++ b/Makefile @@ -128,6 +128,7 @@ verify-benchmark-contracts: banner venv PYTHONPATH=$(BENCH_PYTHONPATH) "$(PYTHON)" scripts/test_benchmark_contracts.py PYTHONPATH=$(BENCH_PYTHONPATH) "$(PYTHON)" scripts/test_thread_study.py PYTHONPATH=$(BENCH_PYTHONPATH) "$(PYTHON)" scripts/test_solver_output_contracts.py + PYTHONPATH=scripts "$(PYTHON)" scripts/test_verify_stock_solverforge_guardrails.py PYTHONPATH=. "$(PYTHON)" scripts/test_reaudit_legacy_publication.py PYTHONPATH=src "$(PYTHON)" scripts/test_git_provenance.py PYTHONPATH=$(BENCH_PYTHONPATH) "$(PYTHON)" scripts/test_reference_catalog.py diff --git a/README.md b/README.md index cce5234..891d2b1 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,10 @@ Use `make verify-stock-solverforge-guardrails` after upstream SolverForge solver architecture changes. It builds the active native adapters, runs the stock JSSP quick group plus a fixed JSPLIB canonical subset, and requires SolverForge JSSP, CVRP, and employee smoke rows to stay hard-feasible with -valid fair-start witnesses. Add `--require-jssp-win` through `GUARDRAIL_ARGS` +valid fair-start witnesses. A reference solver that ends its time limit without +a schedule (`no_solution` or `no_incumbent`) is recorded as its result; any +other reference error, an invalid fair start, or a watchdog kill fails the +guardrail. Add `--require-jssp-win` through `GUARDRAIL_ARGS` when checking the final JSSP win condition. ## SolverForge-Py Adapter Test Gate diff --git a/WIREFRAME.md b/WIREFRAME.md index 098c578..037b87b 100644 --- a/WIREFRAME.md +++ b/WIREFRAME.md @@ -511,6 +511,8 @@ Contract gates: strongest-policy hash so jointly weakening both copies also fails. - `make verify-stock-solverforge-guardrails` builds the active native adapters, runs stock SolverForge guardrail benchmarks, and parses the resulting CSVs. + SolverForge rows fail on any error; reference rows fail on any error except + ending the time limit without a schedule (`no_solution`, `no_incumbent`). Pass `--require-jssp-win` through `GUARDRAIL_ARGS` to enforce that SolverForge ties or beats the best feasible JSSP solver row. - `make verify-solverforge-py-smoke` builds the native SolverForge adapters, diff --git a/scripts/test_verify_stock_solverforge_guardrails.py b/scripts/test_verify_stock_solverforge_guardrails.py new file mode 100644 index 0000000..1f01e1d --- /dev/null +++ b/scripts/test_verify_stock_solverforge_guardrails.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3.14 +"""Regression tests for the stock SolverForge guardrail row checks.""" + +from __future__ import annotations + +import unittest + +import verify_stock_solverforge_guardrails as guardrails + + +def row( + solver: str, run_error: str = "", termination_status: str = "solution_returned" +): + return { + "instance": "swv20", + "solver": solver, + "time_limit_seconds": "1", + "fair_start_valid": "true", + "run_error": run_error, + "termination_status": termination_status, + "watchdog_killed": "false", + } + + +class CommonRowTests(unittest.TestCase): + def test_reference_without_schedule_is_a_result(self) -> None: + for solver, status in (("highs", "no_incumbent"), ("timefold", "no_solution")): + rows = [row(solver, "NoSolutionFoundError: none in time", status)] + self.assertEqual(guardrails.validate_common_rows(rows, "JSSP"), [], solver) + + def test_reference_crash_still_fails(self) -> None: + for status in ("adapter_error", "proved_infeasible", "model_invalid"): + rows = [row("timefold", "RuntimeError: timefold solver failed", status)] + failures = guardrails.validate_common_rows(rows, "JSSP") + self.assertEqual(len(failures), 1, status) + + def test_solverforge_without_schedule_still_fails(self) -> None: + for solver in ("solverforge", "solverforge-py"): + rows = [row(solver, "NoSolutionFoundError: none in time", "no_solution")] + failures = guardrails.validate_common_rows(rows, "JSSP") + self.assertEqual(len(failures), 1, solver) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/verify_stock_solverforge_guardrails.py b/scripts/verify_stock_solverforge_guardrails.py index b4d5f0d..43bb93d 100644 --- a/scripts/verify_stock_solverforge_guardrails.py +++ b/scripts/verify_stock_solverforge_guardrails.py @@ -14,6 +14,8 @@ REPO_ROOT = Path(__file__).resolve().parents[1] DEFAULT_JSSP_SUBSET = ("ft10", "swv20", "yn3", "ta01") DEFAULT_TIME_LIMITS = ("1", "10") +SOLVERFORGE_SOLVERS = frozenset({"solverforge", "solverforge-py"}) +REFERENCE_NO_SCHEDULE_STATUSES = frozenset({"no_solution", "no_incumbent"}) def main() -> None: @@ -204,13 +206,27 @@ def validate_common_rows(rows: list[dict[str, str]], label: str) -> list[str]: ) if not is_true(row["fair_start_valid"]): failures.append(f"{row_label} fair-start witness invalid") - if row["run_error"]: + if row["run_error"] and not is_reference_without_schedule(row): failures.append(f"{row_label} run_error={row['run_error']}") if is_true(row["watchdog_killed"]): failures.append(f"{row_label} watchdog killed solver") return failures +def is_reference_without_schedule(row: dict[str, str]) -> bool: + """A reference solver that ends its time limit without a schedule. + + That is a benchmark result for the reference solver, not a defect in the + harness or in SolverForge. SolverForge rows, and any reference row that + crashed, proved a valid instance infeasible, or rejected the model, still + fail the guardrail. + """ + return ( + row["solver"] not in SOLVERFORGE_SOLVERS + and row["termination_status"] in REFERENCE_NO_SCHEDULE_STATUSES + ) + + def read_rows(path: Path) -> list[dict[str, str]]: with path.open(newline="", encoding="utf-8") as handle: return list(csv.DictReader(handle))