feat(cloud): Kaggle GPU training path and a guard against Modal GPU on production

docs/cloud_gpu_training.md

@@ -1,5 +1,39 @@
 # Cloud GPU Training
 
+## Default: Kaggle (free), not Modal
+
+GPU training runs on Kaggle's free T4/P100 sessions. The production Modal
+workspace hosts OpenAdapt's managed runner, compiler and recorder. On
+2026-10-16 GPU training used up its monthly spend limit, and Modal stopped
+production with it. Modal GPU commands in `openadapt_ml.cloud.modal_cloud` now
+refuse to run unless you set `OPENADAPT_ALLOW_MODAL_GPU=1`, set
+`MODAL_ENVIRONMENT` to a non-production environment, and activate a Modal
+profile that is not in `OPENADAPT_MODAL_PROD_WORKSPACES` (default `abrichr`).
+
+```bash
+# one-time: Kaggle API token (Kaggle → Settings → API → Create New Token) and phone verification
+uv run python -m openadapt_ml.cloud.kaggle_cloud stage --bundle <bundle_dir> --config <train.yaml> --username <kaggle_user>
+uv run python -m openadapt_ml.cloud.kaggle_cloud push --new-dataset    # later runs: push (versions the dataset)
+uv run python -m openadapt_ml.cloud.kaggle_cloud status
+uv run python -m openadapt_ml.cloud.kaggle_cloud pull                  # results → training_output/kaggle/output
+# resume: stage again with --checkpoint training_output/kaggle/output/results
+```
+
+The kernel exits with code 3 if the session has no GPU or no internet, so a
+CPU fallback can't pass for a GPU run. T4s have no bf16; the trainer turns
+bf16 off automatically. Sessions are capped at 12 hours, and the GPU quota is
+weekly.
+
+### Modal workspace budget (production)
+
+In the Modal dashboard, go to **Settings → Usage & Billing** for workspace
+`abrichr` and set a **Workspace budget** and a **spend limit**. The hosted
+services use well under $1/month, so a low cap is enough. Starter-plan budgets
+apply to the whole workspace: a separate environment in the same workspace
+still spends the production budget. Run any Modal GPU work in a separate
+workspace.
+
+
 This guide covers using cloud GPUs for training OpenAdapt-ML models. Two providers are supported:
 
 | Provider | Setup Time | Cost | Best For |

openadapt_ml/cloud/kaggle_cloud.py

@@ -0,0 +1,196 @@
+"""Kaggle GPU training for OpenAdapt (free T4/P100 sessions).
+
+Kaggle replaces Modal for GPU training. The flow uses only the ``kaggle`` CLI:
+
+1. ``stage``: builds a private dataset folder (bundle, config, optional
+   checkpoint) and a script-kernel folder whose ``kernel-metadata.json`` turns
+   on GPU and internet.
+2. ``push``: ``kaggle datasets create`` (or ``version``), then
+   ``kaggle kernels push``. This starts a free Kaggle session; it never
+   touches Modal.
+3. ``status`` / ``pull``: ``kaggle kernels status`` and
+   ``kaggle kernels output`` bring the results back.
+
+To resume, pass the pulled ``results`` directory as ``--checkpoint`` to the
+next ``stage``; it is uploaded with the dataset as ``checkpoint/``.
+
+Credentials: ``~/.kaggle/access_token`` or ``~/.kaggle/kaggle.json``, or the
+``KAGGLE_USERNAME``/``KAGGLE_KEY`` (or ``KAGGLE_API_TOKEN``) environment
+variables. GPU and internet in kernels need a phone-verified Kaggle account.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+KERNEL_FILE = "kaggle_train_kernel.py"
+STAGE_MARKER = "openadapt_kaggle_stage.json"
+SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{4,48}[a-z0-9]$")
+ACCELERATORS = ("NvidiaTeslaT4", "NvidiaTeslaP100")
+
+
+def _check_slug(value: str, what: str) -> str:
+    if not SLUG_RE.fullmatch(value):
+        raise ValueError(
+            f"{what} must be 6-50 lowercase letters, digits or dashes: {value!r}"
+        )
+    return value
+
+
+def kernel_metadata(
+    username: str, kernel_slug: str, dataset_slug: str, accelerator: str
+) -> dict:
+    """Kaggle kernel-metadata.json for a private GPU script kernel."""
+    if accelerator not in ACCELERATORS:
+        raise ValueError(f"accelerator must be one of {ACCELERATORS}")
+    return {
+        "id": f"{username}/{_check_slug(kernel_slug, 'kernel slug')}",
+        "title": kernel_slug,
+        "code_file": KERNEL_FILE,
+        "language": "python",
+        "kernel_type": "script",
+        "is_private": True,
+        "enable_gpu": True,
+        "enable_internet": True,
+        "machine_shape": accelerator,
+        "dataset_sources": [f"{username}/{_check_slug(dataset_slug, 'dataset slug')}"],
+        "competition_sources": [],
+        "kernel_sources": [],
+    }
+
+
+def dataset_metadata(username: str, dataset_slug: str) -> dict:
+    return {
+        "title": dataset_slug,
+        "id": f"{username}/{_check_slug(dataset_slug, 'dataset slug')}",
+        "licenses": [{"name": "other"}],
+    }
+
+
+def stage(
+    bundle: Path,
+    config: Path,
+    out: Path,
+    username: str,
+    dataset_slug: str,
+    kernel_slug: str,
+    accelerator: str = "NvidiaTeslaT4",
+    checkpoint: Path | None = None,
+) -> tuple[Path, Path]:
+    """Write <out>/dataset and <out>/kernel. Returns both paths."""
+    if not (bundle / "training_data.jsonl").is_file():
+        raise FileNotFoundError(f"{bundle}/training_data.jsonl not found")
+    if not config.is_file():
+        raise FileNotFoundError(config)
+    dataset_dir, kernel_dir = out / "dataset", out / "kernel"
+    for d in (dataset_dir, kernel_dir):
+        if d.exists():
+            shutil.rmtree(d)
+        d.mkdir(parents=True)
+    shutil.copytree(bundle, dataset_dir / "bundle")
+    shutil.copy2(config, dataset_dir / "train_config.yaml")
+    if checkpoint is not None:
+        shutil.copytree(checkpoint, dataset_dir / "checkpoint")
+    (dataset_dir / STAGE_MARKER).write_text(
+        json.dumps(
+            {
+                "dataset": dataset_slug,
+                "kernel": kernel_slug,
+                "has_checkpoint": checkpoint is not None,
+            }
+        )
+    )
+    (dataset_dir / "dataset-metadata.json").write_text(
+        json.dumps(dataset_metadata(username, dataset_slug), indent=2)
+    )
+    shutil.copy2(Path(__file__).with_name(KERNEL_FILE), kernel_dir / KERNEL_FILE)
+    (kernel_dir / "kernel-metadata.json").write_text(
+        json.dumps(
+            kernel_metadata(username, kernel_slug, dataset_slug, accelerator), indent=2
+        )
+    )
+    return dataset_dir, kernel_dir
+
+
+def _kaggle(*args: str) -> int:
+    print("+ kaggle", " ".join(args), flush=True)
+    return subprocess.run(["kaggle", *args]).returncode
+
+
+def push(staged: Path, new_dataset: bool, message: str) -> int:
+    dataset_dir, kernel_dir = staged / "dataset", staged / "kernel"
+    if new_dataset:
+        rc = _kaggle("datasets", "create", "-p", str(dataset_dir), "-r", "zip")
+    else:
+        rc = _kaggle(
+            "datasets", "version", "-p", str(dataset_dir), "-m", message, "-r", "zip"
+        )
+    if rc:
+        return rc
+    print(
+        "Wait until the dataset version is ready before the kernel reads it.",
+        flush=True,
+    )
+    return _kaggle("kernels", "push", "-p", str(kernel_dir))
+
+
+def _kernel_ref(staged: Path) -> str:
+    return json.loads((staged / "kernel" / "kernel-metadata.json").read_text())["id"]
+
+
+def cli_main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Kaggle GPU training for OpenAdapt")
+    sub = parser.add_subparsers(dest="command", required=True)
+    s = sub.add_parser(
+        "stage", help="Write the dataset and kernel folders (no network)"
+    )
+    s.add_argument("--bundle", type=Path, required=True)
+    s.add_argument("--config", type=Path, required=True)
+    s.add_argument("--out", type=Path, default=Path("training_output/kaggle"))
+    s.add_argument("--username", required=True, help="Kaggle username")
+    s.add_argument("--dataset-slug", default="openadapt-ml-train-data")
+    s.add_argument("--kernel-slug", default="openadapt-ml-train")
+    s.add_argument("--accelerator", default="NvidiaTeslaT4", choices=ACCELERATORS)
+    s.add_argument("--checkpoint", type=Path, help="Pulled results dir to resume from")
+    p = sub.add_parser("push", help="Upload the dataset and start the kernel")
+    p.add_argument("--staged", type=Path, default=Path("training_output/kaggle"))
+    p.add_argument(
+        "--new-dataset", action="store_true", help="First upload (datasets create)"
+    )
+    p.add_argument("--message", default="openadapt-ml training data")
+    st = sub.add_parser("status", help="Kernel status")
+    st.add_argument("--staged", type=Path, default=Path("training_output/kaggle"))
+    pl = sub.add_parser("pull", help="Download kernel output (results)")
+    pl.add_argument("--staged", type=Path, default=Path("training_output/kaggle"))
+    pl.add_argument("--out", type=Path, default=Path("training_output/kaggle/output"))
+    args = parser.parse_args(argv)
+
+    if args.command == "stage":
+        d, k = stage(
+            args.bundle,
+            args.config,
+            args.out,
+            args.username,
+            args.dataset_slug,
+            args.kernel_slug,
+            args.accelerator,
+            args.checkpoint,
+        )
+        print(f"dataset: {d}\nkernel:  {k}")
+        return 0
+    if args.command == "push":
+        return push(args.staged, args.new_dataset, args.message)
+    if args.command == "status":
+        return _kaggle("kernels", "status", _kernel_ref(args.staged))
+    args.out.mkdir(parents=True, exist_ok=True)
+    return _kaggle("kernels", "output", _kernel_ref(args.staged), "-p", str(args.out))
+
+
+if __name__ == "__main__":
+    sys.exit(cli_main())

openadapt_ml/cloud/kaggle_train_kernel.py

@@ -0,0 +1,91 @@
+"""Kaggle kernel entrypoint for openadapt-ml SFT training.
+
+Pushed by ``openadapt_ml.cloud.kaggle_cloud``. It reads the staged dataset
+(bundle, config, optional checkpoint) from /kaggle/input, trains on the
+session GPU, and writes results to /kaggle/working/results so
+``kaggle kernels output`` can pull them back.
+
+It exits with code 3 before installing anything if the session has no GPU or
+no internet: a run that falls back to CPU must not pass for a GPU run.
+"""
+
+import json
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+PIP_SPEC = os.environ.get("OPENADAPT_ML_PIP_SPEC", "openadapt-ml[training]")
+WORK = Path("/kaggle/working")
+RESULTS = WORK / "results"
+
+
+def _find_staged() -> Path:
+    hits = sorted(Path("/kaggle/input").rglob("openadapt_kaggle_stage.json"))
+    if not hits:
+        raise SystemExit("staged dataset not found under /kaggle/input")
+    return hits[0].parent
+
+
+def _preflight() -> None:
+    probe = subprocess.run(["nvidia-smi", "-L"], capture_output=True, text=True)
+    print("GPU", probe.stdout.strip() or probe.stderr.strip(), flush=True)
+    if probe.returncode != 0 or "GPU" not in probe.stdout:
+        print("No GPU in this session; refusing to train on CPU.", flush=True)
+        raise SystemExit(3) from None
+    try:
+        socket.gethostbyname("pypi.org")
+    except OSError as exc:
+        print(f"No internet in this session ({exc}).", flush=True)
+        raise SystemExit(3) from None
+
+
+def main() -> int:
+    _preflight()
+    staged = _find_staged()
+    stage = json.loads((staged / "openadapt_kaggle_stage.json").read_text())
+    print("stage", stage, flush=True)
+    subprocess.run([sys.executable, "-m", "pip", "install", "-q", PIP_SPEC], check=True)
+
+    bundle = WORK / "bundle"
+    if not bundle.exists():
+        shutil.copytree(staged / "bundle", bundle)
+    RESULTS.mkdir(parents=True, exist_ok=True)
+    if (staged / "checkpoint").is_dir():
+        shutil.copytree(
+            staged / "checkpoint", RESULTS / "checkpoint_in", dirs_exist_ok=True
+        )
+
+    cmd = [
+        sys.executable,
+        "-m",
+        "openadapt_ml.scripts.train",
+        "--config",
+        str(staged / "train_config.yaml"),
+        "--jsonl",
+        str(bundle / "training_data.jsonl"),
+        "--output-dir",
+        str(RESULTS),
+    ]
+    print("Running:", " ".join(cmd), flush=True)
+    started = time.time()
+    with open(RESULTS / "train_stdout.log", "w") as log:
+        rc = subprocess.run(
+            cmd, cwd=bundle, stdout=log, stderr=subprocess.STDOUT
+        ).returncode
+    summary = {
+        "status": "completed" if rc == 0 else "failed",
+        "returncode": rc,
+        "elapsed_time": time.time() - started,
+        "stage": stage,
+    }
+    (RESULTS / "kaggle_run.json").write_text(json.dumps(summary, indent=2))
+    print(json.dumps(summary), flush=True)
+    return rc
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

openadapt_ml/cloud/modal_cloud.py

@@ -43,6 +43,11 @@
 import tempfile
 from pathlib import Path
 
+from openadapt_ml.cloud.modal_gpu_guard import (
+    modal_gpu_refusal,
+    require_nonprod_modal_gpu,
+)
+
 # ---------------------------------------------------------------------------
 # Modal app definition (imported lazily so CLI parsing works without modal)
 # ---------------------------------------------------------------------------
@@ -75,6 +80,7 @@ def _build_app():
     Returns:
         (app, training_image, training_volume)
     """
+    require_nonprod_modal_gpu()
     modal = _get_modal()
 
     app = modal.App(MODAL_APP_NAME)
@@ -281,6 +287,7 @@ def _build_inference_app(
     Returns:
         (app, infer_fn) - the app and the inference function handle.
     """
+    require_nonprod_modal_gpu()
     modal = _get_modal()
 
     app = modal.App(INFERENCE_APP_NAME)
@@ -784,6 +791,10 @@ def _cmd_train(args: argparse.Namespace) -> int:
     3. Run training function remotely
     4. Download results
     """
+    refusal = modal_gpu_refusal()
+    if refusal:
+        print(f"Refusing Modal GPU work: {refusal}")
+        return 1
     modal = _get_modal()
 
     # --- Step 0: Auto-convert demos if --demo-dir provided ---
@@ -924,6 +935,10 @@ def _cmd_serve(args: argparse.Namespace) -> int:
     inference function that clients can call via Modal's .remote() API.
     Alternatively, clients can use the HTTP wrapper in Qwen3VLAgent.
     """
+    refusal = modal_gpu_refusal()
+    if refusal:
+        print(f"Refusing Modal GPU work: {refusal}")
+        return 1
     modal = _get_modal()
 
     adapter_remote = None