Audit the suite's skips and fix what un-gating exposed

.github/workflows/gpu-tests.yaml

@@ -60,11 +60,11 @@ on:
       # family is added HERE and nowhere else -- this entry plus its arm in the
       # Resolve test scope step below.
       suite:
-        description: "Which tests to run: full (all five suites), short (vllm + integration), dev (one fast GPU file), audio (the audio/ASR marker), multi (all five on 2 GPUs)."
+        description: "Which tests to run: full (all five suites), short (vllm + integration), dev (one fast GPU file), audio (the audio/ASR marker), multi (all five on 2 GPUs), skips (the formerly-skipped tests being un-gated)."
         required: false
         default: full
         type: choice
-        options: [full, short, dev, audio, multi]
+        options: [full, short, dev, audio, multi, skips]
       # RESULT CACHE OPT-OUT. Default off, i.e. cache on: the expensive thing here is
       # a GPU run, and re-running one for a commit+scope that was already tested buys
       # nothing in the common case (a reviewer re-reading a PR, a second maintainer
@@ -238,6 +238,42 @@ jobs:
               SUFFIX="audio"
               LEG="$LABEL-audio"
               ;;
+            # The tests that the skip audit un-gated, as one narrow scope. Exists
+            # because `full` is not currently a usable way to check them: every
+            # full-suite run since 2026-10-18 has been cancelled from outside the pod
+            # at ~95-120 min (see the three-timeout note above -- it is none of
+            # ours), so a 3h46m suite is stopped around 40% and reports no summary.
+            # These three paths finish far inside that window.
+            #
+            # What each one is here to prove, since all three were changed by edits
+            # that no CPU box can check:
+            #   tests/hf/test_quantization.py        16 tests that used to skip on an
+            #                                       env flag; 4 of them now gate on a
+            #                                       real Quanto float8 probe that only
+            #                                       means anything on a GPU.
+            #   tests/composer/test_skinning_...py   was @pytest.mark.skip outright; now
+            #                                       gates top-k JSD/Jaccard + argmax,
+            #                                       plus exact config multipliers. This
+            #                                       scope is what measured the bf16 noise
+            #                                       floor the thresholds are calibrated
+            #                                       against -- a CPU box runs it in fp32
+            #                                       and sees none of it. It is also what
+            #                                       caught x86 contracting the logits
+            #                                       0.37%, which retired a numeric
+            #                                       logit-scale gate as unportable.
+            #   tests/unit/test_bitmask_computation  34 tests, no longer 2 skips. Cheap,
+            #                                       and it cannot even be COLLECTED off
+            #                                       GPU: it imports lora_kernel_meta,
+            #                                       which imports vllm.distributed.
+            #
+            # NOT a substitute for `full`, and deliberately not wired into the
+            # /gpu-test comment path's defaults. Delete this arm once the ~95 min
+            # cancellation is fixed and `full` is trustworthy again.
+            skips)
+              PATHS="tests/hf/test_quantization.py tests/composer/test_skinning_equivalence.py tests/unit/test_bitmask_computation.py"
+              SUFFIX="skips"
+              LEG="$LABEL-skips"
+              ;;
             # Everything `full` runs, but on TWO GPUs. Not a different test
             # selection -- a different SHAPE of pod.
             #

.gitignore

@@ -64,3 +64,6 @@ docs/KV_CACHE_OVERHEAD_REMOVAL.md
 docs/KV_CACHE_OVERHEAD_REMOVAL.html
 docs/KV_CACHE_OVERHEAD_REMOVAL*.html
 docs/UNIFIED_SWITCH_FUSED_CANONICAL.md
+# Skip audit: a point-in-time snapshot of which tests were skipped and why. It
+# goes stale the moment a gate changes, so it stays a local working note.
+docs/SKIPPED_TESTS.html

CLAUDE.md

@@ -404,8 +404,45 @@ architectures (4.0, 4.1) use `attention_bias=False` and `mlp_bias=False`.
 The GraniteSwitch HF backend uses fused QKV and gate-up projections, symmetric with the vLLM
 backend architecture. Upstream HuggingFace `GraniteMoeHybridForCausalLM` uses separate projections.
 Fused projections change the floating-point reduction order, so bit-exact skinning equivalence
-with the upstream HF model is not achievable. The HF skinning tests in
-`tests/composer/test_skinning_equivalence.py` are skipped for this reason.
+with the upstream HF model is not achievable. The HF skinning test in
+`tests/composer/test_skinning_equivalence.py` used to be skipped outright for this reason; it
+now runs and gates **distribution** equivalence instead, with the same metric and thresholds
+`tests/vllm/_generation_equivalence_worker.py` already calibrated: top-k JSD + Jaccard swept
+over k, plus an exact argmax check. Three logit checks, all after `log_softmax`:
+
+- **argmax, exact** — what greedy decoding emits; measured to be a strict maximum at every
+  position, so no tolerance is needed even in bf16.
+- **mean + max top-k JSD** — mean catches "wrong everywhere a little", max catches one
+  catastrophically wrong position that a mean would dilute.
+- **mean top-k Jaccard distance**, deliberately loose. An exact top-K *set* match is
+  unreachable in bf16 and was the first attempt: bf16 resolution is ~0.11 at these logits, so
+  the rank-100 boundary is a pile of exact ties (measured 4, 7 and 7 tokens sharing one bf16
+  value at the cut) and which tie-mate lands inside the cut is arbitrary. Jaccard measures how
+  much the top-k overlaps rather than demanding it match.
+Being post-`log_softmax`, those three are blind to a uniform logit *scaling* — the signature of
+a mistransferred Granite multiplier. That is gated separately, and **on the config rather than
+the logits**: `_compare_multipliers` asserts all four of `logits_scaling`,
+`attention_multiplier`, `residual_multiplier` and `embedding_multiplier` match the base config
+exactly. The bug is reachable — `compose_utils.py:189` propagates them with
+`getattr(base_config, field, default)` where the default is `1.0`, so a field upstream renames
+or drops silently becomes 1.0 (verified for all four; absent `logits_scaling`, which is 10 on
+granite-4.0-micro, would compose as 1.0). `test_save_load_compose.py` only checks these survive
+a save/load round-trip (`loaded == built`), never that `built` matches the *base*, so this is
+the only coverage of that gap.
+
+A numeric logit-scale gate was tried first and **could not be made platform-independent**. As an
+OLS slope it read 3.2e-05 on arm64 and 4.0e-03 on CI's x86; switching to `std(sw)/std(ref)` on a
+regression-dilution theory did not help (3.7e-03 on x86 vs 2.8e-04 on arm64). The data refutes
+the noise explanation: both platforms report the same reduction residual (2.318e-02 vs
+2.213e-02, within 5%), and since independent per-side noise bounds `|a-1|` by `residual²/2`,
+arm64 sits at 1.1x that ceiling (pure noise) while x86 sits at **14x** it. x86 genuinely
+contracts the logits ~0.37% on a skin that is otherwise clean. No estimator fixes a real
+platform difference, and a gate loose enough for x86 (~4e-3) would sit only 2.7x below what a 1%
+scaling reads. The scale and residual are still *printed* (ungated) because they make a future
+platform difference diagnosable in one run.
+
+Worth keeping rather than deleting because both vLLM skinning tests need a GPU, making this the
+only CPU-runnable check that the composer's weight transfer reproduces the original model.
 
 The vLLM check (`tests/vllm/test_generation_equivalence.py`) is the authoritative equivalence
 test, but it is **not** token-exact either. The skinned model runs its projections through the

Makefile

@@ -1,11 +1,31 @@
-.PHONY: test test-unit test-composer test-hf test-vllm test-integration test-all test-cpu test-gpu test-gpu-full test-tp test-regression-fast test-regression-real test-regression-hf test-peft-equiv test-peft-equiv-hf lint fmt pre-commit help
+.PHONY: test test-unit test-composer test-hf test-vllm test-integration test-all test-cpu test-gpu test-gpu-full test-tp test-e2e test-regression-fast test-regression-real test-regression-hf test-peft-equiv test-peft-equiv-hf lint fmt pre-commit help
 
 # Default Python (can override: make test PYTHON=python3.11)
 PYTHON ?= python
 
 # Pytest flags per CLAUDE.md guidelines
 PYTEST_FLAGS = -v -s --tb=short
 
+# ── The real-checkpoint (e2e) gate ───────────────────────────────────────────
+# Eleven modules across tests/hf, tests/integration and tests/vllm compose a real
+# ~3B Granite plus adapters, and skip themselves unless GRANITE_SWITCH_E2E_MODELS=1.
+# That flag is an opt-in for COST, not a capability check: nothing is wrong with the
+# machine when they skip, they are just expensive (a multi-GB download on a cold
+# cache). `make test-e2e` is the supported way to turn them on, so the knowledge of
+# which variable that is lives here instead of in everyone's shell history.
+#
+# GRANITE_SWITCH_E2E_DIR is where composed checkpoints are cached between runs; the
+# modules warm-reuse any dir that already has a config.json, so a second `make
+# test-e2e` pays the compose cost once. Override it to keep the cache off /tmp:
+#     make test-e2e GRANITE_SWITCH_E2E_DIR=/scratch/gs-e2e
+#
+# `?=` on both, so an already-exported value wins and CI can set them in the
+# environment instead.
+GRANITE_SWITCH_E2E_MODELS ?= 1
+GRANITE_SWITCH_E2E_DIR ?= /tmp/granite_switch_e2e
+E2E_ENV = GRANITE_SWITCH_E2E_MODELS=$(GRANITE_SWITCH_E2E_MODELS) \
+          GRANITE_SWITCH_E2E_DIR=$(GRANITE_SWITCH_E2E_DIR)
+
 # Individual test suites
 test-unit:
 	$(PYTHON) -m pytest tests/unit/ $(PYTEST_FLAGS)
@@ -34,6 +54,22 @@ test-gpu-full: test-gpu test-regression-fast  # GPU tests + regression suite
 test-tp:
 	$(PYTHON) -m pytest tests/vllm/test_tp_integration.py tests/vllm/test_tp_lora.py $(PYTEST_FLAGS)
 
+# Everything the integration suite covers, WITH the real-checkpoint modules enabled.
+#
+# Three directories, not just tests/integration/: the e2e-gated modules are spread
+# across tests/hf (4), tests/integration (5) and tests/vllm (1), so running only
+# tests/integration/ would leave six of them still skipping and the target would
+# quietly under-deliver.
+#
+# Separate pytest invocations on purpose. These modules each build a vLLM engine or
+# serve a model, and a per-directory process keeps one leaking CUDA context from
+# poisoning the next. No leading `-`: unlike the regression targets below, a failure
+# here should stop the run — it means a real checkpoint behaved wrongly.
+test-e2e:
+	$(E2E_ENV) $(PYTHON) -m pytest tests/hf/ $(PYTEST_FLAGS)
+	$(E2E_ENV) $(PYTHON) -m pytest tests/vllm/ $(PYTEST_FLAGS)
+	$(E2E_ENV) $(PYTHON) -m pytest tests/integration/ $(PYTEST_FLAGS)
+
 test-all: test-cpu test-gpu  # Everything
 
 # Regression test suites (no -x: run all tests even if some fail)
@@ -82,6 +118,10 @@ help:
 	@echo "  test-gpu       - Run all GPU tests"
 	@echo "  test-gpu-full  - Run GPU tests + regression-fast"
 	@echo "  test-tp        - Run TP integration + TP LoRA tests (requires 2+ GPUs)"
+	@echo "  test-e2e       - Run hf + vllm + integration WITH the real-checkpoint"
+	@echo "                   (GRANITE_SWITCH_E2E_MODELS) modules enabled. Composes a"
+	@echo "                   real ~3B checkpoint; cached in GRANITE_SWITCH_E2E_DIR"
+	@echo "                   (default /tmp/granite_switch_e2e). Requires GPU."
 	@echo "  test-all       - Run all tests"
 	@echo "  test-regression-fast  - Run fast synthetic regression tests"
 	@echo "  test-regression-real  - Run real model regression tests (requires_model)"

docs/SWITCH_KERNELS.md

@@ -10,8 +10,8 @@ rank-32 ones, and **up to 5.9× faster** with one rank-512 adapter. At idle (no
 ## Motivation and Availability
 
 As part of our mission to create a modular model architecture where adapter libraries are used to enhance
-a model, we encountered the problem that existing infrastructure for serving adapters introduced 
-unnecessary performance overheads. These overheads are noticeable even when a single adapter, regardless of 
+a model, we encountered the problem that existing infrastructure for serving adapters introduced
+unnecessary performance overheads. These overheads are noticeable even when a single adapter, regardless of
 its size, is introduced, but become very pronounced when the adapters have diverse sizes, which we anticipate will be a common
 scenario when a developer is integrating library components from different contributors into a single checkpoint.
 
@@ -36,7 +36,7 @@ expensive in our setting.
 ## What SWITCH does differently
 
 SWITCH is named after its design: **S**witched delta, **W**-ext fusion, **I**n-place,
-**T**iled, **C**onditional skip, **H**eterogeneous rank. 
+**T**iled, **C**onditional skip, **H**eterogeneous rank.
 
 Existing Punica technology, as implemented in vLLM, exhibits the following properties:
 
@@ -108,7 +108,7 @@ adapter. The figure below compares each system **as it currently implemented**:
 
 ![Idle vs adapter-active throughput at 12 adapters loaded, against the base model with LoRA off](figures/fig3_dispatch_3b_b32.png)
 
-Of note is that the overhead at idle that the SWITCH implementation in Granite Switch and vLLM exhibit is unrelated to the SWITCH kernels themselves, and rather 
+Of note is that the overhead at idle that the SWITCH implementation in Granite Switch and vLLM exhibit is unrelated to the SWITCH kernels themselves, and rather
 related to the fact that SWITCH is operating as a single computational graph, whereas vLLM does not run an adapter specific graph when no adapters are present. It would be relatively
 straightforward to implement SWITCH with two graphs, in which case the performance in the no adapter case would match vLLM/Punica exactly. As indicated earlier, we are currently focused in the setting where the model discovers which adapters need to be applied to which tokens through its switch layer, and currently there is no support in vLLM for adapting the computational graph once the model is already invoked.
 

pyproject.toml

@@ -82,15 +82,31 @@ vllm27     = ["vllm>=0.27.0,<0.28.0"]
 # `audio` is included so the audio tests can actually run: the ASR path needs
 # vLLM's audio deps (av/soundfile/resampy) at runtime, and no group pulled them
 # in before (integration tests failed with ModuleNotFoundError on a synced pod).
-dev        = ["pytest", "pytest-cov", { include-group = "vllm26" }, "granite-switch[hf,compose,audio]"]
-dev-vllm27 = ["pytest", "pytest-cov", { include-group = "vllm27" }, "granite-switch[hf,compose,audio]"]
-dev-vllm30 = ["pytest", "pytest-cov", { include-group = "vllm30" }, "granite-switch[hf,compose,audio]"]
+#
+# `quant` carries the two quantization backends tests/hf/test_quantization.py drives.
+# They are in every dev group because that module no longer opts in through an env
+# flag -- it runs wherever it can, and without these it would skip on importorskip
+# instead, which is the same silent nothing the flag used to produce.
+#
+# PINNED, unlike most of this file. These are JIT-compiling CUDA extensions, so a
+# new release can change whether a kernel builds at all on a given image -- the
+# Quanto float8 `quanto_cuda`/ptxas break is exactly that failure mode, and it is
+# inside one of these two. Floating them would mean a red suite traceable to a
+# background upgrade rather than to a commit. Both versions below are measured green
+# on the Vela A100 image. Raise them deliberately, with a GPU run.
+quant      = ["bitsandbytes==0.50.2", "optimum-quanto==0.2.7"]
+dev        = ["pytest", "pytest-cov", { include-group = "vllm26" }, { include-group = "quant" }, "granite-switch[hf,compose,audio]"]
+dev-vllm27 = ["pytest", "pytest-cov", { include-group = "vllm27" }, { include-group = "quant" }, "granite-switch[hf,compose,audio]"]
+dev-vllm30 = ["pytest", "pytest-cov", { include-group = "vllm30" }, { include-group = "quant" }, "granite-switch[hf,compose,audio]"]
 # The newest vLLM line, as an alias. The GPU workflow (derive_legs.py) and
 # CLAUDE.md ask for `dev-latest` rather than the concrete `dev-vllmNN`, so moving
 # the ceiling is repointing this one line instead of chasing the version through
 # every caller. Currently the 0.30 line.
 dev-latest = [{ include-group = "dev-vllm30" }]
-test       = ["pytest", "pytest-cov", "bitsandbytes", "optimum-quanto", { include-group = "dev" }]
+# The two quantization backends come in through `dev` -> `quant` now, PINNED. They
+# used to be listed here bare (unversioned), which would have silently overridden
+# the pin with "whatever is newest".
+test       = ["pytest", "pytest-cov", { include-group = "dev" }]
 
 [tool.uv]
 default-groups = ["vllm26"]

src/granite_switch/tutorials/vllm_server.py

@@ -6,6 +6,7 @@
 import os
 import signal
 import subprocess
+import sys
 import time
 from collections.abc import Sequence
 
@@ -24,8 +25,16 @@ def launch_vllm(
     extra_args: Sequence[str] = (),
     max_model_len: int = DEFAULT_MAX_MODEL_LEN,
 ) -> subprocess.Popen:
+    # sys.executable, not "python": the server has to run in the SAME environment
+    # as the caller, and the bare name resolves through PATH to whatever comes
+    # first. On a container with a system Python plus a project venv those are
+    # different interpreters, and the one on PATH need not have vLLM installed --
+    # five tests/integration serving modules failed exactly that way under
+    # `make test-e2e PYTHON=.../.venv/bin/python`, where the venv ran the tests
+    # but the child got /usr/local/bin/python and died on
+    # "No module named 'vllm'" before binding its port.
     cmd = [
-        "python",
+        sys.executable,
         "-m",
         "vllm.entrypoints.openai.api_server",
         "--model",