From 9858bfb2a8115e9ad71cc62d941c74e76f496e60 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 11 Nov 2026 13:39:22 +0800 Subject: [PATCH] Resolve "cuda" to the caller's current GPU torch.cuda.set_device is per thread, and ManyPred's prefetch and saving threads start on GPU 0. With set_device(1) and device="cuda", inputs already on cuda:1 were stacked on cuda:0 by the prefetch thread, then copied back, with record_stream given a stream from the other GPU. resolve_device now turns an unindexed "cuda" into cuda: on the calling thread, so every thread gets an explicit index. Tests: unit tests with a mocked current device, and a two-GPU test (skips with fewer). Codex reviewed the fix: no issues. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++++++++ src/manypred/devices.py | 13 +++++++++---- tests/test_core_fixes.py | 41 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3584625..d795bd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ All notable changes are listed here. Until 1.0, a minor version (0.x) may change the API; each change is listed under "Changed". +## [Unreleased] + +### Fixed + +- `device="cuda"` (or no device, on a CUDA machine) after `torch.cuda.set_device(n)` runs on + GPU n throughout. The prefetch thread used to read it as GPU 0, so inputs already on GPU n + were staged on GPU 0 and copied back. + ## [0.1.0] - 2026-11-11 First release. ManyPred is the inference engine of diff --git a/src/manypred/devices.py b/src/manypred/devices.py index 1d772ac..7e907f0 100644 --- a/src/manypred/devices.py +++ b/src/manypred/devices.py @@ -51,10 +51,15 @@ def default_device() -> torch.device: def resolve_device(device: str | torch.device | None) -> torch.device: - """A torch.device from a name, a device, or None (:func:`default_device`).""" - if device is None: - return default_device() - return torch.device(device) + """A torch.device from a name, a device, or None (:func:`default_device`). + + ``"cuda"`` without an index becomes the caller's current GPU (``"cuda:N"``). The current + GPU is set per thread (``torch.cuda.set_device``), so the prefetch and saving threads, + which start on GPU 0, have to be told which one the caller meant.""" + resolved = default_device() if device is None else torch.device(device) + if resolved.type == "cuda" and resolved.index is None and torch.cuda.is_available(): + resolved = torch.device("cuda", torch.cuda.current_device()) + return resolved def same_device(a: torch.device, b: torch.device) -> bool: diff --git a/tests/test_core_fixes.py b/tests/test_core_fixes.py index 26808f0..67a2d15 100644 --- a/tests/test_core_fixes.py +++ b/tests/test_core_fixes.py @@ -16,7 +16,7 @@ from manypred import normalize_numpy, normalize_torch, predict_array, predict_batch from manypred.batch import BatchSink from manypred.collate import collate -from manypred.devices import patch_for_mps, resolve_dtype, same_device +from manypred.devices import patch_for_mps, resolve_device, resolve_dtype, same_device from manypred.inputs import describe_inputs from manypred.normalize import build_normalizer, resolve_scene, validate_spec from manypred.outputs import TilePredictions @@ -380,6 +380,45 @@ def test_same_device_ignores_a_missing_index(): assert not same_device(torch.device("cpu"), torch.device("mps")) +def test_resolve_device_pins_cuda_to_the_callers_gpu(monkeypatch): + # the current GPU is per thread: worker threads start on GPU 0, so "cuda" has to be + # resolved to the caller's GPU before they start + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "current_device", lambda: 1) + assert resolve_device("cuda") == torch.device("cuda", 1) + assert resolve_device(torch.device("cuda")) == torch.device("cuda", 1) + assert resolve_device(None) == torch.device("cuda", 1) + assert resolve_device("cuda:0") == torch.device("cuda", 0) + assert resolve_device("cpu") == torch.device("cpu") + + +def test_resolve_device_leaves_cuda_alone_without_cuda(monkeypatch): + monkeypatch.setattr(torch.cuda, "is_available", lambda: False) + assert resolve_device("cuda") == torch.device("cuda") + + +@pytest.mark.cuda +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.device_count() < 2, reason="needs two GPUs" +) +def test_cuda_means_the_current_gpu_on_every_thread(): + previous = torch.cuda.current_device() + torch.cuda.set_device(1) + try: + model = torch.nn.Conv2d(2, 1, 1).to("cuda:1").eval() + chips = [torch.rand(2, 16, 16, device="cuda:1") for _ in range(6)] + torch.cuda.synchronize(0) + torch.cuda.reset_peak_memory_stats(0) + before = torch.cuda.memory_allocated(0) + out = predict_batch(model, chips, device="cuda", batch_size=2) + torch.cuda.synchronize(0) + assert torch.cuda.max_memory_allocated(0) == before # nothing staged on GPU 0 + expected = model(torch.stack(chips)).detach().cpu().numpy() + np.testing.assert_allclose(np.stack(out), expected, rtol=1e-5, atol=1e-6) + finally: + torch.cuda.set_device(previous) + + @pytest.mark.skipif(not torch.backends.mps.is_available(), reason="needs MPS") def test_a_batch_on_the_device_is_used_as_it_is(): batch = torch.rand(2, 3, 4, 4, device="mps:0")