diff --git a/CHANGELOG.md b/CHANGELOG.md index 3584625..d795bd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ All notable changes are listed here. Until 1.0, a minor version (0.x) may change the API; each change is listed under "Changed". +## [Unreleased] + +### Fixed + +- `device="cuda"` (or no device, on a CUDA machine) after `torch.cuda.set_device(n)` runs on + GPU n throughout. The prefetch thread used to read it as GPU 0, so inputs already on GPU n + were staged on GPU 0 and copied back. + ## [0.1.0] - 2026-11-11 First release. ManyPred is the inference engine of diff --git a/src/manypred/devices.py b/src/manypred/devices.py index 1d772ac..7e907f0 100644 --- a/src/manypred/devices.py +++ b/src/manypred/devices.py @@ -51,10 +51,15 @@ def default_device() -> torch.device: def resolve_device(device: str | torch.device | None) -> torch.device: - """A torch.device from a name, a device, or None (:func:`default_device`).""" - if device is None: - return default_device() - return torch.device(device) + """A torch.device from a name, a device, or None (:func:`default_device`). + + ``"cuda"`` without an index becomes the caller's current GPU (``"cuda:N"``). The current + GPU is set per thread (``torch.cuda.set_device``), so the prefetch and saving threads, + which start on GPU 0, have to be told which one the caller meant.""" + resolved = default_device() if device is None else torch.device(device) + if resolved.type == "cuda" and resolved.index is None and torch.cuda.is_available(): + resolved = torch.device("cuda", torch.cuda.current_device()) + return resolved def same_device(a: torch.device, b: torch.device) -> bool: diff --git a/tests/test_core_fixes.py b/tests/test_core_fixes.py index 26808f0..67a2d15 100644 --- a/tests/test_core_fixes.py +++ b/tests/test_core_fixes.py @@ -16,7 +16,7 @@ from manypred import normalize_numpy, normalize_torch, predict_array, predict_batch from manypred.batch import BatchSink from manypred.collate import collate -from manypred.devices import patch_for_mps, resolve_dtype, same_device +from manypred.devices import patch_for_mps, resolve_device, resolve_dtype, same_device from manypred.inputs import describe_inputs from manypred.normalize import build_normalizer, resolve_scene, validate_spec from manypred.outputs import TilePredictions @@ -380,6 +380,45 @@ def test_same_device_ignores_a_missing_index(): assert not same_device(torch.device("cpu"), torch.device("mps")) +def test_resolve_device_pins_cuda_to_the_callers_gpu(monkeypatch): + # the current GPU is per thread: worker threads start on GPU 0, so "cuda" has to be + # resolved to the caller's GPU before they start + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "current_device", lambda: 1) + assert resolve_device("cuda") == torch.device("cuda", 1) + assert resolve_device(torch.device("cuda")) == torch.device("cuda", 1) + assert resolve_device(None) == torch.device("cuda", 1) + assert resolve_device("cuda:0") == torch.device("cuda", 0) + assert resolve_device("cpu") == torch.device("cpu") + + +def test_resolve_device_leaves_cuda_alone_without_cuda(monkeypatch): + monkeypatch.setattr(torch.cuda, "is_available", lambda: False) + assert resolve_device("cuda") == torch.device("cuda") + + +@pytest.mark.cuda +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.device_count() < 2, reason="needs two GPUs" +) +def test_cuda_means_the_current_gpu_on_every_thread(): + previous = torch.cuda.current_device() + torch.cuda.set_device(1) + try: + model = torch.nn.Conv2d(2, 1, 1).to("cuda:1").eval() + chips = [torch.rand(2, 16, 16, device="cuda:1") for _ in range(6)] + torch.cuda.synchronize(0) + torch.cuda.reset_peak_memory_stats(0) + before = torch.cuda.memory_allocated(0) + out = predict_batch(model, chips, device="cuda", batch_size=2) + torch.cuda.synchronize(0) + assert torch.cuda.max_memory_allocated(0) == before # nothing staged on GPU 0 + expected = model(torch.stack(chips)).detach().cpu().numpy() + np.testing.assert_allclose(np.stack(out), expected, rtol=1e-5, atol=1e-6) + finally: + torch.cuda.set_device(previous) + + @pytest.mark.skipif(not torch.backends.mps.is_available(), reason="needs MPS") def test_a_batch_on_the_device_is_used_as_it_is(): batch = torch.rand(2, 3, 4, 4, device="mps:0")