Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,14 @@
All notable changes are listed here. Until 1.0, a minor version (0.x) may change the API; each
change is listed under "Changed".

## [Unreleased]

### Fixed

- `device="cuda"` (or no device, on a CUDA machine) after `torch.cuda.set_device(n)` runs on
GPU n throughout. The prefetch thread used to read it as GPU 0, so inputs already on GPU n
were staged on GPU 0 and copied back.

## [0.1.0] - 2026-11-11

First release. ManyPred is the inference engine of
Expand Down
13 changes: 9 additions & 4 deletions src/manypred/devices.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,15 @@ def default_device() -> torch.device:


def resolve_device(device: str | torch.device | None) -> torch.device:
"""A torch.device from a name, a device, or None (:func:`default_device`)."""
if device is None:
return default_device()
return torch.device(device)
"""A torch.device from a name, a device, or None (:func:`default_device`).

``"cuda"`` without an index becomes the caller's current GPU (``"cuda:N"``). The current
GPU is set per thread (``torch.cuda.set_device``), so the prefetch and saving threads,
which start on GPU 0, have to be told which one the caller meant."""
resolved = default_device() if device is None else torch.device(device)
if resolved.type == "cuda" and resolved.index is None and torch.cuda.is_available():
resolved = torch.device("cuda", torch.cuda.current_device())
return resolved


def same_device(a: torch.device, b: torch.device) -> bool:
Expand Down
41 changes: 40 additions & 1 deletion tests/test_core_fixes.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from manypred import normalize_numpy, normalize_torch, predict_array, predict_batch
from manypred.batch import BatchSink
from manypred.collate import collate
from manypred.devices import patch_for_mps, resolve_dtype, same_device
from manypred.devices import patch_for_mps, resolve_device, resolve_dtype, same_device
from manypred.inputs import describe_inputs
from manypred.normalize import build_normalizer, resolve_scene, validate_spec
from manypred.outputs import TilePredictions
Expand Down Expand Up @@ -380,6 +380,45 @@ def test_same_device_ignores_a_missing_index():
assert not same_device(torch.device("cpu"), torch.device("mps"))


def test_resolve_device_pins_cuda_to_the_callers_gpu(monkeypatch):
# the current GPU is per thread: worker threads start on GPU 0, so "cuda" has to be
# resolved to the caller's GPU before they start
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.cuda, "current_device", lambda: 1)
assert resolve_device("cuda") == torch.device("cuda", 1)
assert resolve_device(torch.device("cuda")) == torch.device("cuda", 1)
assert resolve_device(None) == torch.device("cuda", 1)
assert resolve_device("cuda:0") == torch.device("cuda", 0)
assert resolve_device("cpu") == torch.device("cpu")


def test_resolve_device_leaves_cuda_alone_without_cuda(monkeypatch):
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
assert resolve_device("cuda") == torch.device("cuda")


@pytest.mark.cuda
@pytest.mark.skipif(
not torch.cuda.is_available() or torch.cuda.device_count() < 2, reason="needs two GPUs"
)
def test_cuda_means_the_current_gpu_on_every_thread():
previous = torch.cuda.current_device()
torch.cuda.set_device(1)
try:
model = torch.nn.Conv2d(2, 1, 1).to("cuda:1").eval()
chips = [torch.rand(2, 16, 16, device="cuda:1") for _ in range(6)]
torch.cuda.synchronize(0)
torch.cuda.reset_peak_memory_stats(0)
before = torch.cuda.memory_allocated(0)
out = predict_batch(model, chips, device="cuda", batch_size=2)
torch.cuda.synchronize(0)
assert torch.cuda.max_memory_allocated(0) == before # nothing staged on GPU 0
expected = model(torch.stack(chips)).detach().cpu().numpy()
np.testing.assert_allclose(np.stack(out), expected, rtol=1e-5, atol=1e-6)
finally:
torch.cuda.set_device(previous)


@pytest.mark.skipif(not torch.backends.mps.is_available(), reason="needs MPS")
def test_a_batch_on_the_device_is_used_as_it_is():
batch = torch.rand(2, 3, 4, 4, device="mps:0")
Expand Down
Loading