From ac230c46a12d3d8dbd586669f788b84562c87373 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 10 Oct 2026 08:12:48 -0400 Subject: [PATCH 1/2] List and report disk-passthrough saves as zarr backend files ZarrStorageBackend saves .json/.csv/.txt/.roi.zip/.zip outputs through the disk backend, and exists() already answers for them, but list_files() and is_dir() only looked inside the zarr store. A directory holding only such saves (for example a result directory of CSV tables and ROI archives) was therefore reported as missing and empty by the backend that wrote it. list_files() now includes passthrough files beside the arrays (never the store's own metadata files), and is_dir() is true for a directory holding passthrough saves. Co-Authored-By: Claude Opus 5.5 --- src/polystore/zarr.py | 31 +++++++++++++++++++++++++++++++ tests/test_zarr_backend.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/src/polystore/zarr.py b/src/polystore/zarr.py index 1ce93d2..84a0bf9 100644 --- a/src/polystore/zarr.py +++ b/src/polystore/zarr.py @@ -39,6 +39,7 @@ DEFAULT_PLATE_NAME = os.getenv("POLYSTORE_PLATE_NAME", "Polystore_Plate") DISK_PASSTHROUGH_EXTENSIONS = (".json", ".csv", ".txt", ".roi.zip", ".zip") +ZARR_METADATA_FILENAMES = frozenset({"zarr.json", ".zgroup", ".zarray", ".zattrs", ".zmetadata"}) def _get_attr(attrs: Any, key: str): @@ -129,6 +130,26 @@ def output_format(self): def _is_disk_passthrough_path(path: str | Path) -> bool: return str(path).endswith(DISK_PASSTHROUGH_EXTENSIONS) + @classmethod + def _disk_passthrough_files(cls, directory: str | Path, *, recursive: bool) -> list[Path]: + """Files this backend saved on disk inside ``directory``, never its own metadata.""" + root = Path(directory) + if not root.is_dir(): + return [] + entries = root.rglob("*") if recursive else root.iterdir() + return sorted( + entry + for entry in entries + if entry.name not in ZARR_METADATA_FILENAMES + and cls._is_disk_passthrough_path(entry) + and entry.is_file() + ) + + @staticmethod + def _declares_array_store(directory: str | Path) -> bool: + root = Path(directory) + return (root / "zarr.json").is_file() or (root / ".zgroup").is_file() + def supports_file_path(self, path: str | Path) -> bool: return self._is_disk_passthrough_path(path) or super().supports_file_path(path) @@ -722,6 +743,12 @@ def _matches_filters(name: str) -> bool: return any(name.lower().endswith(ext.lower()) for ext in extensions) return True + # Disk-passthrough saves are this backend's files too: list them beside the arrays. + saved_on_disk = self._disk_passthrough_files(directory, recursive=recursive) + result.extend(path for path in saved_on_disk if _matches_filters(path.name)) + if saved_on_disk and not self._declares_array_store(directory): + return result + try: # Open zarr group and traverse OME-ZARR structure location = OmeZarrLocation(store, mode="r") @@ -953,6 +980,10 @@ def is_dir(self, path: str | Path) -> bool: NotADirectoryError: If resolved target is not a group StorageResolutionError: For symlink cycles or other failures """ + if not self._declares_array_store(path) and self._disk_passthrough_files( + path, recursive=False + ): + return True if not isinstance(self._resolved_node(path), zarr.Group): raise NotADirectoryError(f"Zarr path is an array (file): {path}") return True diff --git a/tests/test_zarr_backend.py b/tests/test_zarr_backend.py index 55d1630..04cfb9a 100644 --- a/tests/test_zarr_backend.py +++ b/tests/test_zarr_backend.py @@ -65,6 +65,34 @@ def test_zarr_declared_passthrough_support_batch_save_and_native_readback(tmp_pa assert backend.physical_source_path(path, base_path=tmp_path) == path +def test_zarr_lists_and_reports_directories_of_its_passthrough_saves(tmp_path): + """A result directory holding only passthrough saves is this backend's directory.""" + backend = ZarrStorageBackend() + results = tmp_path / "results" + paths = [results / "A01_details.csv", results / "A01_rois.roi.zip"] + for path in paths: + backend.save("payload" if path.suffix == ".csv" else b"zip", path) + assert all(backend.exists(path) for path in paths) + assert backend.is_dir(results) + assert backend.list_files(results) == sorted(paths) + assert backend.list_files(results, extensions={".csv"}) == [paths[0]] + assert backend.list_files(results, extensions={".tif"}) == [] + + +def test_zarr_lists_passthrough_saves_beside_arrays_without_store_metadata(tmp_path): + backend = ZarrStorageBackend() + store = tmp_path / "images" + array_path = store / "A01_s1.tif" + text_path = store / "positions.json" + backend.save(np.zeros((2, 3), dtype=np.uint16), array_path) + backend.save("[]", text_path) + assert backend.is_dir(store) + listed = backend.list_files(store) + assert text_path in listed + assert not any(path.name in zarr_module.ZARR_METADATA_FILENAMES for path in listed) + assert backend.list_files(store, extensions={".json"}) == [text_path] + + @pytest.mark.parametrize("reverse", [False, True]) def test_zarr_mixed_passthrough_array_batch_rejects_before_any_writes(tmp_path, reverse): backend = ZarrStorageBackend() From 82bf0b064c12625b4c60a9b88d851c382782585c Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 10 Oct 2026 08:40:49 -0400 Subject: [PATCH 2/2] Delete every disk-passthrough save through the disk backend delete() kept its own extension list (.json/.csv/.txt) instead of the declared DISK_PASSTHROUGH_EXTENSIONS, so .roi.zip and .zip files this backend saved on disk were opened as zarr groups and failed to delete. Re-running a pipeline over existing zarr-mode outputs therefore failed when it replaced a saved ROI archive. Co-Authored-By: Claude Opus 5.5 --- src/polystore/zarr.py | 4 +--- tests/test_zarr_backend.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/polystore/zarr.py b/src/polystore/zarr.py index 84a0bf9..27f7b4e 100644 --- a/src/polystore/zarr.py +++ b/src/polystore/zarr.py @@ -791,9 +791,7 @@ def delete(self, path: str | Path) -> None: IsADirectoryError: If path is a non-empty group StorageResolutionError: For unexpected failures """ - # Passthrough to disk backend for text files (JSON, CSV, TXT) - path_str = str(path) - if path_str.endswith((".json", ".csv", ".txt")): + if self._is_disk_passthrough_path(path): from .backend_registry import get_backend_instance disk_backend = get_backend_instance(Backend.DISK.value) diff --git a/tests/test_zarr_backend.py b/tests/test_zarr_backend.py index 04cfb9a..3c151c5 100644 --- a/tests/test_zarr_backend.py +++ b/tests/test_zarr_backend.py @@ -79,6 +79,16 @@ def test_zarr_lists_and_reports_directories_of_its_passthrough_saves(tmp_path): assert backend.list_files(results, extensions={".tif"}) == [] +@pytest.mark.parametrize("name", ["table.csv", "rois.roi.zip", "bundle.zip"]) +def test_zarr_deletes_every_passthrough_save_it_wrote(tmp_path, name): + backend = ZarrStorageBackend() + path = tmp_path / "results" / name + backend.save("payload" if name.endswith(".csv") else b"zip", path) + backend.delete(path) + assert not path.exists() + assert not backend.exists(path) + + def test_zarr_lists_passthrough_saves_beside_arrays_without_store_metadata(tmp_path): backend = ZarrStorageBackend() store = tmp_path / "images"