diff --git a/pyproject.toml b/pyproject.toml index 09ed82f..3f5c3b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "polystore" -version = "0.6.2" +version = "0.6.3" description = "Framework-agnostic multi-backend storage abstraction for ML and scientific computing" readme = "README.md" requires-python = ">=3.11" diff --git a/src/polystore/__init__.py b/src/polystore/__init__.py index 9bc26a2..5345e12 100644 --- a/src/polystore/__init__.py +++ b/src/polystore/__init__.py @@ -36,6 +36,9 @@ "storage_registry": ".base", "Backend": ".constants", "MemoryType": ".constants", + "FileContent": ".file_content", + "HashedFileContent": ".file_content", + "UnhashedFileContent": ".file_content", "FileManager": ".filemanager", "DEFAULT_IMAGE_EXTENSIONS": ".formats", "FileFormat": ".formats", diff --git a/src/polystore/base.py b/src/polystore/base.py index 921c13f..305b7d1 100644 --- a/src/polystore/base.py +++ b/src/polystore/base.py @@ -12,6 +12,7 @@ import hashlib import logging +import stat import threading from abc import ABC, abstractmethod from dataclasses import dataclass @@ -23,6 +24,7 @@ from .constants import Backend from .exceptions import StorageResolutionError +from .file_content import FileContent, HashedFileContent, UnhashedFileContent if TYPE_CHECKING: import numpy as np @@ -395,6 +397,33 @@ def read_verified_source_bytes( raise ValueError("Verified resource SHA256 does not match its declaration.") return data + def file_content(self, file_path: str | Path) -> FileContent: + """Say what bytes the file at an absolute address holds, or why none can be stated. + + The physical file is the one this backend's path hook declares. A missing + file raises ``FileNotFoundError``; it is never an unhashed answer. + """ + # An empty base keeps a relative address relative, so it is refused below + # instead of being read from whatever the working directory happens to be. + declared_path = self.physical_source_path(file_path, base_path=Path()) + if declared_path is None: + if not self.exists(file_path): + raise FileNotFoundError(f"{type(self).__name__} has no file at {file_path}.") + return UnhashedFileContent( + f"{type(self).__name__} declares no physical file for {file_path}." + ) + physical_path = Path(declared_path) + if not physical_path.is_absolute(): + raise StorageResolutionError( + f"File content needs an absolute address, got {file_path!s}." + ) + mode = physical_path.stat().st_mode + if stat.S_ISDIR(mode): + return UnhashedFileContent(f"{physical_path} is a directory, not a file.") + if not stat.S_ISREG(mode): + return UnhashedFileContent(f"{physical_path} is not a regular file.") + return HashedFileContent.of_physical_file(physical_path) + def sample( self, file_path: Union[str, Path], diff --git a/src/polystore/bioformats_storage.py b/src/polystore/bioformats_storage.py index 180c783..5827c92 100644 --- a/src/polystore/bioformats_storage.py +++ b/src/polystore/bioformats_storage.py @@ -15,6 +15,7 @@ ) from .constants import Backend from .exceptions import StorageResolutionError +from .file_content import FileContent, UnhashedFileContent @dataclass(frozen=True, slots=True) @@ -108,6 +109,16 @@ def resolve_address( return ref.to_backend_address() return replace(ref, source_path=Path(base_path) / ref.source_path).to_backend_address() + def file_content(self, file_path: str | Path) -> FileContent: + """A plane address names part of a container, never a whole file's bytes.""" + ref = BioFormatsPlaneRef.from_backend_address(str(file_path)) + if not ref.source_path.exists(): + raise FileNotFoundError(f"Bio-Formats container not found: {ref.source_path}") + return UnhashedFileContent( + f"{type(self).__name__} address names plane {ref.plane_index} of series " + f"{ref.series_index} inside {ref.source_path}, not a whole file." + ) + def set_connection_params(self, params: dict[str, Any] | None) -> None: if params is not None: raise ValueError("BioFormatsStorageBackend has no connection parameters.") diff --git a/src/polystore/file_content.py b/src/polystore/file_content.py new file mode 100644 index 0000000..a196703 --- /dev/null +++ b/src/polystore/file_content.py @@ -0,0 +1,78 @@ +"""What bytes a stored file holds, as stated by the backend that owns it.""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from pathlib import Path +from typing import ClassVar + +from metaclass_registry import AutoRegisterMeta + +HASH_CHUNK_BYTES = 1024 * 1024 +SHA256_HEX_LENGTH = 64 +LOWERCASE_HEX_DIGITS = frozenset("0123456789abcdef") + + +class FileContent(metaclass=AutoRegisterMeta): + """Either the hash and size of a file's bytes, or the reason there is none. + + The two answers are registered under ``content_kind``; a stored answer names + its class by that key. + """ + + __registry_key__ = "content_kind" + __slots__ = () + + content_kind: ClassVar[str | None] = None + + +@dataclass(frozen=True, slots=True) +class HashedFileContent(FileContent): + """The SHA-256 and byte count of one physical file.""" + + content_kind: ClassVar[str] = "hashed" + + sha256: str + size_bytes: int + + def __post_init__(self) -> None: + if ( + not isinstance(self.sha256, str) + or len(self.sha256) != SHA256_HEX_LENGTH + or not LOWERCASE_HEX_DIGITS.issuperset(self.sha256) + ): + raise ValueError( + f"sha256 must be {SHA256_HEX_LENGTH} lowercase hex digits: {self.sha256!r}." + ) + if ( + not isinstance(self.size_bytes, int) + or isinstance(self.size_bytes, bool) + or self.size_bytes < 0 + ): + raise ValueError(f"size_bytes must be a nonnegative integer: {self.size_bytes!r}.") + + @classmethod + def of_physical_file(cls, path: Path) -> HashedFileContent: + """Hash a file chunk by chunk, counting the bytes that were hashed.""" + + digest = hashlib.sha256() + size_bytes = 0 + with Path(path).open("rb") as stream: + while chunk := stream.read(HASH_CHUNK_BYTES): + digest.update(chunk) + size_bytes += len(chunk) + return cls(sha256=digest.hexdigest(), size_bytes=size_bytes) + + +@dataclass(frozen=True, slots=True) +class UnhashedFileContent(FileContent): + """Why a backend cannot state the bytes of what a path names.""" + + content_kind: ClassVar[str] = "not_hashed" + + reason: str + + def __post_init__(self) -> None: + if not isinstance(self.reason, str) or not self.reason.strip(): + raise ValueError("An unhashed file content answer must give a reason.") diff --git a/src/polystore/filemanager.py b/src/polystore/filemanager.py index 26cd111..59987ce 100644 --- a/src/polystore/filemanager.py +++ b/src/polystore/filemanager.py @@ -27,6 +27,7 @@ import numpy as np from .exceptions import StorageResolutionError +from .file_content import FileContent from .formats import DEFAULT_IMAGE_EXTENSIONS from .virtual_workspace import SourcePixelRef @@ -258,6 +259,13 @@ def load_text(self, file_path: str | Path, backend: str) -> str: raise StorageResolutionError(f"Backend {backend!r} is not a DataSource.") return source.load_text(file_path) + def file_content(self, file_path: str | Path, backend: str) -> FileContent: + """Ask the declared source backend what bytes a file holds.""" + source = self._get_backend(backend) + if not isinstance(source, DataSource): + raise StorageResolutionError(f"Backend {backend!r} is not a DataSource.") + return source.file_content(file_path) + def load(self, file_path: str | Path, backend: str, **kwargs) -> Any: """ Load data from a file using the specified backend. diff --git a/src/polystore/memory.py b/src/polystore/memory.py index 432a65b..f84366d 100644 --- a/src/polystore/memory.py +++ b/src/polystore/memory.py @@ -18,6 +18,7 @@ from .base import StorageBackend from .constants import Backend from .exceptions import StorageResolutionError +from .file_content import FileContent, UnhashedFileContent logger = logging.getLogger(__name__) @@ -402,6 +403,18 @@ def is_dir(self, path: Union[str, Path]) -> bool: return True + def file_content(self, file_path: str | Path) -> FileContent: + """Memory entries are live objects; there are no file bytes to hash.""" + key = self._normalize(file_path) + if key not in self._memory_store: + raise FileNotFoundError(f"Memory path does not exist: {file_path}") + if self._memory_store[key] is None: + return UnhashedFileContent(f"Memory path {file_path} is a directory, not a file.") + return UnhashedFileContent( + f"{type(self).__name__} holds {file_path} as an object in process memory, " + "not as file bytes." + ) + def _resolve_path(self, path: Union[str, Path]) -> Optional[Any]: """Resolve one path through the backend's flat normalized-key namespace.""" return self._memory_store.get(self._normalize(path)) diff --git a/src/polystore/virtual_workspace.py b/src/polystore/virtual_workspace.py index d3fb4ce..2355231 100644 --- a/src/polystore/virtual_workspace.py +++ b/src/polystore/virtual_workspace.py @@ -22,6 +22,7 @@ ) from .constants import Backend from .exceptions import StorageResolutionError +from .file_content import FileContent, UnhashedFileContent from .metadata_writer import METADATA_CONFIG, MetadataConfig if TYPE_CHECKING: @@ -104,6 +105,22 @@ def resolved_address( ) ) + def file_content( + self, + registry: Mapping[str, BackendBase], + *, + base_path: Path, + ) -> FileContent: + """Say what bytes this reference names, as stated by the source owner.""" + + if self.source_axis_indices: + return UnhashedFileContent( + f"{self!r} selects part of its source file, so it names no whole file's bytes." + ) + return self.source_backend(registry).file_content( + self.resolved_address(registry, base_path=base_path) + ) + def load( self, registry: Mapping[str, BackendBase], @@ -342,6 +359,19 @@ def resolve_address( def load(self, file_path: Union[str, Path], **kwargs: Any) -> Any: return self._load_ref(self._resolve_ref(file_path), **kwargs) + def file_content(self, file_path: str | Path) -> FileContent: + """Answer for the source file a virtual path stands for.""" + if self.is_file(file_path): + return self._resolve_ref(file_path).file_content( + self._registry_for_ref(), + base_path=self.plate_root, + ) + if self.is_dir(file_path): + return UnhashedFileContent( + f"{type(self).__name__} path {file_path} is a directory, not a file." + ) + raise FileNotFoundError(f"Path not in virtual workspace mapping: {file_path}") + def sample( self, file_path: Union[str, Path], diff --git a/tests/test_file_content.py b/tests/test_file_content.py new file mode 100644 index 0000000..2145823 --- /dev/null +++ b/tests/test_file_content.py @@ -0,0 +1,224 @@ +"""The backend that owns a file says what bytes it holds, or why it cannot.""" + +import hashlib +import json +from pathlib import Path + +import numpy as np +import pytest + +from polystore import FileContent, FileManager, HashedFileContent, UnhashedFileContent +from polystore.bioformats_storage import BioFormatsPlaneRef, BioFormatsStorageBackend +from polystore.exceptions import StorageResolutionError +from polystore.file_content import HASH_CHUNK_BYTES +from polystore.metadata_writer import METADATA_CONFIG +from polystore.omero_local import OMEROLocalBackend +from polystore.virtual_workspace import SourcePixelRef, VirtualWorkspaceBackend + +SHA256_OF_NOTHING = hashlib.sha256(b"").hexdigest() + + +def _write(path: Path, data: bytes) -> Path: + path.write_bytes(data) + return path + + +def _workspace(root: Path, mapping: dict[str, SourcePixelRef], registry: dict) -> FileManager: + METADATA_CONFIG.metadata_path(root).write_text( + json.dumps( + { + METADATA_CONFIG.SUBDIRECTORIES_KEY: { + ".": { + "workspace_mapping": { + virtual: ref.to_workspace_mapping() for virtual, ref in mapping.items() + } + } + } + } + ) + ) + return FileManager({**registry, "virtual_workspace": VirtualWorkspaceBackend(root)}) + + +def test_disk_file_hash_and_size_match_its_bytes(file_manager, tmp_path): + data = b"plate A01 measurements\n" * 7 + path = _write(tmp_path / "results.csv", data) + + content = file_manager.file_content(path, "disk") + + assert content == HashedFileContent( + sha256=hashlib.sha256(data).hexdigest(), size_bytes=len(data) + ) + + +def test_one_changed_byte_changes_the_hash_but_not_the_size(file_manager, tmp_path): + path = _write(tmp_path / "image.bin", b"\x00" * 64) + before = file_manager.file_content(path, "disk") + + _write(path, b"\x00" * 63 + b"\x01") + after = file_manager.file_content(path, "disk") + + assert after.size_bytes == before.size_bytes == 64 + assert after.sha256 != before.sha256 + + +def test_empty_disk_file_is_hashed_not_unhashed(file_manager, tmp_path): + path = _write(tmp_path / "empty.txt", b"") + + assert file_manager.file_content(path, "disk") == HashedFileContent(SHA256_OF_NOTHING, 0) + + +def test_hash_streams_a_file_larger_than_one_chunk(file_manager, tmp_path, monkeypatch): + data = bytes(range(256)) * (HASH_CHUNK_BYTES // 256 * 2) + b"tail" + path = _write(tmp_path / "large.bin", data) + hashed_chunk_sizes: list[int] = [] + real_sha256 = hashlib.sha256 + + class RecordingSha256: + def __init__(self): + self._digest = real_sha256() + + def update(self, chunk): + hashed_chunk_sizes.append(len(chunk)) + self._digest.update(chunk) + + def hexdigest(self): + return self._digest.hexdigest() + + def whole_file_read(self): + raise AssertionError("file_content must not read the whole file into memory") + + monkeypatch.setattr(hashlib, "sha256", RecordingSha256) + monkeypatch.setattr(Path, "read_bytes", whole_file_read) + + content = file_manager.file_content(path, "disk") + + assert hashed_chunk_sizes == [HASH_CHUNK_BYTES, HASH_CHUNK_BYTES, len(b"tail")] + assert content == HashedFileContent(real_sha256(data).hexdigest(), len(data)) + + +def test_disk_directory_is_unhashed_with_a_reason(file_manager, tmp_path): + content = file_manager.file_content(tmp_path, "disk") + + assert isinstance(content, UnhashedFileContent) + assert "is a directory" in content.reason + assert str(tmp_path) in content.reason + + +def test_missing_disk_file_raises(file_manager, tmp_path): + with pytest.raises(FileNotFoundError): + file_manager.file_content(tmp_path / "absent.tif", "disk") + + +def test_relative_disk_path_is_refused_rather_than_resolved_against_cwd( + file_manager, tmp_path, monkeypatch +): + _write(tmp_path / "relative.bin", b"x") + monkeypatch.chdir(tmp_path) + + with pytest.raises(StorageResolutionError, match="absolute"): + file_manager.file_content("relative.bin", "disk") + + +def test_memory_file_is_unhashed_and_the_reason_names_the_backend(file_manager, tmp_path): + # A same-named disk file must not be mistaken for the memory entry. + path = _write(tmp_path / "array.npy", b"disk bytes") + file_manager.ensure_directory(tmp_path, "memory") + file_manager.save(np.zeros((2, 2)), path, "memory") + + content = file_manager.file_content(path, "memory") + + assert isinstance(content, UnhashedFileContent) + assert "MemoryStorageBackend" in content.reason + + +def test_memory_directory_is_unhashed_and_missing_memory_path_raises(file_manager): + file_manager.ensure_directory("/plate/images", "memory") + + directory = file_manager.file_content("/plate/images", "memory") + + assert isinstance(directory, UnhashedFileContent) + assert "is a directory" in directory.reason + with pytest.raises(FileNotFoundError): + file_manager.file_content("/plate/images/absent.npy", "memory") + + +def test_virtual_file_has_the_content_of_the_whole_source_file_it_names(registry, tmp_path): + np.save(tmp_path / "source.npy", np.arange(6, dtype=np.uint16)) + data = (tmp_path / "source.npy").read_bytes() + manager = _workspace(tmp_path, {"virtual.npy": SourcePixelRef("disk", "source.npy")}, registry) + + content = manager.file_content(tmp_path / "virtual.npy", "virtual_workspace") + + assert content == HashedFileContent(hashlib.sha256(data).hexdigest(), len(data)) + + +def test_virtual_file_that_is_part_of_a_source_file_is_unhashed(registry, tmp_path): + np.save(tmp_path / "source.npy", np.zeros((2, 4, 5), dtype=np.uint16)) + manager = _workspace( + tmp_path, {"images/plane.npy": SourcePixelRef("disk", "source.npy", (1,))}, registry + ) + + plane = manager.file_content(tmp_path / "images" / "plane.npy", "virtual_workspace") + directory = manager.file_content(tmp_path / "images", "virtual_workspace") + + assert isinstance(plane, UnhashedFileContent) + assert "selects part of its source file" in plane.reason + assert isinstance(directory, UnhashedFileContent) + assert "is a directory" in directory.reason + with pytest.raises(FileNotFoundError): + manager.file_content(tmp_path / "images" / "absent.npy", "virtual_workspace") + + +def test_bioformats_plane_is_unhashed_because_it_is_part_of_its_container(tmp_path): + container = _write(tmp_path / "plate.czi", b"two planes in one container") + manager = FileManager({"bioformats": BioFormatsStorageBackend()}) + + content = manager.file_content( + BioFormatsPlaneRef(container, 0, 1).to_backend_address(), "bioformats" + ) + + assert isinstance(content, UnhashedFileContent) + assert "BioFormatsStorageBackend" in content.reason + assert "plane 1" in content.reason + with pytest.raises(FileNotFoundError): + manager.file_content( + BioFormatsPlaneRef(tmp_path / "absent.czi", 0, 0).to_backend_address(), "bioformats" + ) + + +def test_backend_that_cannot_read_files_is_refused(registry): + sink = object.__new__(OMEROLocalBackend) + manager = FileManager({**registry, "omero_local": sink}) + + with pytest.raises(StorageResolutionError, match="not a DataSource"): + manager.file_content("/plate/image.tif", "omero_local") + + +def test_each_answer_is_registered_under_the_key_that_names_it_in_stored_data(): + assert dict(FileContent.__registry__) == { + "hashed": HashedFileContent, + "not_hashed": UnhashedFileContent, + } + + +@pytest.mark.parametrize( + "sha256, size_bytes", + [ + (SHA256_OF_NOTHING.upper(), 0), + (SHA256_OF_NOTHING[:-1], 0), + (SHA256_OF_NOTHING[:-1] + "g", 0), + (SHA256_OF_NOTHING, -1), + (SHA256_OF_NOTHING, True), + (SHA256_OF_NOTHING, 1.0), + ], +) +def test_hashed_content_refuses_a_malformed_hash_or_size(sha256, size_bytes): + with pytest.raises(ValueError): + HashedFileContent(sha256=sha256, size_bytes=size_bytes) + + +@pytest.mark.parametrize("reason", ["", " "]) +def test_unhashed_content_requires_a_reason(reason): + with pytest.raises(ValueError): + UnhashedFileContent(reason=reason) diff --git a/uv.lock b/uv.lock index 09af66c..fb2946e 100644 --- a/uv.lock +++ b/uv.lock @@ -2905,7 +2905,7 @@ wheels = [ [[package]] name = "polystore" -version = "0.6.2" +version = "0.6.3" source = { editable = "." } dependencies = [ { name = "arraybridge" },