0.6.3: backends state a file's content hash and size, or why they cannot

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "polystore"
-version = "0.6.2"
+version = "0.6.3"
 description = "Framework-agnostic multi-backend storage abstraction for ML and scientific computing"
 readme = "README.md"
 requires-python = ">=3.11"

src/polystore/__init__.py

@@ -36,6 +36,9 @@
     "storage_registry": ".base",
     "Backend": ".constants",
     "MemoryType": ".constants",
+    "FileContent": ".file_content",
+    "HashedFileContent": ".file_content",
+    "UnhashedFileContent": ".file_content",
     "FileManager": ".filemanager",
     "DEFAULT_IMAGE_EXTENSIONS": ".formats",
     "FileFormat": ".formats",

src/polystore/base.py

@@ -12,6 +12,7 @@
 
 import hashlib
 import logging
+import stat
 import threading
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
@@ -23,6 +24,7 @@
 
 from .constants import Backend
 from .exceptions import StorageResolutionError
+from .file_content import FileContent, HashedFileContent, UnhashedFileContent
 
 if TYPE_CHECKING:
     import numpy as np
@@ -395,6 +397,33 @@ def read_verified_source_bytes(
             raise ValueError("Verified resource SHA256 does not match its declaration.")
         return data
 
+    def file_content(self, file_path: str | Path) -> FileContent:
+        """Say what bytes the file at an absolute address holds, or why none can be stated.
+
+        The physical file is the one this backend's path hook declares. A missing
+        file raises ``FileNotFoundError``; it is never an unhashed answer.
+        """
+        # An empty base keeps a relative address relative, so it is refused below
+        # instead of being read from whatever the working directory happens to be.
+        declared_path = self.physical_source_path(file_path, base_path=Path())
+        if declared_path is None:
+            if not self.exists(file_path):
+                raise FileNotFoundError(f"{type(self).__name__} has no file at {file_path}.")
+            return UnhashedFileContent(
+                f"{type(self).__name__} declares no physical file for {file_path}."
+            )
+        physical_path = Path(declared_path)
+        if not physical_path.is_absolute():
+            raise StorageResolutionError(
+                f"File content needs an absolute address, got {file_path!s}."
+            )
+        mode = physical_path.stat().st_mode
+        if stat.S_ISDIR(mode):
+            return UnhashedFileContent(f"{physical_path} is a directory, not a file.")
+        if not stat.S_ISREG(mode):
+            return UnhashedFileContent(f"{physical_path} is not a regular file.")
+        return HashedFileContent.of_physical_file(physical_path)
+
     def sample(
         self,
         file_path: Union[str, Path],

src/polystore/bioformats_storage.py

@@ -15,6 +15,7 @@
 )
 from .constants import Backend
 from .exceptions import StorageResolutionError
+from .file_content import FileContent, UnhashedFileContent
 
 
 @dataclass(frozen=True, slots=True)
@@ -108,6 +109,16 @@ def resolve_address(
             return ref.to_backend_address()
         return replace(ref, source_path=Path(base_path) / ref.source_path).to_backend_address()
 
+    def file_content(self, file_path: str | Path) -> FileContent:
+        """A plane address names part of a container, never a whole file's bytes."""
+        ref = BioFormatsPlaneRef.from_backend_address(str(file_path))
+        if not ref.source_path.exists():
+            raise FileNotFoundError(f"Bio-Formats container not found: {ref.source_path}")
+        return UnhashedFileContent(
+            f"{type(self).__name__} address names plane {ref.plane_index} of series "
+            f"{ref.series_index} inside {ref.source_path}, not a whole file."
+        )
+
     def set_connection_params(self, params: dict[str, Any] | None) -> None:
         if params is not None:
             raise ValueError("BioFormatsStorageBackend has no connection parameters.")

src/polystore/file_content.py

@@ -0,0 +1,78 @@
+"""What bytes a stored file holds, as stated by the backend that owns it."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import ClassVar
+
+from metaclass_registry import AutoRegisterMeta
+
+HASH_CHUNK_BYTES = 1024 * 1024
+SHA256_HEX_LENGTH = 64
+LOWERCASE_HEX_DIGITS = frozenset("0123456789abcdef")
+
+
+class FileContent(metaclass=AutoRegisterMeta):
+    """Either the hash and size of a file's bytes, or the reason there is none.
+
+    The two answers are registered under ``content_kind``; a stored answer names
+    its class by that key.
+    """
+
+    __registry_key__ = "content_kind"
+    __slots__ = ()
+
+    content_kind: ClassVar[str | None] = None
+
+
+@dataclass(frozen=True, slots=True)
+class HashedFileContent(FileContent):
+    """The SHA-256 and byte count of one physical file."""
+
+    content_kind: ClassVar[str] = "hashed"
+
+    sha256: str
+    size_bytes: int
+
+    def __post_init__(self) -> None:
+        if (
+            not isinstance(self.sha256, str)
+            or len(self.sha256) != SHA256_HEX_LENGTH
+            or not LOWERCASE_HEX_DIGITS.issuperset(self.sha256)
+        ):
+            raise ValueError(
+                f"sha256 must be {SHA256_HEX_LENGTH} lowercase hex digits: {self.sha256!r}."
+            )
+        if (
+            not isinstance(self.size_bytes, int)
+            or isinstance(self.size_bytes, bool)
+            or self.size_bytes < 0
+        ):
+            raise ValueError(f"size_bytes must be a nonnegative integer: {self.size_bytes!r}.")
+
+    @classmethod
+    def of_physical_file(cls, path: Path) -> HashedFileContent:
+        """Hash a file chunk by chunk, counting the bytes that were hashed."""
+
+        digest = hashlib.sha256()
+        size_bytes = 0
+        with Path(path).open("rb") as stream:
+            while chunk := stream.read(HASH_CHUNK_BYTES):
+                digest.update(chunk)
+                size_bytes += len(chunk)
+        return cls(sha256=digest.hexdigest(), size_bytes=size_bytes)
+
+
+@dataclass(frozen=True, slots=True)
+class UnhashedFileContent(FileContent):
+    """Why a backend cannot state the bytes of what a path names."""
+
+    content_kind: ClassVar[str] = "not_hashed"
+
+    reason: str
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.reason, str) or not self.reason.strip():
+            raise ValueError("An unhashed file content answer must give a reason.")

src/polystore/filemanager.py

@@ -27,6 +27,7 @@
     import numpy as np
 
 from .exceptions import StorageResolutionError
+from .file_content import FileContent
 from .formats import DEFAULT_IMAGE_EXTENSIONS
 from .virtual_workspace import SourcePixelRef
 
@@ -258,6 +259,13 @@ def load_text(self, file_path: str | Path, backend: str) -> str:
             raise StorageResolutionError(f"Backend {backend!r} is not a DataSource.")
         return source.load_text(file_path)
 
+    def file_content(self, file_path: str | Path, backend: str) -> FileContent:
+        """Ask the declared source backend what bytes a file holds."""
+        source = self._get_backend(backend)
+        if not isinstance(source, DataSource):
+            raise StorageResolutionError(f"Backend {backend!r} is not a DataSource.")
+        return source.file_content(file_path)
+
     def load(self, file_path: str | Path, backend: str, **kwargs) -> Any:
         """
         Load data from a file using the specified backend.

src/polystore/memory.py

@@ -18,6 +18,7 @@
 from .base import StorageBackend
 from .constants import Backend
 from .exceptions import StorageResolutionError
+from .file_content import FileContent, UnhashedFileContent
 
 logger = logging.getLogger(__name__)
 
@@ -402,6 +403,18 @@ def is_dir(self, path: Union[str, Path]) -> bool:
 
         return True
 
+    def file_content(self, file_path: str | Path) -> FileContent:
+        """Memory entries are live objects; there are no file bytes to hash."""
+        key = self._normalize(file_path)
+        if key not in self._memory_store:
+            raise FileNotFoundError(f"Memory path does not exist: {file_path}")
+        if self._memory_store[key] is None:
+            return UnhashedFileContent(f"Memory path {file_path} is a directory, not a file.")
+        return UnhashedFileContent(
+            f"{type(self).__name__} holds {file_path} as an object in process memory, "
+            "not as file bytes."
+        )
+
     def _resolve_path(self, path: Union[str, Path]) -> Optional[Any]:
         """Resolve one path through the backend's flat normalized-key namespace."""
         return self._memory_store.get(self._normalize(path))

src/polystore/virtual_workspace.py

@@ -22,6 +22,7 @@
 )
 from .constants import Backend
 from .exceptions import StorageResolutionError
+from .file_content import FileContent, UnhashedFileContent
 from .metadata_writer import METADATA_CONFIG, MetadataConfig
 
 if TYPE_CHECKING:
@@ -104,6 +105,22 @@ def resolved_address(
             )
         )
 
+    def file_content(
+        self,
+        registry: Mapping[str, BackendBase],
+        *,
+        base_path: Path,
+    ) -> FileContent:
+        """Say what bytes this reference names, as stated by the source owner."""
+
+        if self.source_axis_indices:
+            return UnhashedFileContent(
+                f"{self!r} selects part of its source file, so it names no whole file's bytes."
+            )
+        return self.source_backend(registry).file_content(
+            self.resolved_address(registry, base_path=base_path)
+        )
+
     def load(
         self,
         registry: Mapping[str, BackendBase],
@@ -342,6 +359,19 @@ def resolve_address(
     def load(self, file_path: Union[str, Path], **kwargs: Any) -> Any:
         return self._load_ref(self._resolve_ref(file_path), **kwargs)
 
+    def file_content(self, file_path: str | Path) -> FileContent:
+        """Answer for the source file a virtual path stands for."""
+        if self.is_file(file_path):
+            return self._resolve_ref(file_path).file_content(
+                self._registry_for_ref(),
+                base_path=self.plate_root,
+            )
+        if self.is_dir(file_path):
+            return UnhashedFileContent(
+                f"{type(self).__name__} path {file_path} is a directory, not a file."
+            )
+        raise FileNotFoundError(f"Path not in virtual workspace mapping: {file_path}")
+
     def sample(
         self,
         file_path: Union[str, Path],