From 00050271d8ac6485fd72b71f69ccee6c7bd458fa Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 12 Oct 2026 15:29:43 -0400 Subject: [PATCH] 0.6.3: arraybridge 0.5.0 storage_numpy_array asks the library that made the array to download it (ArrayLibrary.of(data).to_numpy(data)); it no longer names a device, and the removed gpu_id keyword is gone. polystore.constants no longer re-exports CPU_MEMORY_TYPES, GPU_MEMORY_TYPES and SUPPORTED_MEMORY_TYPES, which arraybridge 0.5.0 removed; nothing read them. polystore.MemoryType stays. polystore.streaming.handlers is deleted: it imported metaclass_registry.lazy, which metaclass-registry 0.4 (already required) does not have, so the package could not be imported and nothing imported it. A new test imports every module in a child interpreter and fails on any first-party import error. Co-Authored-By: Claude Opus 5.5 --- docs/source/api/backends.rst | 2 +- pyproject.toml | 4 +- src/polystore/array_payload.py | 17 +- src/polystore/constants.py | 13 +- src/polystore/streaming/handlers/__init__.py | 51 ----- .../streaming/handlers/fiji_images.py | 70 ------ src/polystore/streaming/handlers/fiji_rois.py | 206 ------------------ tests/test_array_payload.py | 40 ++-- tests/test_every_module_imports.py | 46 ++++ uv.lock | 11 +- 10 files changed, 81 insertions(+), 379 deletions(-) delete mode 100644 src/polystore/streaming/handlers/__init__.py delete mode 100644 src/polystore/streaming/handlers/fiji_images.py delete mode 100644 src/polystore/streaming/handlers/fiji_rois.py create mode 100644 tests/test_every_module_imports.py diff --git a/docs/source/api/backends.rst b/docs/source/api/backends.rst index cad2bcf..5985c77 100644 --- a/docs/source/api/backends.rst +++ b/docs/source/api/backends.rst @@ -37,7 +37,7 @@ Array payload boundary ---------------------- ArrayBridge owns the memory-type taxonomy and framework conversion semantics. -``polystore.MemoryType`` is a compatibility re-export of +``polystore.MemoryType`` is a re-export of ``arraybridge.MemoryType`` with the same class identity. Disk, Zarr, and streaming sinks normalize supported framework arrays through ArrayBridge when a host NumPy payload is required; individual storage backends do not maintain diff --git a/pyproject.toml b/pyproject.toml index 09ed82f..8f95d16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "polystore" -version = "0.6.2" +version = "0.6.3" description = "Framework-agnostic multi-backend storage abstraction for ML and scientific computing" readme = "README.md" requires-python = ">=3.11" @@ -37,7 +37,7 @@ classifiers = [ ] dependencies = [ - "arraybridge>=0.3.0", + "arraybridge>=0.5.0,<0.6", "numpy>=1.26.0", "portalocker>=2.8.0", # Cross-platform file locking "metaclass-registry>=0.4.0", diff --git a/src/polystore/array_payload.py b/src/polystore/array_payload.py index b1590b8..e326e79 100644 --- a/src/polystore/array_payload.py +++ b/src/polystore/array_payload.py @@ -1,24 +1,15 @@ """ArrayBridge-backed normalization at persistent storage boundaries.""" -from typing import Any - import numpy as np -from arraybridge import ArrayPayload, convert_memory, detect_memory_type -from arraybridge.types import MemoryType +from arraybridge import ArrayPayload +from arraybridge.libraries import ArrayLibrary -def storage_numpy_array(data: Any) -> np.ndarray: +def storage_numpy_array(data: object) -> np.ndarray: """Return one supported array payload as a host NumPy array.""" if isinstance(data, ArrayPayload): data = data.array_payload_data() if isinstance(data, np.ndarray): return data - source_type = detect_memory_type(data) - converted = convert_memory( - data, - source_type=source_type, - target_type=MemoryType.NUMPY, - gpu_id=0, - ) - return np.asarray(converted) + return ArrayLibrary.of(data).to_numpy(data) diff --git a/src/polystore/constants.py b/src/polystore/constants.py index 4c8d0fa..f56fdf2 100644 --- a/src/polystore/constants.py +++ b/src/polystore/constants.py @@ -4,18 +4,7 @@ from enum import Enum -from arraybridge.types import ( - CPU_MEMORY_TYPES as CPU_MEMORY_TYPES, -) -from arraybridge.types import ( - GPU_MEMORY_TYPES as GPU_MEMORY_TYPES, -) -from arraybridge.types import ( - SUPPORTED_MEMORY_TYPES as SUPPORTED_MEMORY_TYPES, -) -from arraybridge.types import ( - MemoryType as MemoryType, -) +from arraybridge.types import MemoryType as MemoryType class Backend(Enum): diff --git a/src/polystore/streaming/handlers/__init__.py b/src/polystore/streaming/handlers/__init__.py deleted file mode 100644 index 0403427..0000000 --- a/src/polystore/streaming/handlers/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Handler registry with metaclass-driven discovery. - -Uses AutoRegisterMeta for automatic registration of item handlers -with lazy loading and caching support. -""" - -from typing import Type, Dict -from metaclass_registry.core import AutoRegisterMeta, RegistryConfig -from metaclass_registry.lazy import LazyDiscoveryDict -from polystore.streaming.base import ItemHandler - -# Handler registry dict (will be populated by AutoRegisterMeta) -_ITEM_HANDLERS: Dict[str, Type[ItemHandler]] = {} - -# Configure metaclass for handler discovery -ITEM_HANDLER_REGISTRY_CONFIG = RegistryConfig( - registry_dict=_ITEM_HANDLERS, - key_attribute='_handler_data_type', - key_extractor=None, - skip_if_no_key=True, - discovery_package='polystore.streaming.handlers', - discovery_recursive=True, - log_registration=True, - registry_name='item handler' -) - -# Create lazy registry for caching -_ITEM_HANDLERS_LAZY = LazyDiscoveryDict(enable_cache=True) -# Point to the lazy dict (auto-populated by metaclass) -_ITEM_HANDLERS.update(_ITEM_HANDLERS_LAZY) - - -class HandlerBase(ItemHandler, metaclass=AutoRegisterMeta): - """ - Base class for all item handlers. - - Handlers auto-register when defined with metaclass. - Just set _handler_data_type attribute. - - Example: - class FijiImageHandler(HandlerBase): - _handler_data_type = "image" - # Auto-registered! - """ - _handler_data_type: str - - # Optional: override for custom key extraction - @classmethod - def _extract_handler_key(cls, handler_class: Type) -> str | None: - return getattr(handler_class, '_handler_data_type', None) diff --git a/src/polystore/streaming/handlers/fiji_images.py b/src/polystore/streaming/handlers/fiji_images.py deleted file mode 100644 index 5354f5e..0000000 --- a/src/polystore/streaming/handlers/fiji_images.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -Fiji-specific handler for image data. - -Handles ImageJ hyperstack building from accumulated image batches. -""" - -import logging -from polystore.streaming.handlers import HandlerBase -from polystore.streaming.base import HandlerContext - -logger = logging.getLogger(__name__) - - -class ImageData: - """Concrete type for image items with validation.""" - - def __init__(self, items: list): - """Validate all items have required fields.""" - for item in items: - assert 'data' in item, "Image items must have 'data' field" - assert 'metadata' in item, "Image items must have 'metadata' field" - self._items = items - - @property - def data(self) -> list: - """Get the validated image items.""" - return self._items - - -class FijiImageHandler(HandlerBase): - """Handler for ImageJ hyperstack display.""" - - _handler_data_type = "image" - - @staticmethod - def can_handle(data_type: str) -> bool: - """Check if this handler can process the given data type.""" - return data_type == "image" - - @staticmethod - def handle(context: HandlerContext) -> None: - """Build hyperstack from accumulated images.""" - images = context.data.items - channel_comps = context.components.get_by_mode("channel") - slice_comps = context.components.get_by_mode("slice") - frame_comps = context.components.get_by_mode("frame") - - # Collect values for each component group - channel_values = context.components.collect_values(channel_comps) - slice_values = context.components.collect_values(slice_comps) - frame_values = context.components.collect_values(frame_comps) - - logger.info( - f"🔬 FIJI IMAGE HANDLER: Processing {len(images)} images: " - f"{len(channel_values)}C x {len(slice_values)}Z x {len(frame_values)}T" - ) - context.server._build_single_hyperstack( - window_key=context.window_key, - images=images, - display_config_dict=context.display_config, - channel_components=channel_comps, - slice_components=slice_comps, - frame_components=frame_comps, - channel_values=channel_values, - slice_values=slice_values, - frame_values=frame_values, - component_names_metadata=context.display_config.get( - "component_names_metadata", {} - ), - ) diff --git a/src/polystore/streaming/handlers/fiji_rois.py b/src/polystore/streaming/handlers/fiji_rois.py deleted file mode 100644 index b17fa73..0000000 --- a/src/polystore/streaming/handlers/fiji_rois.py +++ /dev/null @@ -1,206 +0,0 @@ -""" -Fiji-specific handler for ROI data. - -Handles ImageJ ROI Manager integration with proper component positioning. -""" - -import logging -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field - -from polystore.streaming.handlers import HandlerBase -from polystore.streaming.base import HandlerContext -from zmqruntime.messages import ImageTransferIdentity -from zmqruntime.viewer_protocol import ( - ViewerBatchItemWireField, - ViewerWireMapping, - ViewerWireValue, -) - -logger = logging.getLogger(__name__) -FijiROIComponentValue = ( - str - | int - | float - | bool - | None - | tuple["FijiROIComponentValue", ...] -) -FijiROIComponentKey = tuple[FijiROIComponentValue, ...] - - -def fiji_roi_component_value(value: ViewerWireValue) -> FijiROIComponentValue: - """Return a hashable ImageJ-axis component value from viewer wire data.""" - if isinstance(value, (str, int, float, bool)) or value is None: - return value - if isinstance(value, tuple): - return tuple(fiji_roi_component_value(item) for item in value) - if isinstance(value, list): - return tuple(fiji_roi_component_value(item) for item in value) - raise TypeError( - "Fiji ROI component metadata values must be scalar or tuple-like, " - f"got {type(value).__name__}." - ) - - -@dataclass(frozen=True, slots=True) -class FijiROIWireItem: - """Typed ROI wire item with its original producer ACK contract.""" - - payload: ViewerWireMapping - transfer: ImageTransferIdentity | None = field(kw_only=True) - - @classmethod - def from_payload(cls, payload: Mapping[str, ViewerWireValue]) -> "FijiROIWireItem": - return cls(payload, transfer=ImageTransferIdentity.from_item(payload)) - - @property - def rois(self) -> list[str]: - field = ViewerBatchItemWireField.ROIS.value - if field not in self.payload: - raise ValueError("Fiji ROI item missing required 'rois' field.") - value = self.payload[field] - if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): - raise TypeError("Fiji ROI item 'rois' field must be a sequence.") - return [str(encoded_roi) for encoded_roi in value] - - @property - def metadata(self) -> ViewerWireMapping: - field = ViewerBatchItemWireField.METADATA.value - if field not in self.payload: - raise ValueError("Fiji ROI item missing required 'metadata' field.") - value = self.payload[field] - if not isinstance(value, Mapping): - raise TypeError("Fiji ROI item 'metadata' field must be a mapping.") - return value - - def component_value_tuple( - self, - component_names: Sequence[str], - ) -> FijiROIComponentKey: - return tuple( - fiji_roi_component_value(self.metadata[component_name]) - for component_name in component_names - ) - - -@dataclass(frozen=True, slots=True) -class FijiROIAxisPosition: - """Strict one-based ImageJ coordinate resolver for legacy Fiji ROI handling.""" - - component_names: Sequence[str] - values: Sequence[FijiROIComponentKey] - - @classmethod - def from_items( - cls, - items: Sequence[FijiROIWireItem], - component_names: Sequence[str], - ) -> "FijiROIAxisPosition": - values = tuple( - sorted( - { - item.component_value_tuple(component_names) - for item in items - } - ) - ) - return cls(component_names, values) - - def one_based_position(self, item: FijiROIWireItem) -> int: - if not self.component_names: - return 1 - value_tuple = item.component_value_tuple(self.component_names) - if value_tuple not in self.values: - raise ValueError( - f"Fiji ROI component value {value_tuple!r} is outside axis domain " - f"{self.values!r}." - ) - return self.values.index(value_tuple) + 1 - - -class FijiROIHandler(HandlerBase): - """Handler for ImageJ ROI Manager display.""" - - _handler_data_type = "rois" - - @staticmethod - def can_handle(data_type: str) -> bool: - """Check if this handler can process the given data type.""" - return data_type == "rois" - - @staticmethod - def handle(context: HandlerContext) -> None: - """Add ROIs to ImageJ ROI Manager.""" - # Get or create RoiManager on EDT - import scyjava as sj - RoiManager = sj.jimport("ij.plugin.frame.RoiManager") - rm = RoiManager.getInstance() - - if rm is None: - from jpype import JImplements, JOverride - - @JImplements("java.lang.Runnable") - class CreateRoiManagerRunnable: - @JOverride - def run(self): - holder[0] = RoiManager() - - from javax.swing import SwingUtilities - holder = [None] - SwingUtilities.invokeAndWait(CreateRoiManagerRunnable()) - rm = holder[0] - - # Get or assign integer group ID for this window - group_id = context.server._get_or_create_group_id(context.window_key) - - roi_items = tuple( - FijiROIWireItem.from_payload(item) - for item in context.data.items - ) - - # Process ROIs with component positioning - channel_comps = context.components.get_by_mode("channel") - slice_comps = context.components.get_by_mode("slice") - frame_comps = context.components.get_by_mode("frame") - - channel_position = FijiROIAxisPosition.from_items(roi_items, channel_comps) - slice_position = FijiROIAxisPosition.from_items(roi_items, slice_comps) - frame_position = FijiROIAxisPosition.from_items(roi_items, frame_comps) - - total_rois_added = 0 - - from polystore.roi_converters import FijiROIConverter - - for roi_item in roi_items: - rois_encoded = roi_item.rois - if not rois_encoded: - context.server.send_ack(roi_item.transfer, status="success") - continue - - logger.info(f"🔬 FIJI ROI HANDLER: Processing {len(rois_encoded)} ROIs") - - # Convert ROIs to ImageJ format - java_rois = FijiROIConverter.transmission_to_java_rois( - rois_encoded, - sj, - ) - imagej_position = ( - channel_position.one_based_position(roi_item), - slice_position.one_based_position(roi_item), - frame_position.one_based_position(roi_item), - ) - - # Add ROIs to manager with group ID - for roi_obj in java_rois: - roi_obj.setPosition(*imagej_position) - roi_obj.setGroup(group_id) - rm.addRoi(roi_obj) - - total_rois_added += len(java_rois) - - context.server.send_ack(roi_item.transfer, status="success") - - logger.info( - f"🔬 FIJI ROI HANDLER: Added {total_rois_added} ROIs to window '{context.window_key}'" - ) diff --git a/tests/test_array_payload.py b/tests/test_array_payload.py index 67a5a27..7921232 100644 --- a/tests/test_array_payload.py +++ b/tests/test_array_payload.py @@ -63,29 +63,31 @@ def test_storage_numpy_array_preserves_numpy_identity() -> None: assert array_payload.storage_numpy_array(data) is data -def test_storage_numpy_array_delegates_framework_semantics_to_arraybridge( - monkeypatch, -) -> None: +def test_storage_numpy_array_asks_the_library_that_made_the_array(monkeypatch) -> None: data = SimpleNamespace() - converted = np.arange(4, dtype=np.float32) + downloaded = np.arange(4, dtype=np.float32) calls = [] - monkeypatch.setattr(array_payload, "detect_memory_type", lambda value: "tensor") + + class _Library: + def to_numpy(self, value): + calls.append(value) + return downloaded + + library = _Library() + owners = [] monkeypatch.setattr( - array_payload, - "convert_memory", - lambda value, **kwargs: calls.append((value, kwargs)) or converted, + array_payload.ArrayLibrary, + "of", + classmethod(lambda cls, value: owners.append(value) or library), ) result = array_payload.storage_numpy_array(data) - assert result is converted - assert calls == [ - ( - data, - { - "source_type": "tensor", - "target_type": array_payload.MemoryType.NUMPY, - "gpu_id": 0, - }, - ) - ] + assert result is downloaded + assert owners == [data] + assert calls == [data] + + +def test_storage_numpy_array_rejects_an_object_no_library_made() -> None: + with pytest.raises(ValueError, match="Unknown memory type"): + array_payload.storage_numpy_array(SimpleNamespace()) diff --git a/tests/test_every_module_imports.py b/tests/test_every_module_imports.py new file mode 100644 index 0000000..5f9e6ee --- /dev/null +++ b/tests/test_every_module_imports.py @@ -0,0 +1,46 @@ +"""Every PolyStore module imports against the installed first-party libraries. + +The imports run in a child interpreter: importing every module in the test +process would leave process-wide state (ImageJ environment, registries) behind +for the tests that follow. +""" + +import json +import subprocess +import sys + +FIRST_PARTY = ("polystore", "arraybridge", "metaclass_registry", "zmqruntime") + +IMPORT_EVERY_MODULE = """ +import importlib, json, pkgutil, traceback +import polystore + +report = {"imported": [], "optional_missing": {}, "failed": {}} +for module in pkgutil.walk_packages(polystore.__path__, prefix="polystore."): + try: + importlib.import_module(module.name) + except ModuleNotFoundError as error: + missing = (error.name or "").split(".")[0] + kind = "failed" if missing in FIRST_PARTY else "optional_missing" + report[kind][module.name] = traceback.format_exc() if kind == "failed" else missing + except Exception: + report["failed"][module.name] = traceback.format_exc() + else: + report["imported"].append(module.name) +print(json.dumps(report)) +""" + + +def test_every_module_imports() -> None: + completed = subprocess.run( + [sys.executable, "-c", f"FIRST_PARTY = {FIRST_PARTY!r}\n{IMPORT_EVERY_MODULE}"], + capture_output=True, + text=True, + check=True, + ) + report = json.loads(completed.stdout.splitlines()[-1]) + + # A missing optional third-party package is not a PolyStore fault; a + # missing first-party module or name, or any other error, always is. + assert report["failed"] == {} + assert {"polystore.constants", "polystore.array_payload"} <= set(report["imported"]) diff --git a/uv.lock b/uv.lock index 09af66c..43bd04b 100644 --- a/uv.lock +++ b/uv.lock @@ -253,15 +253,16 @@ wheels = [ [[package]] name = "arraybridge" -version = "0.3.1" +version = "0.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "metaclass-registry" }, { name = "numpy" }, + { name = "python-introspect" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ff/da/1a984e7d6bfb8508e222a5dd3a8125df7faffbbbab2c453e99f7d4d322f6/arraybridge-0.3.1.tar.gz", hash = "sha256:93af611ce4b8bd21ad0ed4f93f7e6c1be104c52e23fbd952324b150c28dc83c1", size = 95672, upload-time = "2026-08-22T03:49:32.297Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dd/e0/d9bf8fff91238becf13a6abb0cd0b48e8cd71ed6c945e1b1285dd13e53c9/arraybridge-0.5.0.tar.gz", hash = "sha256:43ae14f682884bd1f6a3e76f5b8967f0a3da0b630a68861a466d2474992169f6", size = 114594, upload-time = "2026-10-12T19:21:08.16Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/69/c88c43f32102bea78139745072408eff814ebd2c0b59023ce2d4643e1dfa/arraybridge-0.3.1-py3-none-any.whl", hash = "sha256:0589ec5501db38471cb759d9027754fde751211ab9ef5983f89d1263795cee22", size = 33604, upload-time = "2026-08-22T03:49:30.815Z" }, + { url = "https://files.pythonhosted.org/packages/2c/0a/85073cf0b4498108eb6f622856005841b512d457ccf1cd02d7b0fc2e9344/arraybridge-0.5.0-py3-none-any.whl", hash = "sha256:86dc05b609c94002cebaf40f632eab6d3872a848ce23272b828a348f312ea341", size = 38590, upload-time = "2026-10-12T19:21:07.019Z" }, ] [[package]] @@ -2905,7 +2906,7 @@ wheels = [ [[package]] name = "polystore" -version = "0.6.2" +version = "0.6.3" source = { editable = "." } dependencies = [ { name = "arraybridge" }, @@ -2994,7 +2995,7 @@ torch = [ [package.metadata] requires-dist = [ - { name = "arraybridge", specifier = ">=0.3.0" }, + { name = "arraybridge", specifier = ">=0.5.0,<0.6" }, { name = "black", marker = "extra == 'dev'", specifier = ">=23.0.0" }, { name = "coverage", marker = "extra == 'dev'", specifier = ">=7.3.2" }, { name = "cucim-cu12", marker = "extra == 'gpu'", specifier = ">=25.6.0,<26.0.0" },