diff --git a/docs/api.rst b/docs/api.rst index 6ceef1b..3f4edfa 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -4,9 +4,9 @@ API orientation ``Assignment`` and ``CodeBlock`` Structured top-level source items. -``generate_python_source(obj, header="", clean_mode=False, *, context=None)`` - Generate complete executable source and required imports. A caller-supplied - ``FormatContext`` is retained across both render passes and owns clean mode. +``generate_python_source(obj, header="", clean_mode=False)`` + Generate complete executable source and required imports, with shared and + repeated absolute paths bound once at the top. ``to_source(value, context=None)`` Format one value into a ``SourceFragment``. @@ -19,8 +19,17 @@ API orientation Resolve import requirements and aliases for colliding names. ``FormatContext`` and ``SourceFragment`` - Immutable typed formatting state and output records. ``FormatContext`` may - carry exact-type ``extensions`` and exposes them through ``extension(type)``. + Immutable formatting state and output records. + +``imported_name(module, qualname, context)`` + Spell an importable object by its qualified name, honouring collision aliases. + +``PythonSourceLiteral`` + Base for values that spell their own source through ``source_fragment``. + +``PathRendering``, ``PathCollector``, ``PathFactoring``, ``PathConstructor`` + How a render pass spells paths: plainly, while recording, or through the + document's path bindings. The canonical import surface is ``pycodify.__all__``. diff --git a/docs/architecture.rst b/docs/architecture.rst index 6296280..3aaf187 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -6,7 +6,8 @@ Two-Pass Algorithm Generating executable source requires knowing import aliases before emitting code. But aliases depend on detecting collisions, which requires visiting all types first. This creates a dependency: code generation requires alias resolution, but alias resolution requires traversing the object graph. -pycodify solves this with two passes: +pycodify solves this with two passes, preceded by a path-discovery pass +(see *Path factoring* below): 1. **Collection pass**: Traverse the object, emit code fragments, and collect ``(module, name)`` import pairs. @@ -31,12 +32,26 @@ Each type maps to a ``SourceFormatter`` that emits a ``SourceFragment(code, impo return isinstance(value, Enum) def format(self, value: Enum, context: FormatContext) -> SourceFragment: - cls = value.__class__ - import_pair = (cls.__module__, cls.__name__) - name = context.name_mappings.get(import_pair, cls.__name__) - return SourceFragment(f"{name}.{value.name}", frozenset([import_pair])) + enum_type = imported_name(type(value).__module__, type(value).__qualname__, context) + return SourceFragment(f"{enum_type.code}.{value.name}", enum_type.imports) -Priority-based dispatch selects the most specific formatter. Domain extensions add formatters without modifying core code. +Priority-based dispatch selects the most specific formatter. Domain extensions add formatters without modifying core code. A value no formatter accepts raises ``TypeError``: pycodify never emits ``repr`` text it cannot load back. + +Qualified names +--------------- + +``imported_name(module, qualname, context)`` is the one rule for spelling an +importable object. It imports the root of ``__qualname__`` and spells the rest as +attribute access, so ``Outer.Mode.FAST`` imports ``Outer``. Enum members, +classes, functions, class-bound methods, named tuples and dataclass constructors +all use it. Objects defined inside a function raise ``TypeError``. + +Values that own their spelling +------------------------------ + +A value whose source is not its constructor subclasses ``PythonSourceLiteral`` +and implements ``source_fragment(context)``, typically building on +``imported_name`` so collision aliases apply. Core Components --------------- @@ -49,7 +64,7 @@ Core Components **FormatContext** Carries indentation depth, ``clean_mode``, the ``(module, name)`` to - local-name mapping, and caller-owned typed extensions. ``indented()`` returns + local-name mapping, and the pass's ``PathRendering``. ``indented()`` returns a copied context and does not mutate its caller. **resolve_imports** @@ -60,27 +75,17 @@ Core Components Orchestrates collection, resolution, and regeneration to produce a complete source string. -Immutable typed context extensions ----------------------------------- - -``FormatContext`` is frozen. Its default mappings are read-only mapping proxies, -and context evolution uses dataclass replacement rather than mutation. A host -may supply an ``extensions`` mapping keyed by the exact nominal type of each -rendering policy or state record. A formatter retrieves one value with -``context.extension(ExtensionType)``; pycodify does not search base classes, -string keys, or a fallback chain. - -``generate_python_source(..., context=context)`` threads the caller's context -through collection and regeneration. Import resolution replaces only -``name_mappings`` for the second pass, so typed extensions and all other context -fields survive. When a context is supplied, its ``clean_mode`` is authoritative; -the separate ``clean_mode=`` argument is used only when pycodify constructs the -context. +Path factoring +-------------- -Extensions provide formatting context, not formatter dispatch. New value -families still extend the ``SourceFormatter`` registry, while the exact extension -type allows that formatter to obtain host-owned rendering information without -adding host imports or fields to pycodify core. +``generate_python_source`` first renders the document with a ``PathCollector`` +on ``FormatContext.paths`` to record every ``pathlib.Path`` value. A +``PathFactoring`` plan then binds each directory shared by two or more absolute +paths (``path_root = Path('/data/screen')``) and each repeated absolute path +(``path_1 = path_root / 'plate_A'``) once, at the top of the document, and the +two rendering passes spell those paths through the bindings. Relative paths and +strings are never factored. ``to_source`` alone uses the base ``PathRendering``, +which spells each path as ``Path('...')``. Clean Mode ---------- @@ -107,25 +112,13 @@ not sandboxing or validation of generated programs. Hosts must restrict inputs and custom formatters to trusted sources, review persisted code before running it, and define their own compatibility policy for imported domain APIs. -Lazy Dataclass Integration ---------------------------- - -For dataclasses with ``__getattribute__`` interception (used for hierarchical config inheritance), formatters use ``object.__getattribute__`` to access raw field values without triggering lazy resolution: - -.. code-block:: python - - # In DataclassFormatter - if hasattr(instance, "_resolve_field_value"): - # Bypass __getattribute__ to get raw None vs concrete value - current_value = object.__getattribute__(instance, field_name) - else: - current_value = getattr(instance, field_name) - -This distinguishes explicitly-set values from inherited ones during serialization. - Module Structure ---------------- - **core.py**: Main API (``Assignment``, ``generate_python_source``) -- **formatters.py**: Built-in formatters (dataclass, enum, primitive types) +- **formatters.py**: Built-in formatters (dataclass, enum, class, callable, primitive types) +- **paths.py**: Document-level path factoring + +Lazy configuration dataclasses (ObjectState) are formatted by +``objectstate.codegen``, which registers its own formatter. - **imports.py**: Import resolution and collision handling diff --git a/docs/usage.rst b/docs/usage.rst index c811b72..52cf3a9 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -117,62 +117,37 @@ level. ``name_mappings`` maps ``(module, imported_name)`` pairs to the local names chosen by ``resolve_imports``; callers normally let ``generate_python_source`` populate it during the second pass. -Typed rendering extensions --------------------------- +Values that own their source +---------------------------- -A host can attach immutable rendering context without adding host-specific state -to pycodify. Keys are exact nominal types, and the same context reaches both -render passes: +A value whose source is not its dataclass constructor implements +``PythonSourceLiteral``: .. code-block:: python from dataclasses import dataclass - from types import MappingProxyType from pycodify import ( Assignment, FormatContext, - SourceFormatter, + PythonSourceLiteral, SourceFragment, generate_python_source, + imported_name, ) @dataclass(frozen=True) - class RenderPolicy: - marker: str - - class PolicyValue: - pass - - class PolicyValueFormatter(SourceFormatter): - priority = 200 - - def can_format(self, value): - return isinstance(value, PolicyValue) - - def format(self, value, context): - del value - policy = context.extension(RenderPolicy) - if policy is None: - raise RuntimeError("RenderPolicy is required") - return SourceFragment(repr(policy.marker)) - - context = FormatContext( - clean_mode=True, - extensions=MappingProxyType( - {RenderPolicy: RenderPolicy(marker="preserved")} - ), - ) - source = generate_python_source( - Assignment("value", PolicyValue()), - context=context, - ) - assert source == "value = 'preserved'" + class RegistryLookup(PythonSourceLiteral): + key: str + + def source_fragment(self, context: FormatContext) -> SourceFragment: + lookup = imported_name("myapp.registry", "get_function", context) + return SourceFragment(f"{lookup.code}({self.key!r})", lookup.imports) -``extension(RenderPolicy)`` does not return a value stored under a base class or -string key. This exact-type lookup keeps extension ownership explicit. Treat the -extension mapping as immutable; use a new ``FormatContext`` when the host needs -different state. + source = generate_python_source(Assignment("func", RegistryLookup("gauss"))) + # from myapp.registry import get_function + # + # func = get_function('gauss') Nested Dataclasses ------------------- diff --git a/pyproject.toml b/pyproject.toml index c2c770d..ad0e2fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "pycodify" -version = "0.1.3" +version = "0.2.0" description = "Python source code as a serialization format with automatic import resolution" authors = [{name = "Tristan Simas", email = "tristan.simas@mail.mcgill.ca"}] license = {text = "MIT"} diff --git a/src/pycodify/__init__.py b/src/pycodify/__init__.py index 9ee9c92..b8f2947 100644 --- a/src/pycodify/__init__.py +++ b/src/pycodify/__init__.py @@ -1,20 +1,23 @@ +# Register default formatters +from . import formatters as _formatters # noqa: F401 from .core import ( Assignment, BlankLine, CodeBlock, Comment, FormatContext, + PathRendering, + PythonSourceLiteral, SourceFormatter, SourceFragment, generate_python_source, + imported_name, to_source, ) from .imports import resolve_imports +from .paths import PathCollector, PathConstructor, PathFactoring -# Register default formatters -from . import formatters as _formatters # noqa: F401 - -__version__ = "0.1.3" +__version__ = "0.2.0" __all__ = [ "Assignment", @@ -22,9 +25,15 @@ "CodeBlock", "Comment", "FormatContext", + "PathCollector", + "PathConstructor", + "PathFactoring", + "PathRendering", + "PythonSourceLiteral", "SourceFormatter", "SourceFragment", "generate_python_source", + "imported_name", "resolve_imports", "to_source", ] diff --git a/src/pycodify/core.py b/src/pycodify/core.py index 4e76873..285cabd 100644 --- a/src/pycodify/core.py +++ b/src/pycodify/core.py @@ -1,10 +1,11 @@ from __future__ import annotations from abc import ABC, abstractmethod +from collections.abc import Iterable, Mapping from dataclasses import dataclass, field, replace -from collections.abc import Mapping +from pathlib import Path from types import MappingProxyType -from typing import Any, ClassVar, FrozenSet, Iterable, List, Tuple +from typing import Any, ClassVar @dataclass(frozen=True) @@ -12,39 +13,78 @@ class SourceFragment: """Result of serializing a value to Python source.""" code: str - imports: FrozenSet[Tuple[str, str]] = frozenset() + imports: frozenset[tuple[str, str]] = frozenset() - def __add__(self, other: "SourceFragment") -> "SourceFragment": + def __add__(self, other: SourceFragment) -> SourceFragment: return SourceFragment( code=self.code + other.code, imports=self.imports | other.imports, ) +class PathRendering: + """How one render pass spells :class:`pathlib.Path` values. + + The base spells every path as its own constructor call. Subclasses in + :mod:`pycodify.paths` record or substitute paths for document-level + factoring. + """ + + def path_source(self, value: Path, context: FormatContext) -> SourceFragment: + constructor = imported_name("pathlib", "Path", context) + return SourceFragment( + f"{constructor.code}({str(value)!r})", + constructor.imports, + ) + + @dataclass(frozen=True) class FormatContext: """Context passed through serialization.""" indent: int = 0 clean_mode: bool = False - name_mappings: Mapping[Tuple[str, str], str] = field( - default_factory=lambda: MappingProxyType({}) - ) - extensions: Mapping[type[object], object] = field( + name_mappings: Mapping[tuple[str, str], str] = field( default_factory=lambda: MappingProxyType({}) ) + paths: PathRendering = field(default_factory=PathRendering) - def indented(self) -> "FormatContext": + def indented(self) -> FormatContext: return replace(self, indent=self.indent + 1) @property def indent_str(self) -> str: return " " * self.indent - def extension(self, extension_type: type[Any]) -> Any | None: - """Return one caller-owned rendering extension by exact nominal type.""" - return self.extensions.get(extension_type) +def imported_name(module: str, qualname: str, context: FormatContext) -> SourceFragment: + """Spell an importable object by its module and ``__qualname__``. + + The root of the qualified name is imported (under its collision alias, if + any) and the rest is spelled as attribute access, so nested classes, enums + and functions load back as themselves. Objects defined inside a function + have no import path and are rejected. + """ + + if "" in qualname: + raise TypeError( + f"{module}.{qualname} is defined inside a function and cannot be " + "imported by generated source." + ) + root, dot, nested = qualname.partition(".") + if module == "builtins": + return SourceFragment(qualname) + import_pair = (module, root) + name = context.name_mappings.get(import_pair, root) + return SourceFragment(f"{name}{dot}{nested}", frozenset({import_pair})) + + +class PythonSourceLiteral(ABC): + """A value that spells its own Python source expression.""" + + @abstractmethod + def source_fragment(self, context: FormatContext) -> SourceFragment: + """Return the expression that reconstructs this value, with its imports.""" @dataclass(frozen=True) @@ -65,10 +105,10 @@ class Assignment: @dataclass(frozen=True) class CodeBlock: - items: Tuple[Any, ...] + items: tuple[Any, ...] @classmethod - def from_items(cls, items: Iterable[Any]) -> "CodeBlock": + def from_items(cls, items: Iterable[Any]) -> CodeBlock: return cls(tuple(items)) @@ -77,11 +117,11 @@ class SourceFormatter(ABC): priority: int = 0 - _registry: ClassVar[List["SourceFormatter"]] = [] - _sorted_registry: ClassVar[List["SourceFormatter"]] = [] + _registry: ClassVar[list[SourceFormatter]] = [] + _sorted_registry: ClassVar[list[SourceFormatter]] = [] _registry_dirty: ClassVar[bool] = False - def __init_subclass__(cls, **kwargs): + def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) if getattr(cls, "__abstract__", False): return @@ -99,12 +139,12 @@ def format(self, value: Any, context: FormatContext) -> SourceFragment: raise NotImplementedError @classmethod - def register(cls, formatter: "SourceFormatter") -> None: + def register(cls, formatter: SourceFormatter) -> None: cls._registry.append(formatter) cls._registry_dirty = True @classmethod - def _iter_registry(cls) -> List["SourceFormatter"]: + def _iter_registry(cls) -> list[SourceFormatter]: if cls._registry_dirty: cls._sorted_registry = sorted( cls._registry, @@ -115,7 +155,7 @@ def _iter_registry(cls) -> List["SourceFormatter"]: return cls._sorted_registry @classmethod - def get_formatter(cls, value: Any) -> "SourceFormatter": + def get_formatter(cls, value: Any) -> SourceFormatter: for formatter in cls._iter_registry(): if formatter.can_format(value): return formatter @@ -135,21 +175,28 @@ def generate_python_source( obj: Any, header: str = "", clean_mode: bool = False, - *, - context: FormatContext | None = None, ) -> str: - """Generate complete Python source with imports.""" + """Generate complete Python source with imports. + + Absolute paths that share a root or repeat are bound once at the top of the + document (see :mod:`pycodify.paths`). + """ from .imports import resolve_imports + from .paths import PathCollector, PathFactoring - ctx = context or FormatContext(clean_mode=clean_mode) - fragment = to_source(obj, ctx) + ctx = FormatContext(clean_mode=clean_mode) + collector = PathCollector() + to_source(obj, replace(ctx, paths=collector)) + factoring = PathFactoring.from_occurrences(collector.values) + body = factoring.document(obj) + ctx = replace(ctx, paths=factoring) + fragment = to_source(body, ctx) import_lines, name_mappings = resolve_imports(fragment.imports) - ctx = replace(ctx, name_mappings=MappingProxyType(name_mappings)) - fragment = to_source(obj, ctx) + fragment = to_source(body, ctx) - code_lines: List[str] = [] + code_lines: list[str] = [] if header: code_lines.extend(header.splitlines()) code_lines.append("") diff --git a/src/pycodify/formatters.py b/src/pycodify/formatters.py index b58290a..18e0895 100644 --- a/src/pycodify/formatters.py +++ b/src/pycodify/formatters.py @@ -1,11 +1,12 @@ from __future__ import annotations import dataclasses -import typing +import inspect +import math from dataclasses import is_dataclass from enum import Enum from pathlib import Path -from typing import Any, Tuple +from typing import Any from .core import ( Assignment, @@ -13,8 +14,10 @@ CodeBlock, Comment, FormatContext, + PythonSourceLiteral, SourceFormatter, SourceFragment, + imported_name, to_source, ) @@ -66,7 +69,7 @@ def can_format(self, value: Any) -> bool: def format(self, value: CodeBlock, context: FormatContext) -> SourceFragment: codes = [] - imports = set() + imports: set[tuple[str, str]] = set() for item in value.items: fragment = to_source(item, context) imports |= fragment.imports @@ -74,6 +77,16 @@ def format(self, value: CodeBlock, context: FormatContext) -> SourceFragment: return SourceFragment("\n".join(codes), frozenset(imports)) +class PythonSourceLiteralFormatter(SourceFormatter): + priority = 120 + + def can_format(self, value: Any) -> bool: + return isinstance(value, PythonSourceLiteral) + + def format(self, value: PythonSourceLiteral, context: FormatContext) -> SourceFragment: + return value.source_fragment(context) + + class EnumFormatter(SourceFormatter): priority = 70 @@ -81,10 +94,24 @@ def can_format(self, value: Any) -> bool: return isinstance(value, Enum) def format(self, value: Enum, context: FormatContext) -> SourceFragment: - cls = value.__class__ - import_pair = (cls.__module__, cls.__name__) - name = context.name_mappings.get(import_pair, cls.__name__) - return SourceFragment(f"{name}.{value.name}", frozenset([import_pair])) + enum_type = imported_name(type(value).__module__, type(value).__qualname__, context) + if type(value).__members__.get(value.name) is value: + return SourceFragment(f"{enum_type.code}.{value.name}", enum_type.imports) + member_value = to_source(value.value, context) + return SourceFragment( + f"{enum_type.code}({member_value.code})", + enum_type.imports | member_value.imports, + ) + + +class TypeFormatter(SourceFormatter): + priority = 70 + + def can_format(self, value: Any) -> bool: + return isinstance(value, type) + + def format(self, value: type, context: FormatContext) -> SourceFragment: + return imported_name(value.__module__, value.__qualname__, context) class PathFormatter(SourceFormatter): @@ -94,9 +121,7 @@ def can_format(self, value: Any) -> bool: return isinstance(value, Path) def format(self, value: Path, context: FormatContext) -> SourceFragment: - import_pair = ("pathlib", "Path") - name = context.name_mappings.get(import_pair, "Path") - return SourceFragment(f"{name}({repr(str(value))})", frozenset([import_pair])) + return context.paths.path_source(value, context) class StrFormatter(SourceFormatter): @@ -106,29 +131,64 @@ def can_format(self, value: Any) -> bool: return isinstance(value, str) def format(self, value: str, context: FormatContext) -> SourceFragment: - return SourceFragment(repr(value), frozenset()) + return SourceFragment(repr(str(value)), frozenset()) class LiteralFormatter(SourceFormatter): + """``None`` and builtin numbers, spelled by their builtin type's ``repr``. + + Subclass instances (for example NumPy scalars) are spelled as the builtin + value they equal; non-finite floats as ``float('nan')``/``float('inf')``. + """ + priority = 50 + builtin_types = (bool, int, float, complex, bytes) def can_format(self, value: Any) -> bool: - return value is None or isinstance(value, (bool, int, float)) + return value is None or isinstance(value, self.builtin_types) def format(self, value: Any, context: FormatContext) -> SourceFragment: - return SourceFragment(repr(value), frozenset()) + if value is None: + return SourceFragment("None", frozenset()) + builtin_type = next(t for t in self.builtin_types if isinstance(value, t)) + builtin_value: Any = builtin_type(value) + if builtin_type is float and not math.isfinite(builtin_value): + return SourceFragment(f"float({str(builtin_value)!r})", frozenset()) + return SourceFragment(repr(builtin_value), frozenset()) class CallableFormatter(SourceFormatter): + """Module-level functions, builtins and class-bound methods, by import.""" + priority = 40 def can_format(self, value: Any) -> bool: - return callable(value) and not isinstance(value, type) + if inspect.isfunction(value): + return True + if inspect.isbuiltin(value): + return inspect.ismodule(value.__self__) or value.__self__ is None + return inspect.ismethod(value) and isinstance(value.__self__, type) + + def format(self, value: Any, context: FormatContext) -> SourceFragment: + return imported_name(value.__module__, value.__qualname__, context) + + +class NamedTupleFormatter(SourceFormatter): + priority = 31 + + def can_format(self, value: Any) -> bool: + return isinstance(value, tuple) and hasattr(type(value), "_fields") def format(self, value: Any, context: FormatContext) -> SourceFragment: - import_pair = (value.__module__, value.__name__) - name = context.name_mappings.get(import_pair, value.__name__) - return SourceFragment(name, frozenset([import_pair])) + tuple_type = imported_name(type(value).__module__, type(value).__qualname__, context) + field_ctx = context.indented() + fragments = [(name, to_source(item, field_ctx)) for name, item in zip(value._fields, value)] + imports = tuple_type.imports.union(*(frag.imports for _, frag in fragments)) + if not fragments: + return SourceFragment(f"{tuple_type.code}()", imports) + inner = f",\n{field_ctx.indent_str}".join(f"{name}={frag.code}" for name, frag in fragments) + code = f"{tuple_type.code}(\n{field_ctx.indent_str}{inner}\n{context.indent_str})" + return SourceFragment(code, imports) class ListFormatter(SourceFormatter): @@ -181,7 +241,7 @@ def format(self, value: dict, context: FormatContext) -> SourceFragment: item_ctx = context.indented() entries = [] - imports = set() + imports: set[tuple[str, str]] = set() for key, val in value.items(): key_frag = to_source(key, item_ctx) val_frag = to_source(val, item_ctx) @@ -195,20 +255,28 @@ def format(self, value: dict, context: FormatContext) -> SourceFragment: class SetFormatter(SourceFormatter): + """Sets and frozensets; members are sorted by their source for stable output.""" + priority = 30 def can_format(self, value: Any) -> bool: - return isinstance(value, set) + return isinstance(value, (set, frozenset)) - def format(self, value: set, context: FormatContext) -> SourceFragment: + def format(self, value: set | frozenset, context: FormatContext) -> SourceFragment: + constructor = "frozenset" if isinstance(value, frozenset) else "set" if not value: - return SourceFragment("set()", frozenset()) + return SourceFragment(f"{constructor}()", frozenset()) item_ctx = context.indented() - item_frags = [to_source(item, item_ctx) for item in value] + item_frags = sorted( + (to_source(item, item_ctx) for item in value), + key=lambda frag: frag.code, + ) imports = frozenset().union(*(frag.imports for frag in item_frags)) inner = f",\n{item_ctx.indent_str}".join(frag.code for frag in item_frags) code = f"{{\n{item_ctx.indent_str}{inner}\n{context.indent_str}}}" + if constructor == "frozenset": + code = f"frozenset({code})" return SourceFragment(code, imports) @@ -223,49 +291,10 @@ def format(self, value: Any, context: FormatContext) -> SourceFragment: return SourceFragment(code, frozenset(imports)) -def _resolve_dataclass_class_from_type(field_type: Any) -> Any | None: - if field_type is None: - return None - - origin = typing.get_origin(field_type) - if origin is typing.Union: - for arg in typing.get_args(field_type): - if arg is type(None): - continue - resolved = _resolve_dataclass_class_from_type(arg) - if resolved: - return resolved - return None - - if isinstance(field_type, type) and dataclasses.is_dataclass(field_type): - return field_type - - return None - - -def _create_placeholder_dataclass_instance(dataclass_cls: type) -> Any: - instance = object.__new__(dataclass_cls) - for dc_field in dataclasses.fields(dataclass_cls): - object.__setattr__(instance, dc_field.name, None) - - if hasattr(dataclass_cls, "_is_lazy_dataclass"): - object.__setattr__(instance, "_is_lazy_dataclass", True) - - return instance - - def _create_default_instance(instance: Any) -> Any: - if hasattr(instance, "_resolve_field_value"): - default_instance = object.__new__(instance.__class__) - for dc_field in dataclasses.fields(instance): - object.__setattr__(default_instance, dc_field.name, None) - if hasattr(instance.__class__, "_is_lazy_dataclass"): - object.__setattr__(default_instance, "_is_lazy_dataclass", True) - return default_instance - try: return instance.__class__() - except Exception: + except (TypeError, ValueError): default_instance = object.__new__(instance.__class__) for dc_field in dataclasses.fields(instance): if dc_field.default is not dataclasses.MISSING: @@ -273,7 +302,7 @@ def _create_default_instance(instance: Any) -> Any: elif dc_field.default_factory is not dataclasses.MISSING: # type: ignore value = dc_field.default_factory() # type: ignore else: - value = None + value = dataclasses.MISSING object.__setattr__(default_instance, dc_field.name, value) return default_instance @@ -281,37 +310,22 @@ def _create_default_instance(instance: Any) -> Any: def _format_dataclass_expression( instance: Any, context: FormatContext, -) -> Tuple[str, set[Tuple[str, str]], bool]: +) -> tuple[str, set[tuple[str, str]], bool]: cls = instance.__class__ - import_pair = (cls.__module__, cls.__name__) - class_name = context.name_mappings.get(import_pair, cls.__name__) - imports: set[Tuple[str, str]] = {import_pair} + class_ref = imported_name(cls.__module__, cls.__qualname__, context) + imports: set[tuple[str, str]] = set(class_ref.imports) default_instance = _create_default_instance(instance) - try: - type_hints = typing.get_type_hints(instance.__class__) - except Exception: - type_hints = {} - lines: list[str] = [] field_ctx = context.indented() for dc_field in dataclasses.fields(instance): + if not dc_field.init: + continue field_name = dc_field.name - - if hasattr(instance, "_resolve_field_value"): - current_value = object.__getattribute__(instance, field_name) - default_value = object.__getattribute__(default_instance, field_name) - else: - current_value = getattr(instance, field_name) - default_value = getattr(default_instance, field_name) - - if (not context.clean_mode) and current_value is None: - field_type = type_hints.get(field_name, dc_field.type) - dataclass_cls = _resolve_dataclass_class_from_type(field_type) - if dataclass_cls is not None: - current_value = _create_placeholder_dataclass_instance(dataclass_cls) + current_value = getattr(instance, field_name) + default_value = getattr(default_instance, field_name) if context.clean_mode and current_value == default_value: continue @@ -342,18 +356,8 @@ def _format_dataclass_expression( lines.append(f"{field_name}={frag.code}") if not lines: - return f"{class_name}()", imports, False + return f"{class_ref.code}()", imports, False inner = f",\n{field_ctx.indent_str}".join(lines) - code = f"{class_name}(\n{field_ctx.indent_str}{inner}\n{context.indent_str})" + code = f"{class_ref.code}(\n{field_ctx.indent_str}{inner}\n{context.indent_str})" return code, imports, True - - -class FallbackFormatter(SourceFormatter): - priority = -100 - - def can_format(self, value: Any) -> bool: - return True - - def format(self, value: Any, context: FormatContext) -> SourceFragment: - return SourceFragment(repr(value), frozenset()) diff --git a/src/pycodify/imports.py b/src/pycodify/imports.py index e29d26c..21690bc 100644 --- a/src/pycodify/imports.py +++ b/src/pycodify/imports.py @@ -1,55 +1,47 @@ from __future__ import annotations from collections import defaultdict -from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple +from collections.abc import Iterable - -ImportSpec = Tuple[str, Optional[str]] +ImportSpec = tuple[str, str | None] def _alias_for_collision(name: str, module: str) -> str: return f"{name}_{module.replace('.', '_')}" -def _format_import_target(name: str, alias: Optional[str]) -> str: +def _format_import_target(name: str, alias: str | None) -> str: if alias is None: return name return f"{name} as {alias}" -def _format_module_import(module: str, targets: List[ImportSpec]) -> List[str]: +def _format_module_import(module: str, targets: list[ImportSpec]) -> list[str]: if len(targets) == 1: name, alias = targets[0] return [f"from {module} import {_format_import_target(name, alias)}"] lines = [f"from {module} import ("] - lines.extend( - f" {_format_import_target(name, alias)}," - for name, alias in targets - ) + lines.extend(f" {_format_import_target(name, alias)}," for name, alias in targets) lines.append(")") return lines def resolve_imports( - imports: FrozenSet[Tuple[str, str]] | Iterable[Tuple[str, str]], -) -> Tuple[List[str], Dict[Tuple[str, str], str]]: + imports: frozenset[tuple[str, str]] | Iterable[tuple[str, str]], +) -> tuple[list[str], dict[tuple[str, str], str]]: """Resolve import collisions and generate import lines.""" import_pairs = {pair for pair in imports if pair and pair[0] and pair[1]} # Filter out builtins and None modules - filtered = [ - (module, name) - for module, name in import_pairs - if module not in (None, "builtins") - ] + filtered = [(module, name) for module, name in import_pairs if module not in (None, "builtins")] name_to_modules = defaultdict(list) for module, name in filtered: name_to_modules[name].append(module) - imports_by_module: Dict[str, List[ImportSpec]] = defaultdict(list) - name_mappings: Dict[Tuple[str, str], str] = {} + imports_by_module: dict[str, list[ImportSpec]] = defaultdict(list) + name_mappings: dict[tuple[str, str], str] = {} for module, name in sorted(filtered): if len(name_to_modules[name]) > 1: @@ -60,7 +52,7 @@ def resolve_imports( imports_by_module[module].append((name, None)) name_mappings[(module, name)] = name - import_lines: List[str] = [] + import_lines: list[str] = [] for module in sorted(imports_by_module): import_lines.extend( _format_module_import( diff --git a/src/pycodify/paths.py b/src/pycodify/paths.py new file mode 100644 index 0000000..7d338aa --- /dev/null +++ b/src/pycodify/paths.py @@ -0,0 +1,148 @@ +"""Document-level factoring of absolute paths. + +A document that names several paths under one directory binds that directory +once (``path_root = Path('/data/screen')``) and spells each path below it as +``path_root / 'plate_A'``; a path that occurs more than once is bound to its +own name (``path_1``). Relative paths and strings are never factored. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from pathlib import Path +from types import MappingProxyType +from typing import Any + +from .core import ( + Assignment, + BlankLine, + CodeBlock, + FormatContext, + PathRendering, + PythonSourceLiteral, + SourceFragment, +) + + +@dataclass(frozen=True) +class PathConstructor(PythonSourceLiteral): + """A path spelled as its own constructor call, never substituted.""" + + value: Path + + def source_fragment(self, context: FormatContext) -> SourceFragment: + return PathRendering().path_source(self.value, context) + + +@dataclass +class PathCollector(PathRendering): + """Discovery pass: records every path it renders, in order.""" + + values: list[Path] = field(default_factory=list) + + def path_source(self, value: Path, context: FormatContext) -> SourceFragment: + self.values.append(value) + return super().path_source(value, context) + + +@dataclass(frozen=True) +class PathFactoring(PathRendering): + """Path bindings for one document and the expressions that replace paths.""" + + bindings: tuple[Assignment, ...] = () + expressions: Mapping[Path, SourceFragment] = field(default_factory=lambda: MappingProxyType({})) + + @classmethod + def from_occurrences(cls, occurrences: Iterable[Path]) -> PathFactoring: + normalized = tuple( + _lexically_normalized(path) for path in occurrences if path.is_absolute() + ) + counts = Counter(normalized) + ordered_values = tuple(dict.fromkeys(normalized)) + partitions: dict[tuple[str, str], list[Path]] = {} + for value in ordered_values: + first_component = value.parts[1] if len(value.parts) > 1 else "" + partitions.setdefault((value.anchor, first_component), []).append(value) + + bindings: list[Assignment] = [] + root_by_value: dict[Path, tuple[str, Path]] = {} + for values in partitions.values(): + if len(values) < 2: + continue + root = _common_ancestor(values) + if root is None or root == Path(root.anchor): + continue + root_name = "path_root" if not bindings else f"path_root_{len(bindings) + 1}" + bindings.append(Assignment(root_name, PathConstructor(root))) + for value in values: + root_by_value[value] = (root_name, root) + + expressions: dict[Path, SourceFragment] = {} + path_index = 0 + for value in ordered_values: + root_binding = root_by_value.get(value) + below_root = ( + None + if root_binding is None + else _descendant_expression(root_binding[0], root_binding[1], value) + ) + is_root = root_binding is not None and value == root_binding[1] + if counts[value] > 1 and not is_root: + path_index += 1 + path_name = f"path_{path_index}" + bindings.append(Assignment(path_name, below_root or PathConstructor(value))) + expressions[value] = SourceFragment(path_name) + continue + if below_root is not None: + expressions[value] = below_root + + return cls(tuple(bindings), MappingProxyType(expressions)) + + def path_source(self, value: Path, context: FormatContext) -> SourceFragment: + if value.is_absolute(): + expression = self.expressions.get(_lexically_normalized(value)) + if expression is not None: + return expression + return super().path_source(value, context) + + def document(self, body: Any) -> Any: + """Return ``body`` preceded by this factoring's bindings.""" + + if not self.bindings: + return body + items = body.items if isinstance(body, CodeBlock) else (body,) + return CodeBlock((*self.bindings, BlankLine(), *items)) + + +def _lexically_normalized(value: Path) -> Path: + components: list[str] = [] + for component in value.parts[1:]: + if component in ("", "."): + continue + if component == "..": + if components: + components.pop() + continue + components.append(component) + return Path(value.anchor, *components) + + +def _common_ancestor(values: list[Path]) -> Path | None: + first_parts = values[0].parts + common_count = 0 + for components in zip(*(value.parts for value in values)): + if len(set(components)) != 1: + break + common_count += 1 + if common_count == 0: + return None + return Path(first_parts[0], *first_parts[1:common_count]) + + +def _descendant_expression(root_name: str, root: Path, value: Path) -> SourceFragment: + code = root_name + for part in value.parts[len(root.parts) :]: + code = f"{code} / {part!r}" + return SourceFragment(code) diff --git a/tests/test_core.py b/tests/test_core.py index e703a4e..51837c7 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -3,13 +3,9 @@ import pytest from dataclasses import dataclass, field from enum import Enum -from types import MappingProxyType from pycodify import ( Assignment, - FormatContext, - SourceFormatter, - SourceFragment, generate_python_source, ) @@ -59,22 +55,6 @@ def test_clean_nested_unchanged_default_remains_omitted(): assert "image=" not in source -class ExtensionProbe: - pass - - -class ExtensionProbeFormatter(SourceFormatter): - priority = 200 - - def can_format(self, value): - return isinstance(value, ExtensionProbe) - - def format(self, value, context): - del value - marker = context.extension(ExtensionProbe) - return SourceFragment(repr(marker), frozenset()) - - class TestBasicGeneration: """Test basic source code generation.""" @@ -131,18 +111,6 @@ def test_executable_source(self): except Exception as e: pytest.fail(f"Generated source is not executable: {e}\n{source}") - def test_context_extensions_survive_both_render_passes(self): - context = FormatContext( - extensions=MappingProxyType({ExtensionProbe: "preserved"}) - ) - - source = generate_python_source( - Assignment("value", ExtensionProbe()), - context=context, - ) - - assert source == "value = 'preserved'" - class TestImportCollisions: """Test handling of import name collisions.""" diff --git a/tests/test_paths.py b/tests/test_paths.py new file mode 100644 index 0000000..67d68fc --- /dev/null +++ b/tests/test_paths.py @@ -0,0 +1,115 @@ +"""Document-level factoring of absolute paths. + +The contract is on native ``Path`` values: an absolute path on this platform +(``/data`` on POSIX, ``C:\\data`` on Windows) is factored, a relative one is +not, and the generated source loads back as equal ``Path`` objects. Paths are +spelled as ``Path(str(path))`` with the platform's own separators. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from pycodify import Assignment, CodeBlock, generate_python_source + +ANCHOR = Path(Path.cwd().anchor) +"""The filesystem root of this platform (``/`` or ``C:\\``).""" + + +def spelled(path: Path) -> str: + """How generated source spells one path constructor.""" + + return f"Path({str(path)!r})" + + +@dataclass(frozen=True) +class PathBearing: + path: Path + lookalike: str + + +def _load(source: str) -> dict[str, object]: + namespace: dict[str, object] = {} + exec(source, namespace) + return namespace + + +def test_factors_deepest_common_root_and_repeated_values() -> None: + screen = ANCHOR / "media" / "alice" / "T7" / "screen" + plate_a = screen / "plate_A" + plate_b = screen / "plate_B" + template = screen / "templates" / "fiducial.png" + source = generate_python_source( + CodeBlock( + ( + Assignment("plate_paths", [plate_a, plate_b]), + Assignment("pipeline_data", {plate_a: template, plate_b: None}), + ) + ) + ) + + assert f"path_root = {spelled(screen)}" in source + assert "path_1 = path_root / 'plate_A'" in source + assert "path_2 = path_root / 'plate_B'" in source + assert "path_root / 'templates' / 'fiducial.png'" in source + assert source.count(repr(str(screen))[1:-1]) == 1 + namespace = _load(source) + assert namespace["plate_paths"] == [plate_a, plate_b] + assert namespace["pipeline_data"] == {plate_a: template, plate_b: None} + + +def test_relative_paths_and_strings_are_not_factored() -> None: + relative = Path("templates") / "fiducial.png" + lookalike = str(ANCHOR / "data" / "screen" / "a") + values = [relative, lookalike, lookalike] + source = generate_python_source(Assignment("values", values)) + + assert "path_" not in source + assert spelled(relative) in source + assert source.count(repr(lookalike)) == 2 + assert _load(source)["values"] == values + + +def test_unrelated_trees_stay_separate_and_roots_follow_first_occurrence() -> None: + zeta = ANCHOR / "mnt" / "zeta" + alpha = ANCHOR / "data" / "alpha" + lone = ANCHOR / "srv" / "lone" + values = [zeta / "one", zeta / "two", alpha / "one", alpha / "two", lone] + source = generate_python_source(Assignment("values", values)) + + assert source.index(f"path_root = {spelled(zeta)}") < source.index( + f"path_root_2 = {spelled(alpha)}" + ) + assert spelled(lone) in source + assert _load(source)["values"] == values + + +def test_one_repeated_path_is_bound_without_a_root() -> None: + model = ANCHOR / "opt" / "models" / "model.bin" + source = generate_python_source(Assignment("values", [model, model])) + + assert "path_root" not in source + assert f"path_1 = {spelled(model)}" in source + assert source.count(repr(str(model))[1:-1]) == 1 + assert _load(source)["values"] == [model, model] + + +def test_dataclass_paths_are_factored_but_string_lookalikes_are_not() -> None: + absolute = ANCHOR / "data" / "screen" / "config" / "model.bin" + value = [PathBearing(absolute, str(absolute))] * 2 + source = generate_python_source(Assignment("configs", value)) + + assert source.count(repr(str(absolute))[1:-1]) == 3 + assert f"path_1 = {spelled(absolute)}" in source + assert _load(source)["configs"] == value + + +def test_temporary_directory_paths_load_back_equal(tmp_path: Path) -> None: + """A real directory on this platform round-trips through factoring.""" + + values = {tmp_path / "plate_A": [tmp_path / "plate_A" / "image.tif"], tmp_path / "plate_B": []} + source = generate_python_source(Assignment("values", values)) + + assert "path_root" in source + assert _load(source)["values"] == values diff --git a/tests/test_round_trip.py b/tests/test_round_trip.py new file mode 100644 index 0000000..9c03441 --- /dev/null +++ b/tests/test_round_trip.py @@ -0,0 +1,147 @@ +"""Generated source loads back as an equal value, for every value family.""" + +from __future__ import annotations + +import enum +import math +from dataclasses import dataclass, field +from pathlib import Path +from typing import NamedTuple + +import pytest + +from pycodify import ( + Assignment, + CodeBlock, + FormatContext, + PythonSourceLiteral, + SourceFragment, + generate_python_source, + imported_name, + to_source, +) + + +class Outer: + class Mode(enum.Enum): + FAST = "fast" + SLOW = "slow" + + class Inner: + pass + + @staticmethod + def helper(value): + return value + + @classmethod + def build(cls): + return cls() + + +class Permission(enum.Flag): + READ = 1 + WRITE = 2 + + +def module_function(value): + return value + + +class Point(NamedTuple): + x: int + y: int + + +@dataclass(frozen=True) +class Leaf: + size: int = 1 + + +@dataclass(frozen=True) +class Owner: + mode: Outer.Mode = Outer.Mode.FAST + kind: type = int + leaf: Leaf | None = None + derived: int = field(default=0, init=False) + + +@dataclass(frozen=True) +class Twice(PythonSourceLiteral): + value: int + + def source_fragment(self, context: FormatContext) -> SourceFragment: + owner = imported_name(type(self).__module__, type(self).__qualname__, context) + return SourceFragment(f"{owner.code}({self.value} * 1)", owner.imports) + + +VALUES = ( + Outer.Mode.SLOW, + Permission.READ | Permission.WRITE, + Outer.Inner, + Outer.Mode, + int, + module_function, + Outer.helper, + Outer.build, + len, + math.sqrt, + Point(1, 2), + frozenset({"b", "a"}), + {3, 1, 2}, + b"bytes", + 1 + 2j, + float("inf"), + -float("inf"), + {"key": [Path("/data/screen/a"), Path("/data/screen/b"), Path("relative/c")]}, + (Path("/opt/model.bin"), Path("/opt/model.bin")), + Owner(mode=Outer.Mode.SLOW, kind=Outer.Inner, leaf=None), + Owner(leaf=Leaf(size=3)), + Twice(4), +) + + +@pytest.mark.parametrize("clean_mode", (True, False)) +@pytest.mark.parametrize("value", VALUES, ids=repr) +def test_generated_source_loads_back_equal(value, clean_mode): + source = generate_python_source(Assignment("value", value), clean_mode=clean_mode) + namespace: dict[str, object] = {} + exec(source, namespace) + assert namespace["value"] == value, source + + +def test_nan_loads_back_as_nan(): + namespace: dict[str, object] = {} + exec(generate_python_source(Assignment("value", float("nan"))), namespace) + assert math.isnan(namespace["value"]) + + +def test_nested_enum_imports_its_outermost_class(): + source = generate_python_source(Assignment("value", Outer.Mode.FAST)) + assert f"from {__name__} import Outer" in source + assert "value = Outer.Mode.FAST" in source + + +def test_qualified_names_use_collision_aliases(): + source = generate_python_source( + CodeBlock((Assignment("a", Outer.Mode.FAST), Assignment("b", enum.Enum))) + ) + assert "a = Outer.Mode.FAST" in source + assert "b = Enum" in source + + +class Unformattable: + pass + + +def test_values_without_a_formatter_fail_loudly(): + with pytest.raises(TypeError, match="No formatter"): + to_source(Unformattable()) + + +def test_function_local_objects_fail_loudly(): + class Local(enum.Enum): + ONE = 1 + + with pytest.raises(TypeError, match="defined inside a function"): + to_source(Local.ONE)