diff --git a/synth_lib/benchmark/snapshot.py b/synth_lib/benchmark/snapshot.py index 7c84049..63d8374 100644 --- a/synth_lib/benchmark/snapshot.py +++ b/synth_lib/benchmark/snapshot.py @@ -27,6 +27,8 @@ ) _PARTITION_RE = re.compile(r"^date=(\d{4}-\d{2}-\d{2})\.parquet$") +# RealizedPathStore's directory under market_data. +REALIZED_SUBDIR = "realized" # DATA.md lists an asset's fixed prompt start times only when there are at most this many a day. MAX_LISTED_SLOTS = 24 # The clocks DATA.md lists daylight-saving switches for, by the name it gives them. @@ -38,14 +40,27 @@ def _partition_day(path: Path) -> date | None: return date.fromisoformat(m.group(1)) if m else None +def _last_day(path: Path) -> date | None: + """The last day a partition's data reaches: its own day, or the next for a realized-path partition, + whose paths run on from the prompts that start on its day.""" + day = _partition_day(path) + # realized//_/date=*.parquet + if day is None or len(path.parents) < 3 or path.parents[2].name != REALIZED_SUBDIR: + return day + return day + timedelta(days=1) + + def build_snapshot(source_root: Path, dest_root: Path, cutoff: date, start: date | None = None) -> dict[str, str]: - """Hard-links every partition in [start, cutoff] (all assets) and writes manifest.json. + """Hard-links every partition whose data ends in [start, cutoff] (all assets) and writes manifest.json. + + A realized-path partition holds paths that run into the next day, so the one dated on the cutoff + stays out. `start` decouples what is on disk from what a campaign sees: keep 9 months of ingested market_data and snapshot only the 2 months a campaign should train on. None = no lower bound, i.e. everything up to the cutoff. A rebuilt snapshot keeps its partitions and its frozen calendar.""" - manifest = link_partitions(source_root, dest_root, "date=*.parquet", _partition_day, cutoff, start) + manifest = link_partitions(source_root, dest_root, "date=*.parquet", _last_day, cutoff, start) if not manifest: window = f"in [{start}, {cutoff}]" if start is not None else f"<= {cutoff}" raise ValueError(f"no partition {window} under {source_root}") @@ -102,7 +117,7 @@ def render_data_md(snapshot_root: Path) -> str: lines.append(row) lines += _timestamp_lines(store) lines += _offline_bundle_lines(snapshot_root / "offline_data") - lines += _realized_lines(snapshot_root / "realized") + lines += _realized_lines(snapshot_root / REALIZED_SUBDIR) lines += _daylight_saving_lines(store) lines += _calendar_lines(snapshot_root / CALENDAR_DIR / VIEW_FILE, store) return "\n".join(lines) + "\n" diff --git a/tests/benchmark/test_snapshot.py b/tests/benchmark/test_snapshot.py index c7ef95b..df18416 100644 --- a/tests/benchmark/test_snapshot.py +++ b/tests/benchmark/test_snapshot.py @@ -39,6 +39,20 @@ def test_snapshot_stops_at_cutoff_and_writes_manifest(tmp_path): assert manifest == entries +def test_snapshot_leaves_out_realized_paths_that_start_on_the_cutoff(tmp_path): + """A 24h path starting on the cutoff day runs into the forward window.""" + src = _make_source(tmp_path) + realized = src / "realized" / "BTC" / "86400_300" + realized.mkdir(parents=True) + for day in ["2026-07-14", "2026-07-15"]: + (realized / f"date={day}.parquet").write_bytes(day.encode()) + dest = tmp_path / "snapshot" + manifest = build_snapshot(src, dest, cutoff=date(2026, 7, 15)) + assert sorted(p.name for p in (dest / "realized" / "BTC" / "86400_300").iterdir()) == ["date=2026-07-14.parquet"] + assert "pyth/BTC/1m/date=2026-07-15.parquet" in manifest + assert "realized/BTC/86400_300/date=2026-07-15.parquet" not in manifest + + def test_snapshot_start_bound_excludes_earlier_partitions(tmp_path): """The disk may hold months; a campaign gets only its window (e.g. post-competition-split).""" src = _make_source(tmp_path)