Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions synth_lib/benchmark/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
)

_PARTITION_RE = re.compile(r"^date=(\d{4}-\d{2}-\d{2})\.parquet$")
# RealizedPathStore's directory under market_data.
REALIZED_SUBDIR = "realized"
# DATA.md lists an asset's fixed prompt start times only when there are at most this many a day.
MAX_LISTED_SLOTS = 24
# The clocks DATA.md lists daylight-saving switches for, by the name it gives them.
Expand All @@ -38,14 +40,27 @@ def _partition_day(path: Path) -> date | None:
return date.fromisoformat(m.group(1)) if m else None


def _last_day(path: Path) -> date | None:
"""The last day a partition's data reaches: its own day, or the next for a realized-path partition,
whose paths run on from the prompts that start on its day."""
day = _partition_day(path)
# realized/<asset>/<time_length>_<time_increment>/date=*.parquet
if day is None or len(path.parents) < 3 or path.parents[2].name != REALIZED_SUBDIR:
return day
return day + timedelta(days=1)


def build_snapshot(source_root: Path, dest_root: Path, cutoff: date, start: date | None = None) -> dict[str, str]:
"""Hard-links every partition in [start, cutoff] (all assets) and writes manifest.json.
"""Hard-links every partition whose data ends in [start, cutoff] (all assets) and writes manifest.json.

A realized-path partition holds paths that run into the next day, so the one dated on the cutoff
stays out.

`start` decouples what is on disk from what a campaign sees: keep 9 months of ingested
market_data and snapshot only the 2 months a campaign should train on. None = no lower
bound, i.e. everything up to the cutoff. A rebuilt snapshot keeps its partitions and its frozen
calendar."""
manifest = link_partitions(source_root, dest_root, "date=*.parquet", _partition_day, cutoff, start)
manifest = link_partitions(source_root, dest_root, "date=*.parquet", _last_day, cutoff, start)
if not manifest:
window = f"in [{start}, {cutoff}]" if start is not None else f"<= {cutoff}"
raise ValueError(f"no partition {window} under {source_root}")
Expand Down Expand Up @@ -102,7 +117,7 @@ def render_data_md(snapshot_root: Path) -> str:
lines.append(row)
lines += _timestamp_lines(store)
lines += _offline_bundle_lines(snapshot_root / "offline_data")
lines += _realized_lines(snapshot_root / "realized")
lines += _realized_lines(snapshot_root / REALIZED_SUBDIR)
lines += _daylight_saving_lines(store)
lines += _calendar_lines(snapshot_root / CALENDAR_DIR / VIEW_FILE, store)
return "\n".join(lines) + "\n"
Expand Down
14 changes: 14 additions & 0 deletions tests/benchmark/test_snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,20 @@ def test_snapshot_stops_at_cutoff_and_writes_manifest(tmp_path):
assert manifest == entries


def test_snapshot_leaves_out_realized_paths_that_start_on_the_cutoff(tmp_path):
"""A 24h path starting on the cutoff day runs into the forward window."""
src = _make_source(tmp_path)
realized = src / "realized" / "BTC" / "86400_300"
realized.mkdir(parents=True)
for day in ["2026-07-14", "2026-07-15"]:
(realized / f"date={day}.parquet").write_bytes(day.encode())
dest = tmp_path / "snapshot"
manifest = build_snapshot(src, dest, cutoff=date(2026, 7, 15))
assert sorted(p.name for p in (dest / "realized" / "BTC" / "86400_300").iterdir()) == ["date=2026-07-14.parquet"]
assert "pyth/BTC/1m/date=2026-07-15.parquet" in manifest
assert "realized/BTC/86400_300/date=2026-07-15.parquet" not in manifest


def test_snapshot_start_bound_excludes_earlier_partitions(tmp_path):
"""The disk may hold months; a campaign gets only its window (e.g. post-competition-split)."""
src = _make_source(tmp_path)
Expand Down
Loading