From b8449a5730d92adefcf70d0eb78b62b767231b21 Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 18:55:48 +0100 Subject: [PATCH 01/13] Context length: the champion's CONTEXT_DAYS, read by predict.py and the verdict A champion sets CONTEXT_DAYS, a plain int from 3 to 30, at module level in its modeling.py; a module without it gets 7. generate_predictions.py reads it through one helper, context_days_of, which raises on any other value (the check is type(v) is int, so True and 14.0 fail too). predict.py reads it at import, so an invalid value fails the first backtest, and prints it on each summary line. The verdict's generator loads the module before the prices, writes context_days into each day's index, and run_verdict checks that every (competition, asset) run reports the same value and writes it as a top-level context_days in verdict.json. Both offline contexts are one slice, context_slice: from CONTEXT_DAYS before the prompt to context_end (predict.py took the last N rows, which started a minute or two early). The days before an asset's first partition are NaN on every minute, as a live context holds them; a missing partition after the first still raises, and predict.py's coverage check no longer counts the former as missing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- synth_lib/benchmark/generate_predictions.py | 76 ++++++++--- .../scaffold/workspace/agent/predict.py | 35 ++--- synth_lib/benchmark/verdict/run_verdict.py | 33 ++++- tests/benchmark/test_verdict.py | 34 +++++ .../test_generate_predictions_and_gate.py | 120 +++++++++++++++++- tests/evaluation/test_scaffold_predict.py | 70 +++++++++- 6 files changed, 328 insertions(+), 40 deletions(-) diff --git a/synth_lib/benchmark/generate_predictions.py b/synth_lib/benchmark/generate_predictions.py index cf392e8..18b77c3 100644 --- a/synth_lib/benchmark/generate_predictions.py +++ b/synth_lib/benchmark/generate_predictions.py @@ -18,9 +18,10 @@ Predictions land one file per day, under `//_/date=.npy` — float32, shaped -(prompts, simulations, steps) — beside a `.json` naming the prompt start_times it holds. A day -whose file already covers every requested start_time is skipped, so pointing --out-dir at a -directory that survives the run makes a re-score reuse the paths instead of regenerating them. +(prompts, simulations, steps) — beside a `.json` naming the prompt start_times it holds and the +champion's CONTEXT_DAYS. A day whose file already covers every requested start_time is skipped, so +pointing --out-dir at a directory that survives the run makes a re-score reuse the paths instead of +regenerating them. Usage: python generate_predictions.py --modeling agent/modeling.py --asset BTC \\ @@ -35,14 +36,18 @@ import importlib.util import json import sys -from datetime import timedelta +from datetime import date, timedelta from pathlib import Path +from types import ModuleType from typing import Callable import numpy as np import pandas as pd -CONTEXT_MINUTES = 7 * 24 * 60 +# The champion's context length: `CONTEXT_DAYS` in its modeling.py, a plain int within these bounds, +# and this default when the module does not set it. +DEFAULT_CONTEXT_DAYS = 7 +CONTEXT_DAYS_BOUNDS = (3, 30) # Duplicated from synth_lib.preparation.config rather than imported: this file runs inside a # champion's own venv, which does not have synth_lib. OHLCV_COLUMNS = ["open", "high", "low", "close", "volume", "trade_count"] @@ -65,6 +70,22 @@ def context_end(start: pd.Timestamp, time_length: int, issuance_lead: int | None return start - pd.Timedelta(seconds=lead + BAR_SECONDS) +def context_slice( + frame: pd.DataFrame, start: pd.Timestamp, time_length: int, context_days: int, issuance_lead: int | None = None +) -> pd.DataFrame: + """The context of the prompt at `start`: the bars from `context_days` before it to `context_end`.""" + return frame.loc[start - pd.Timedelta(days=context_days) : context_end(start, time_length, issuance_lead)] + + +def context_days_of(module: ModuleType) -> int: + """The champion's `CONTEXT_DAYS`, or the default when its module does not set it.""" + value = getattr(module, "CONTEXT_DAYS", DEFAULT_CONTEXT_DAYS) + low, high = CONTEXT_DAYS_BOUNDS + if type(value) is not int or not low <= value <= high: + raise ValueError(f"CONTEXT_DAYS must be a plain int from {low} to {high} in modeling.py, got {value!r}") + return value + + # What simulate() receives as context_prices: the minute OHLCV frame, or the Series of closes (NaN # kept) of the contract before OHLCV, for a champion written against that one. CONTEXT_MODES = ("ohlcv", "close") @@ -95,7 +116,7 @@ def covered_start_times(index_path: Path, paths_path: Path) -> set[str]: return set() # truncated by an interrupted write; regenerate the day -def load_simulate(modeling_path: Path) -> Callable: +def load_module(modeling_path: Path) -> ModuleType: # A champion imports the modules beside it by name, as when its folder is the script's own. folder = str(modeling_path.resolve().parent) if folder not in sys.path: @@ -104,7 +125,11 @@ def load_simulate(modeling_path: Path) -> Callable: assert spec is not None and spec.loader is not None # always true for a .py file path module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - return module.simulate + return module + + +def load_simulate(modeling_path: Path) -> Callable: + return load_module(modeling_path).simulate def store_root(data_root: Path, asset: str) -> Path: @@ -115,11 +140,16 @@ def store_root(data_root: Path, asset: str) -> Path: def load_minute_prices(data_root: Path, asset: str, start: pd.Timestamp, end: pd.Timestamp) -> pd.DataFrame: - """Minute OHLCV over [start, end] from daily partitions. Missing partitions raise: a silent - hole here would shrink every context that spans it without anyone noticing.""" + """Minute OHLCV over [start, end] from daily partitions. + + The days before the asset's first partition, which the store does not have before a market's + first trade, are NaN on every minute, as in a live context. A missing partition after it raises: + a silent hole here would shrink every context that spans it without anyone noticing.""" root = store_root(data_root, asset) + stored = (date.fromisoformat(p.stem.removeprefix("date=")) for p in root.glob("date=*.parquet")) + first_day = min(stored, default=start.date()) frames = [] - day = start.date() + day = max(start.date(), first_day) while day <= end.date(): path = root / f"date={day.isoformat()}.parquet" if not path.exists(): @@ -132,9 +162,18 @@ def load_minute_prices(data_root: Path, asset: str, start: pd.Timestamp, end: pd f"skips settled partitions that already exist." ) from exc day += timedelta(days=1) + if not frames: + raise FileNotFoundError( + f"no partition for {asset} from {start.date()} to {end.date()}: its first is {first_day}" + ) frame = pd.concat(frames, ignore_index=True) frame["timestamp"] = pd.to_datetime(frame["timestamp"], utc=True) frame = frame.set_index("timestamp").sort_index() + if start.date() < first_day: + head = pd.date_range( + pd.Timestamp(start.date(), tz="UTC"), frame.index[0], freq="1min", inclusive="left", name="timestamp" + ) + frame = pd.concat([pd.DataFrame(np.nan, index=head, columns=OHLCV_COLUMNS), frame]) return frame.loc[start:end] @@ -169,6 +208,7 @@ def generate_day( num_simulations: int, issuance_lead: int | None = None, context_mode: str = "ohlcv", + context_days: int = DEFAULT_CONTEXT_DAYS, ) -> int: """Generate one day's prompts and write the pair. Returns how many were generated, 0 if the day was already covered. @@ -185,9 +225,7 @@ def generate_day( for t in times: # THE no-lookahead line: the context stops at the last bar the prompt's issuance allows, # never at t — the bar labelled t closes after the prompt starts. - context = price_frame.loc[ - t - pd.Timedelta(minutes=CONTEXT_MINUTES) : context_end(t, time_length, issuance_lead) - ] + context = context_slice(price_frame, t, time_length, context_days, issuance_lead) if context_mode == "close": context = context["close"] out = simulate_fn( @@ -212,6 +250,7 @@ def generate_day( "time_length": time_length, "num_simulations": int(stacked.shape[1]), "num_steps": int(stacked.shape[2]), + "context_days": context_days, } ) ) @@ -232,6 +271,7 @@ def generate( prompt_times: list[pd.Timestamp] | None = None, issuance_lead: int | None = None, context_mode: str = "ohlcv", + context_days: int = DEFAULT_CONTEXT_DAYS, ) -> tuple[int, int]: """Returns (generated, reused) prompt counts.""" by_day: dict[str, list[pd.Timestamp]] = {} @@ -252,6 +292,7 @@ def generate( num_simulations, issuance_lead, context_mode, + context_days, ) generated += made reused += 0 if made else len(times) @@ -301,9 +342,11 @@ def main() -> None: prompt_times = None if args.prompt_times is not None: prompt_times = [_utc(t) for t in json.loads(args.prompt_times.read_text())] - prices = load_minute_prices(args.data_root, args.asset, start - pd.Timedelta(minutes=CONTEXT_MINUTES), end) + module = load_module(args.modeling) + days = context_days_of(module) + prices = load_minute_prices(args.data_root, args.asset, start - pd.Timedelta(days=days), end) generated, reused = generate( - load_simulate(args.modeling), + module.simulate, args.asset, start, end, @@ -316,8 +359,9 @@ def main() -> None: prompt_times=prompt_times, issuance_lead=args.issuance_lead_seconds, context_mode=args.context, + context_days=days, ) - print(f"{args.asset} tl={args.time_length}: {generated} predictions ({reused} reused)") + print(f"{args.asset} tl={args.time_length}: {generated} predictions ({reused} reused), CONTEXT_DAYS {days}") if __name__ == "__main__": diff --git a/synth_lib/benchmark/scaffold/workspace/agent/predict.py b/synth_lib/benchmark/scaffold/workspace/agent/predict.py index 6560259..5edfb8c 100644 --- a/synth_lib/benchmark/scaffold/workspace/agent/predict.py +++ b/synth_lib/benchmark/scaffold/workspace/agent/predict.py @@ -16,6 +16,8 @@ Works on every asset. A minute without a trade is a flat candle with trade_count 0. Prices load through the same lenient path the scored evaluation uses, so a NaN close (no price at all) reaches simulate() rather than stopping the run — your model must tolerate them (drop, never interpolate). +The context spans the CONTEXT_DAYS your modeling.py sets (7 without it), and the days before an +asset's first trade are NaN. When the snapshot carries an event calendar ($SYNTH_BENCHMARK_CALENDAR), the summary is followed by your gap to each prompt's best miner, over the prompts whose horizon holds an event of each type and @@ -50,7 +52,7 @@ from synth_lib.backtester.config import PREDICTION_MATCH_TOLERANCE_MINUTES from synth_lib.backtester.loading import get_miner_scores from synth_lib.backtester.orchestration import backtest, score_crps -from synth_lib.benchmark.generate_predictions import context_end, load_minute_prices +from synth_lib.benchmark.generate_predictions import context_days_of, context_slice, load_minute_prices from synth_lib.preparation.config import STORE_SUBDIR from synth_lib.preparation.event_calendar.events import CALENDAR_ENV, concerns @@ -61,9 +63,10 @@ WORKSPACE = AGENT_DIR.parent sys.path.insert(0, str(AGENT_DIR)) -from modeling import simulate # noqa: E402 (import after adding AGENT_DIR to sys.path) +import modeling # noqa: E402 (import after adding AGENT_DIR to sys.path) -CONTEXT_MINUTES = 7 * 24 * 60 +# Your modeling.py's CONTEXT_DAYS, read as the verdict and the live miner read it. +CONTEXT_DAYS = context_days_of(modeling) # Each run writes its predictions to its own directory under this one. PREDICTIONS_ROOT = WORKSPACE / "miner_outputs" / "campaign" # Must match synth_lib.backtester.config._OFFLINE_ENV_VAR. @@ -110,21 +113,25 @@ def store_root(asset: str) -> Path: def check_snapshot_coverage(asset: str, start: datetime, end: datetime) -> None: """Fails EARLY and clearly if the requested window falls outside the frozen snapshot. + The days before the asset's first partition are not missing: the context holds them as NaN. Without this guard, synth-lib would try to ingest the missing days (network + write) and the error would surface in an unreadable form.""" root = store_root(asset) + first = min((p.name for p in root.glob("date=*.parquet")), default="") missing = [] cursor = start.date() while cursor <= end.date(): - if not (root / f"date={cursor.isoformat()}.parquet").exists(): + name = f"date={cursor.isoformat()}.parquet" + if name > first and not (root / name).exists(): missing.append(cursor.isoformat()) cursor += timedelta(days=1) if missing: raise SystemExit( f"window outside snapshot for {asset}: missing partitions {missing[:5]}" f"{' …' if len(missing) > 5 else ''}\n" - f"The snapshot is frozen (read-only) and stops at the campaign's data_cutoff. " - f"Reduce --eval-end / --days to stay within the available coverage ({root})." + f"The snapshot is frozen (read-only): it ends at the campaign's data_cutoff, and DATA.md gives " + f"each asset's first and last day. Choose --eval-end / --days so that the window and its " + f"CONTEXT_DAYS stay on days {root} holds." ) @@ -163,14 +170,12 @@ def generate( # The same loader the scored evaluation uses. NOT MinutePriceStore.load_range, which raises # on a single NaN close. NaN reaches simulate() untouched, which is the contract — never # interpolate it away. - prices = load_minute_prices( - WORKSPACE / "market_data", asset, times[0] - pd.Timedelta(minutes=CONTEXT_MINUTES), times[-1] - ) + prices = load_minute_prices(WORKSPACE / "market_data", asset, times[0] - pd.Timedelta(days=CONTEXT_DAYS), times[-1]) out_dir.mkdir(parents=True, exist_ok=True) for t in times: # Never `prices.loc[:t]` — see the module docstring. Same cut the scored evaluation uses. - context = prices.loc[: context_end(t, time_length)][-CONTEXT_MINUTES:] - out = simulate( + context = context_slice(prices, t, time_length, CONTEXT_DAYS) + out = modeling.simulate( asset=asset, start_time=t.isoformat(), time_increment=time_increment, @@ -270,8 +275,8 @@ def main() -> None: if args.crps_only: cadence = cadence or CRPS_ONLY_CADENCE_MINUTES[args.time_length] times = prompt_times(args.asset, end, args.days, args.time_length, args.time_increment, cadence_minutes=cadence) - # the context reaches back 7 days before the first prompt: coverage must extend that far - check_snapshot_coverage(args.asset, times[0] - timedelta(minutes=CONTEXT_MINUTES), end) + # the context reaches back CONTEXT_DAYS before the first prompt: coverage must extend that far + check_snapshot_coverage(args.asset, times[0] - timedelta(days=CONTEXT_DAYS), end) PREDICTIONS_ROOT.mkdir(parents=True, exist_ok=True) out_dir = Path(tempfile.mkdtemp(prefix="run-", dir=PREDICTIONS_ROOT)) try: @@ -292,7 +297,7 @@ def main() -> None: time_increment=args.time_increment, predictions_dir=out_dir, ) - print("summary:", result.summary) + print(f"summary (CONTEXT_DAYS {CONTEXT_DAYS}):", result.summary) return result = backtest( miner_name="campaign", @@ -305,7 +310,7 @@ def main() -> None: latest_formula=args.latest_formula, match_tolerance_minutes=args.cadence_minutes or PREDICTION_MATCH_TOLERANCE_MINUTES, ) - print("summary:", result.summary) + print(f"summary (CONTEXT_DAYS {CONTEXT_DAYS}):", result.summary) if os.environ.get(CALENDAR_ENV): print_event_split(result.prompt_df, args.asset, args.time_length) finally: diff --git a/synth_lib/benchmark/verdict/run_verdict.py b/synth_lib/benchmark/verdict/run_verdict.py index 97a57a0..9bc878b 100644 --- a/synth_lib/benchmark/verdict/run_verdict.py +++ b/synth_lib/benchmark/verdict/run_verdict.py @@ -51,7 +51,7 @@ from synth_lib.backtester.loading import get_miner_scores, get_registrations, get_scores_save_order from synth_lib.backtester.scoring import Registrations, SaveOrder from synth_lib.benchmark import workspace -from synth_lib.benchmark.generate_predictions import CONTEXT_MODES +from synth_lib.benchmark.generate_predictions import CONTEXT_MODES, DEFAULT_CONTEXT_DAYS from synth_lib.benchmark.nomination import parse_champion from synth_lib.benchmark.campaign import PACKAGED_BASELINE, baseline_modeling_path from synth_lib.benchmark.sandbox.run_sandbox import DEFAULT_IMAGE, image_identity, sandbox_cmd @@ -296,6 +296,18 @@ def generate_all( print(f" generated {asset} tl={comp.time_length}", flush=True) +def reported_context_days(predictions: Path) -> int | None: + """The CONTEXT_DAYS every (competition, asset) run reported in its day indexes, None when there is + no index. An index written before the field reads as the default.""" + reported = { + json.loads(index.read_text()).get("context_days", DEFAULT_CONTEXT_DAYS) + for index in predictions.glob("*/*/date=*.json") + } + if len(reported) > 1: + raise RuntimeError(f"the generation runs under {predictions} report different CONTEXT_DAYS: {sorted(reported)}") + return reported.pop() if reported else None + + def generate_baseline( modeling: Path, data_root: Path, @@ -466,6 +478,7 @@ def verdict_payload( engine: str | None = None, context_mode: str = "ohlcv", calendar: dict | None = None, + context_days: int | None = None, ) -> dict: return { "score": result["score"], @@ -504,6 +517,8 @@ def verdict_payload( "engine_rev": engine, # What simulate() received as context_prices (generate_predictions.CONTEXT_MODES). "context_prices": context_mode, + # How many days of it: the champion's CONTEXT_DAYS, as its generation runs reported it. + "context_days": context_days, # The event calendar the champion was handed: the one its campaign's agents saw, or None. "calendar": calendar, } @@ -661,7 +676,8 @@ def main() -> None: # noqa: C901 — a linear operator script; splitting it wou context_mode=args.context, calendar=calendar if calendar_sha is not None else None, ) - print(f"[{leg}] scoring", flush=True) + context_days = reported_context_days(cache) + print(f"[{leg}] scoring (CONTEXT_DAYS {context_days})", flush=True) result = score(leg, cache, window_end, window_days, verdict_dir, sim_reg, registrations, save_order) payload = verdict_payload( result, @@ -674,6 +690,7 @@ def main() -> None: # noqa: C901 — a linear operator script; splitting it wou engine, args.context, calendar_record, + context_days, ) verdict_dir.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(payload, indent=2) + "\n") @@ -703,7 +720,17 @@ def main() -> None: # noqa: C901 — a linear operator script; splitting it wou result = score( "synth_default", predictions, window_end, window_days, baseline_dir, sim_reg, registrations, save_order ) - payload = verdict_payload(result, None, window, args.seed, sim_reg, args.live_prompts, record, engine) + payload = verdict_payload( + result, + None, + window, + args.seed, + sim_reg, + args.live_prompts, + record, + engine, + context_days=reported_context_days(predictions), + ) baseline_dir.mkdir(parents=True, exist_ok=True) baseline_out.write_text(json.dumps(payload, indent=2) + "\n") print( diff --git a/tests/benchmark/test_verdict.py b/tests/benchmark/test_verdict.py index 7e3ef83..0a1457f 100644 --- a/tests/benchmark/test_verdict.py +++ b/tests/benchmark/test_verdict.py @@ -1092,3 +1092,37 @@ def test_the_verdict_records_the_calendar(tmp_path): assert rv.verdict_payload(result, "abc1234", window, 7, None, True)["calendar"] is None calendar = {"sha256": "f" * 64, "events": 120} assert rv.verdict_payload(result, "abc1234", window, 7, None, True, calendar=calendar)["calendar"] == calendar + + +def _day_index(root, asset: str, prompt_format: str, **fields) -> None: + path = root / asset / prompt_format / "date=2026-08-30.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"start_times": [], **fields})) + + +def test_the_verdict_records_the_context_days_every_run_reported(tmp_path): + _day_index(tmp_path, "BTC", "86400_300", context_days=14) + _day_index(tmp_path, "BTC", "3600_60", context_days=14) + assert rv.reported_context_days(tmp_path) == 14 + result = { + "score": 12.3, + "mean_competition_rank": 40, + "mean_reward_vs_top": 0.123, + "mean_competition_percentile": 0.8, + "per_competition": {}, + } + payload = rv.verdict_payload(result, "abc1234", ("2026-08-30", "2026-09-09"), 7, None, True, context_days=14) + assert payload["context_days"] == 14 + + +def test_runs_reporting_different_context_days_stop_the_verdict(tmp_path): + _day_index(tmp_path, "BTC", "86400_300", context_days=14) + _day_index(tmp_path, "XAU", "86400_300", context_days=7) + with pytest.raises(RuntimeError, match=r"\[7, 14\]"): + rv.reported_context_days(tmp_path) + + +def test_an_index_written_before_the_field_reads_as_the_default(tmp_path): + _day_index(tmp_path, "BTC", "86400_300") + assert rv.reported_context_days(tmp_path) == 7 + assert rv.reported_context_days(tmp_path / "nothing generated") is None diff --git a/tests/evaluation/test_generate_predictions_and_gate.py b/tests/evaluation/test_generate_predictions_and_gate.py index 62f6023..4bd3e90 100644 --- a/tests/evaluation/test_generate_predictions_and_gate.py +++ b/tests/evaluation/test_generate_predictions_and_gate.py @@ -7,8 +7,10 @@ produce, not what the campaign backtester tolerates. """ +import sys from datetime import timezone from pathlib import Path +from types import SimpleNamespace import json @@ -18,9 +20,11 @@ from synth.simulation_input import SimulationInput # type: ignore[import-untyped] from synth.validator import response_validation_v2 # type: ignore[import-untyped] +import synth_lib.benchmark.generate_predictions as gp from synth_lib.benchmark.generate_predictions import ( + context_days_of, context_end, - CONTEXT_MINUTES, + DEFAULT_CONTEXT_DAYS, generate, load_minute_prices, load_simulate, @@ -81,7 +85,7 @@ def spy(asset, start_time, time_increment, time_length, num_simulations, context # The bar labelled t closes at t + 60s and its close is the prompt's first scored point, so # "up to t" would already be a leak. A 24h prompt is issued 120s early, plus the bar width. assert hi <= t - pd.Timedelta(seconds=180), f"context leaked past the issuance: {hi} vs {t}" - assert lo >= t - pd.Timedelta(minutes=CONTEXT_MINUTES) + assert lo >= t - pd.Timedelta(days=DEFAULT_CONTEXT_DAYS) def test_the_close_context_is_the_series_of_closes_with_its_gaps(tmp_path): @@ -139,6 +143,7 @@ def test_prediction_file_format(tmp_path): index = json.loads((root / "date=2026-07-30.json").read_text()) assert index["start_times"] == ["2026-07-30T00:00:00+00:00"] assert index["num_simulations"] == 5 and index["num_steps"] == 289 + assert index["context_days"] == DEFAULT_CONTEXT_DAYS def test_a_day_already_generated_is_not_generated_again(tmp_path): @@ -195,6 +200,117 @@ def test_load_minute_prices_raises_on_missing_partition(tmp_path): ) +def _partitions(root: Path, first: str, last: str) -> None: + for day in pd.date_range(first, last, freq="D", tz="UTC"): + idx = pd.date_range(day, periods=1440, freq="1min", tz="UTC") + pd.DataFrame({"timestamp": idx, **{c: 1.0 for c in gp.OHLCV_COLUMNS}}).to_parquet( + root / f"date={day.date()}.parquet" + ) + + +def test_days_before_the_first_partition_are_a_nan_head(tmp_path): + """A listing inside the context: the days before the asset's first trade are NaN on every + minute, as a live context holds them, and the context keeps its full length.""" + root = tmp_path / "prices" / "SPCX" / "1m" + root.mkdir(parents=True) + _partitions(root, "2026-07-30", "2026-08-02") + t = pd.Timestamp("2026-08-02 12:00", tz="UTC") + prices = load_minute_prices(tmp_path, "SPCX", t - pd.Timedelta(days=7), t) + context = gp.context_slice(prices, t, 86_400, 7) + assert context.index[0] == t - pd.Timedelta(days=7) + assert context.loc[:"2026-07-29 23:59"].isna().all().all() + assert context.loc["2026-07-30":].notna().all().all() + assert len(context) == 7 * 1440 - 2 + + +def test_a_hole_after_the_first_partition_still_raises(tmp_path): + root = tmp_path / "prices" / "SPCX" / "1m" + root.mkdir(parents=True) + _partitions(root, "2026-07-28", "2026-08-02") + (root / "date=2026-07-30.parquet").unlink() + with pytest.raises(FileNotFoundError, match="2026-07-30"): + load_minute_prices(tmp_path, "SPCX", pd.Timestamp("2026-07-26", tz="UTC"), pd.Timestamp("2026-08-02", tz="UTC")) + + +def test_a_range_before_the_first_partition_raises(tmp_path): + root = tmp_path / "prices" / "SPCX" / "1m" + root.mkdir(parents=True) + _partitions(root, "2026-07-30", "2026-08-02") + with pytest.raises(FileNotFoundError, match="no partition for SPCX"): + load_minute_prices(tmp_path, "SPCX", pd.Timestamp("2026-07-20", tz="UTC"), pd.Timestamp("2026-07-25", tz="UTC")) + + +# -- CONTEXT_DAYS ------------------------------------------------------------------------------- + + +@pytest.mark.parametrize("value", [3, 14, 30]) +def test_context_days_within_the_bounds(value): + assert context_days_of(SimpleNamespace(CONTEXT_DAYS=value)) == value + + +def test_a_champion_without_context_days_gets_the_default(): + assert context_days_of(SimpleNamespace()) == DEFAULT_CONTEXT_DAYS == 7 + + +@pytest.mark.parametrize("value", [2, 31, True, 14.0]) +def test_context_days_outside_the_bounds_raise(value): + with pytest.raises(ValueError, match="from 3 to 30"): + context_days_of(SimpleNamespace(CONTEXT_DAYS=value)) + + +RECORDING_MODELING = """ +import json +from pathlib import Path +{constant} + +def simulate(asset, start_time, time_increment, time_length, num_simulations, context_prices): + with open(Path(__file__).with_name("seen.jsonl"), "a") as seen: + seen.write(json.dumps([str(context_prices.index[0]), len(context_prices)]) + "\\n") + steps = time_length // time_increment + 1 + return (start_time, time_increment, *[[1.0] * steps] * num_simulations) +""" + + +def _run_generator(tmp_path, monkeypatch, constant: str) -> Path: + """generate_predictions.py's main over a store holding 20 days, one 24h prompt on 08-20.""" + agent = tmp_path / "agent" + agent.mkdir() + (agent / "modeling.py").write_text(RECORDING_MODELING.format(constant=constant)) + root = tmp_path / "market_data" / "prices" / "BTC" / "1m" + root.mkdir(parents=True) + _partitions(root, "2026-08-01", "2026-08-20") + argv = ["generate_predictions.py", "--modeling", str(agent / "modeling.py"), "--asset", "BTC"] + argv += ["--window-start", "2026-08-20", "--window-end", "2026-08-20T01:00", "--cadence-minutes", "60"] + argv += ["--data-root", str(tmp_path / "market_data"), "--out-dir", str(tmp_path / "out"), "--num-simulations", "2"] + monkeypatch.setattr(sys, "argv", argv) + gp.main() + return agent / "seen.jsonl" + + +@pytest.mark.parametrize("constant, days", [("CONTEXT_DAYS = 14", 14), ("", 7)]) +def test_the_generator_hands_the_champion_its_context_days(tmp_path, monkeypatch, constant, days): + seen = _run_generator(tmp_path, monkeypatch, constant) + first, rows = json.loads(seen.read_text()) + t = pd.Timestamp("2026-08-20", tz="UTC") + assert pd.Timestamp(first) == t - pd.Timedelta(days=days) + assert rows == days * 1440 - 2 # up to the bar labelled t - 3 min (issuance lead + bar width) + index = json.loads((tmp_path / "out" / "BTC" / "86400_300" / "date=2026-08-20.json").read_text()) + assert index["context_days"] == days + + +@pytest.mark.parametrize("value", ["2", "31", "True", "14.0"]) +def test_the_generator_refuses_context_days_outside_the_bounds_before_reading_prices(tmp_path, monkeypatch, value): + agent = tmp_path / "agent" + agent.mkdir() + (agent / "modeling.py").write_text(RECORDING_MODELING.format(constant=f"CONTEXT_DAYS = {value}")) + argv = ["generate_predictions.py", "--modeling", str(agent / "modeling.py"), "--asset", "BTC"] + argv += ["--window-start", "2026-08-20", "--window-end", "2026-08-21", "--cadence-minutes", "60"] + argv += ["--data-root", str(tmp_path / "no-store"), "--out-dir", str(tmp_path / "out")] + monkeypatch.setattr(sys, "argv", argv) + with pytest.raises(ValueError, match="CONTEXT_DAYS"): + gp.main() + + def test_load_simulate_resolves_a_sibling_import(tmp_path): """The verdict runs this script from the workspace root; a champion's imports of the modules beside its modeling.py must still resolve, as they do when it is served.""" diff --git a/tests/evaluation/test_scaffold_predict.py b/tests/evaluation/test_scaffold_predict.py index 108c858..5fe4250 100644 --- a/tests/evaluation/test_scaffold_predict.py +++ b/tests/evaluation/test_scaffold_predict.py @@ -9,12 +9,13 @@ import sys from datetime import datetime, timedelta, timezone from pathlib import Path -from types import SimpleNamespace +from types import ModuleType, SimpleNamespace import pandas as pd import pytest from synth_lib.benchmark.verdict.run_verdict import CADENCE_MINUTES +from synth_lib.preparation.config import OHLCV_COLUMNS from synth_lib.preparation.event_calendar.events import CALENDAR_ENV, event_frame, write_atomic SCAFFOLD_PREDICT = ( @@ -23,9 +24,7 @@ END = datetime(2026, 9, 14, tzinfo=timezone.utc) -@pytest.fixture -def predict(tmp_path, monkeypatch): - """The scaffold's predict.py, its workspace in tmp_path, the offline bundle set.""" +def _load_predict(tmp_path, monkeypatch): spec = importlib.util.spec_from_file_location("scaffold_predict", SCAFFOLD_PREDICT) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -35,6 +34,69 @@ def predict(tmp_path, monkeypatch): return module +@pytest.fixture +def predict(tmp_path, monkeypatch): + """The scaffold's predict.py, its workspace in tmp_path, the offline bundle set.""" + return _load_predict(tmp_path, monkeypatch) + + +def _with_modeling(monkeypatch, **attributes) -> list: + """Stands in for agent/modeling.py: a simulate() recording each context, plus `attributes`.""" + seen: list = [] + + def simulate(asset, start_time, time_increment, time_length, num_simulations, context_prices): + seen.append(context_prices) + return (start_time, time_increment, *[[1.0] * (time_length // time_increment + 1)] * num_simulations) + + modeling = ModuleType("modeling") + modeling.simulate = simulate + for name, value in attributes.items(): + setattr(modeling, name, value) + monkeypatch.setitem(sys.modules, "modeling", modeling) + return seen + + +def _snapshot(tmp_path, asset: str, first: str, last: str) -> Path: + root = tmp_path / "market_data" / "prices" / asset / "1m" + root.mkdir(parents=True) + for day in pd.date_range(first, last, freq="D", tz="UTC"): + idx = pd.date_range(day, periods=1440, freq="1min", tz="UTC") + pd.DataFrame({"timestamp": idx, **{c: 1.0 for c in OHLCV_COLUMNS}}).to_parquet( + root / f"date={day.date()}.parquet" + ) + return root + + +@pytest.mark.parametrize("attributes, days", [({"CONTEXT_DAYS": 14}, 14), ({}, 7)]) +def test_backtests_hand_the_champion_its_context_days(tmp_path, monkeypatch, attributes, days): + seen = _with_modeling(monkeypatch, **attributes) + predict = _load_predict(tmp_path, monkeypatch) + assert predict.CONTEXT_DAYS == days + _snapshot(tmp_path, "BTC", "2026-08-01", "2026-08-20") + t = pd.Timestamp("2026-08-20", tz="UTC") + predict.generate("BTC", [t], tmp_path / "out", num_simulations=2) + assert seen[0].index[0] == t - pd.Timedelta(days=days) + assert len(seen[0]) == days * 1440 - 2 # the verdict's slice: up to the bar labelled t - 3 min + + +@pytest.mark.parametrize("value", [2, 31, True, 14.0]) +def test_backtests_refuse_context_days_outside_the_bounds(tmp_path, monkeypatch, value): + _with_modeling(monkeypatch, CONTEXT_DAYS=value) + with pytest.raises(ValueError, match="from 3 to 30"): + _load_predict(tmp_path, monkeypatch) + + +def test_a_listing_inside_the_context_is_not_missing_but_a_hole_is(predict, tmp_path): + """The days before an asset's first partition are the context's NaN head; a missing day after it + is a hole the run must not paper over.""" + root = _snapshot(tmp_path, "SPCX", "2026-08-10", "2026-08-20") + start, end = datetime(2026, 8, 5, tzinfo=timezone.utc), datetime(2026, 8, 20, tzinfo=timezone.utc) + predict.check_snapshot_coverage("SPCX", start, end) + (root / "date=2026-08-12.parquet").unlink() + with pytest.raises(SystemExit, match=r"for SPCX: missing partitions \['2026-08-12'\]"): + predict.check_snapshot_coverage("SPCX", start, end) + + def test_refuses_to_run_without_the_offline_bundle(predict, monkeypatch): monkeypatch.delenv(predict.OFFLINE_ROOT_ENV) with pytest.raises(SystemExit, match="live API"): From 871ab0d3e4fe0ddff7e62933617031ffcc72ea7b Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 18:55:48 +0100 Subject: [PATCH 02/13] Interface probe: a synthetic context of CONTEXT_DAYS, and one call timed per format The probe builds its synthetic context at the champion's CONTEXT_DAYS and fails the check, with the usual hand-back, on a value outside 3 to 30. On success the BUDGET.md note reads "ok (24h + 1h), CONTEXT_DAYS 14, one call 2.1 s (24h) and 0.3 s (1h)": the value as the working tree sets it, and the wall time of one call of each format at that length. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- synth_lib/benchmark/nomination.py | 28 +++++++++++++----- tests/benchmark/test_nomination.py | 46 ++++++++++++++++++++++++------ 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/synth_lib/benchmark/nomination.py b/synth_lib/benchmark/nomination.py index 74d05ba..e72036d 100644 --- a/synth_lib/benchmark/nomination.py +++ b/synth_lib/benchmark/nomination.py @@ -7,12 +7,15 @@ import os import subprocess import sys +import time from dataclasses import dataclass from pathlib import Path import pandas as pd import yaml +from synth_lib.benchmark.generate_predictions import context_days_of, load_module + REPO_ROOT = Path(__file__).resolve().parent.parent @@ -98,16 +101,19 @@ def probe_simulate( (synth.validator.prompt_config.PromptConfig.num_simulations = 1000) rather than a smaller convenience value: a champion that breaks at 1000 paths (memory, a hardcoded shape) must fail here, at nomination time, instead of later at verdict/live time where - synth.validator.response_validation_v2 rejects any response with a different path count.""" + synth.validator.response_validation_v2 rejects any response with a different path count. + + The context spans the champion's CONTEXT_DAYS. On success the message is the call's wall time.""" try: - simulate = load_simulate(modeling_path) + module = load_module(modeling_path) start = pd.Timestamp("2026-01-05T00:00:00Z") - idx = pd.date_range(end=start, periods=7 * 24 * 60, freq="1min", tz="UTC") + idx = pd.date_range(end=start, periods=context_days_of(module) * 24 * 60, freq="1min", tz="UTC") context = pd.DataFrame( {c: 100.0 for c in ("open", "high", "low", "close")} | {"volume": 1.0, "trade_count": 1.0}, index=idx, ) - out = simulate( + began = time.monotonic() + out = module.simulate( asset=asset, start_time=start.isoformat(), time_increment=time_increment, @@ -115,6 +121,7 @@ def probe_simulate( num_simulations=num_simulations, context_prices=context, ) + seconds = time.monotonic() - began if not isinstance(out, tuple) or len(out) != 2 + num_simulations: got = f"tuple of length {len(out)}" if isinstance(out, tuple) else type(out).__name__ return False, f"expected return: tuple(start, increment, *{num_simulations} paths) — got {got}" @@ -124,7 +131,7 @@ def probe_simulate( return False, f"path {i}: length {len(path)} != {expected_steps} or non-numeric values" except Exception as exc: # any champion failure (including a malformed shape) is a probe failure return False, f"simulate() raised: {type(exc).__name__}: {exc}" - return True, "ok" + return True, f"{seconds:.1f} s" def probe_simulate_subprocess( @@ -178,16 +185,23 @@ def probe_simulate_subprocess( def probe_all_shapes(modeling_path: Path) -> tuple[bool, str]: - """Probes BOTH competition formats: 24h (86400/300) and 1h (3600/60). + """Probes BOTH competition formats: 24h (86400/300) and 1h (3600/60), and reports the champion's + CONTEXT_DAYS with the wall time of one call of each. A champion that ignores time_increment/time_length passes the 24h format and breaks on crypto-1h — the check must therefore cover both, otherwise the failure only shows up at verdict time, too late for the agent to fix it.""" + try: + days = context_days_of(load_module(modeling_path)) + except Exception as exc: # an import that fails, or a CONTEXT_DAYS out of bounds + return False, f"{type(exc).__name__}: {exc}" + seconds = {} for label, ti, tl in (("24h", 300, 86_400), ("1h", 60, 3_600)): ok, msg = probe_simulate(modeling_path, time_increment=ti, time_length=tl) if not ok: return False, f"[{label}] {msg}" - return True, "ok (24h + 1h)" + seconds[label] = msg + return True, f"ok (24h + 1h), CONTEXT_DAYS {days}, one call {seconds['24h']} (24h) and {seconds['1h']} (1h)" def _main() -> None: diff --git a/tests/benchmark/test_nomination.py b/tests/benchmark/test_nomination.py index 59014b4..ba3670f 100644 --- a/tests/benchmark/test_nomination.py +++ b/tests/benchmark/test_nomination.py @@ -1,6 +1,14 @@ from pathlib import Path -from synth_lib.benchmark.nomination import Champion, parse_champion, probe_simulate, probe_simulate_subprocess +import pytest + +from synth_lib.benchmark.nomination import ( + Champion, + parse_champion, + probe_all_shapes, + probe_simulate, + probe_simulate_subprocess, +) TOY_MODELING = """ def simulate(asset, start_time, time_increment=300, time_length=86400, num_simulations=100, @@ -34,6 +42,17 @@ def simulate(asset, start_time, time_increment=300, time_length=86400, num_simul SIBLING_MODULE = "def flat_paths(steps, n):\n return [[100.0] * steps for _ in range(n)]\n" +# Passes only on a synthetic context of exactly `days` days. +LENGTH_MODELING = """ +{constant} + +def simulate(asset, start_time, time_increment=300, time_length=86400, num_simulations=100, + context_prices=None): + assert len(context_prices) == {days} * 24 * 60, len(context_prices) + steps = time_length // time_increment + 1 + return (start_time, time_increment, *[[100.0] * steps for _ in range(num_simulations)]) +""" + SLEEPY_MODELING = """ import time @@ -117,7 +136,8 @@ def test_probe_subprocess_ok(tmp_path): (tmp_path / "modeling.py").write_text(TOY_MODELING) ok, msg = probe_simulate_subprocess(tmp_path / "modeling.py") assert ok is True, msg - assert msg == "ok (24h + 1h)" # the probe covers both competition formats + assert msg.startswith("ok (24h + 1h), CONTEXT_DAYS 7, one call ") # both competition formats + assert msg.endswith(" s (1h)") def test_probe_subprocess_imports_a_sibling_module(tmp_path): @@ -137,8 +157,6 @@ def test_probe_subprocess_times_out(tmp_path): def test_probe_all_shapes_rejects_hardcoded_24h_model(tmp_path): """A simulate() that hardcodes 289 points passes the 24h probe but must fail on the 1h one.""" - from synth_lib.benchmark.nomination import probe_all_shapes - (tmp_path / "modeling.py").write_text( "def simulate(asset, start_time, time_increment=300, time_length=86400,\n" " num_simulations=100, context_prices=None):\n" @@ -149,8 +167,6 @@ def test_probe_all_shapes_rejects_hardcoded_24h_model(tmp_path): def test_probe_all_shapes_accepts_generic_model(tmp_path): - from synth_lib.benchmark.nomination import probe_all_shapes - (tmp_path / "modeling.py").write_text( "def simulate(asset, start_time, time_increment=300, time_length=86400,\n" " num_simulations=100, context_prices=None):\n" @@ -167,8 +183,6 @@ def test_probe_all_shapes_rejects_model_hardcoded_to_100_paths(tmp_path): for 1000 paths and response_validation_v2 rejects anything else, so probing at the real serving size (1000, the new default) must catch this — probing at 100 would have let it through, only to break at verdict/live time.""" - from synth_lib.benchmark.nomination import probe_all_shapes - (tmp_path / "modeling.py").write_text( "def simulate(asset, start_time, time_increment=300, time_length=86400,\n" " num_simulations=100, context_prices=None):\n" @@ -180,6 +194,22 @@ def test_probe_all_shapes_rejects_model_hardcoded_to_100_paths(tmp_path): assert "1000" in msg and "102" in msg, msg # names both the expected and actual path counts +@pytest.mark.parametrize("constant, days", [("CONTEXT_DAYS = 14", 14), ("", 7)]) +def test_the_probe_hands_the_champion_its_context_days(tmp_path, constant, days): + (tmp_path / "modeling.py").write_text(LENGTH_MODELING.format(constant=constant, days=days)) + ok, msg = probe_all_shapes(tmp_path / "modeling.py") + assert ok is True, msg + assert f"CONTEXT_DAYS {days}," in msg and "(24h) and" in msg + + +@pytest.mark.parametrize("value", ["2", "31", "True", "14.0"]) +def test_the_probe_fails_context_days_outside_the_bounds(tmp_path, value): + (tmp_path / "modeling.py").write_text(LENGTH_MODELING.format(constant=f"CONTEXT_DAYS = {value}", days=7)) + ok, msg = probe_all_shapes(tmp_path / "modeling.py") + assert ok is False + assert msg == f"ValueError: CONTEXT_DAYS must be a plain int from 3 to 30 in modeling.py, got {value}" + + def test_probe_subprocess_hands_the_champion_its_calendar(tmp_path): """A champion that reads its calendar must pass the probe: the probe gets the env it is entitled to.""" (tmp_path / "modeling.py").write_text(CALENDAR_MODELING) From 7b09c2174d0692f5492fae7d21a844ad340cd818 Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 18:55:58 +0100 Subject: [PATCH 03/13] Serving: CONTEXT_DAYS live, a warm-up bounded by the venue's retention, one context per prompt ChampionMiner gains context_days, 7 by default for a miner.py that does not set it; the generated miner sets it from modeling.py, and the miner logs it at startup. Each request builds a context of that length, and the warm-up and the background refresh cover CONTEXT_DAYS + 1 date partitions. The warm-up now goes through refresh_recent, so it stops at the venue's retention as the refresh does instead of asking Hyperliquid for days it no longer serves, and it logs the share of each asset's context that holds a real close. Built contexts are kept per (asset, time_length, start_time), the newest 32, so validators sending the same prompt share one. The format is in the key because a crypto-24h and a crypto-1h prompt for the same asset can share a start_time. simulate() gets a copy of the cached frame. The unpack contract gate runs on a synthetic context of CONTEXT_DAYS + 1 days and PROVENANCE.md records the value. black 26 reformats unpack_champion.py's templates and test_champion_serving.py, as the lint check on changed files requires. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- synth_lib/serving/champion_miner.py | 32 ++-- synth_lib/serving/serve.py | 72 ++++++--- synth_lib/serving/unpack_champion.py | 55 ++++--- tests/serving/test_champion_miner_calendar.py | 2 +- tests/serving/test_champion_miner_refresh.py | 42 ++++-- tests/serving/test_champion_miner_startup.py | 4 +- tests/serving/test_champion_serving.py | 137 +++++++++++++++++- tests/serving/test_fresh_tail.py | 4 +- 8 files changed, 273 insertions(+), 75 deletions(-) diff --git a/synth_lib/serving/champion_miner.py b/synth_lib/serving/champion_miner.py index fbf2327..87813c2 100644 --- a/synth_lib/serving/champion_miner.py +++ b/synth_lib/serving/champion_miner.py @@ -1,11 +1,12 @@ """ChampionMiner: a generic Bittensor SN50 miner that serves one benchmark champion. -Subclasses set `simulate_fn` and nothing else — `unpack_champion.py` generates that subclass. +Subclasses set `simulate_fn` and `context_days`, the champion's CONTEXT_DAYS — `unpack_champion.py` +generates that subclass, and one that leaves `context_days` unset serves the default. Startup does not wait for data: a background thread warms each venue-routed minute store up, asset by asset, then keeps them all fresh, and an asset is served as soon as its own warm-up is done. Each -request slices the trailing 7-day context — a crypto-1h one topped up with the bars that closed -since the last refresh — and adapts the champion's output to the live validator contract. A second -thread refreshes the event calendar at startup and daily, and the champion finds it at +request slices the trailing `context_days` context — a crypto-1h one topped up with the bars that +closed since the last refresh — and adapts the champion's output to the live validator contract. A +second thread refreshes the event calendar at startup and daily, and the champion finds it at `$SYNTH_BENCHMARK_CALENDAR`, as in its campaign. A calendar already stored on the volume, no older than `CALENDAR_MAX_AGE_SECONDS`, is served until the first refresh replaces it. @@ -32,13 +33,14 @@ from neurons.miner import Miner # type: ignore[import-untyped] from synth.protocol import Simulation # type: ignore[import-untyped] +from synth_lib.benchmark.generate_predictions import DEFAULT_CONTEXT_DAYS from synth_lib.preparation.config import MARKET_DATA_DIR from synth_lib.preparation.event_calendar import build as calendar_build from synth_lib.preparation.event_calendar.events import CALENDAR_DIR, CALENDAR_ENV, VIEW_FILE from synth_lib.preparation.minute_price_store import MinutePriceStore from synth_lib.serving.serve import ( FRESH_TAIL_TIME_LENGTH, - WARMUP_DAYS, + ContextCache, FreshTail, serve_request, servable_assets, @@ -69,6 +71,7 @@ class ChampionMiner(Miner): """Serves one champion's simulate() over all venue-routed competition assets.""" simulate_fn: Callable | None = None + context_days: int = DEFAULT_CONTEXT_DAYS def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -81,6 +84,7 @@ def _start(self, assets: list[str]) -> None: axon comes up while the data is still being fetched.""" self._stores: dict[str, MinutePriceStore] = {asset: venue_store(asset) for asset in assets} self._tails: dict[str, FreshTail] = {asset: FreshTail(asset) for asset in assets} + self._contexts = ContextCache() # An asset joins once its whole warm-up window is in: a partly warmed store can end days back. self._ready: set[str] = set() self._calendar_root = MARKET_DATA_DIR @@ -94,22 +98,23 @@ def _start(self, assets: list[str]) -> None: self._calendar_ready.set() threading.Thread(target=self._refresh_calendar_daily, daemon=True).start() threading.Thread(target=self._warm_up_then_refresh, daemon=True).start() - logger.info("warming up %d assets in the background", len(assets)) + logger.info("CONTEXT_DAYS %d; warming up %d assets in the background", self.context_days, len(assets)) def _warm_up_then_refresh(self) -> None: calendar_deadline = time.monotonic() + CALENDAR_WAIT_SECONDS for asset in self._stores: - warm_up([asset]) + warm_up([asset], self.context_days) self._calendar_ready.wait(max(0.0, calendar_deadline - time.monotonic())) self._ready.add(asset) logger.info("warm-up complete; refreshing every %d s", REFRESH_SECONDS) self._background_refresh() def _refresh_once(self) -> None: - """One pass over every store. A venue outage must not stop the others being refreshed.""" + """One pass over every store, over the context's date partitions. A venue outage must not stop + the others being refreshed.""" for asset, store in self._stores.items(): try: - store.refresh_recent(days=WARMUP_DAYS) + store.refresh_recent(days=self.context_days + 1) except Exception as exc: logger.warning("refresh failed for %s: %s", asset, exc) @@ -149,5 +154,12 @@ async def forward_miner(self, synapse: Simulation) -> Simulation: # must not run on two threads at once. fresh_tail = self._tails[simulation_input.asset] tail = await asyncio.to_thread(fresh_tail.bars, start_time_of(simulation_input)) - synapse.simulation_output = serve_request(type(self).simulate_fn, store, simulation_input, tail=tail) + synapse.simulation_output = serve_request( + type(self).simulate_fn, + store, + simulation_input, + tail=tail, + context_days=self.context_days, + contexts=self._contexts, + ) return synapse diff --git a/synth_lib/serving/serve.py b/synth_lib/serving/serve.py index 2deb74d..7c60b21 100644 --- a/synth_lib/serving/serve.py +++ b/synth_lib/serving/serve.py @@ -17,16 +17,18 @@ Warm-up: `warm_up(...)` fills the local minute store from each asset's own venue before it is served. Be aware of the retention asymmetry — Binance serves deep minute history, while Hyperliquid's candle endpoint keeps roughly the last 5000 minutes (~3.5 days). A freshly-started miner therefore -has a full 7-day context for the cryptos and a shorter one for the HL-routed assets, which fills -in as the background refresh accumulates days. Run the miner a few days before you care about its -com-equ scores, or pre-populate the store from your own archive. +has a full context for the cryptos, and for the HL-routed assets the last 3 days of it, which fills +in as the background refresh accumulates days. Run the miner as long as the champion's context +before you care about its com-equ scores, or pre-populate the store from your own archive. """ from __future__ import annotations import logging import threading +from collections import OrderedDict from datetime import UTC, datetime, timedelta +from functools import partial from typing import Callable, Sequence import pandas as pd @@ -34,7 +36,7 @@ from synth.simulation_input import SimulationInput # type: ignore[import-untyped] from synth.validator.competition_config import ALL_COMPETITIONS # type: ignore[import-untyped] -from synth_lib.benchmark.generate_predictions import context_end +from synth_lib.benchmark.generate_predictions import DEFAULT_CONTEXT_DAYS, context_end from synth_lib.preparation.config import ( BINANCE_SYMBOLS, HYPERLIQUID_SYMBOLS, @@ -46,8 +48,6 @@ logger = logging.getLogger(__name__) -WARMUP_DAYS = 8 # 7-day context + 1 day of slack -CONTEXT_MINUTES = 7 * 24 * 60 MIN_REAL_BARS = 60 # below this a trimmed context is worse than a slightly stale one # Prompts whose context is topped up at request time (crypto-1h); the 24 h formats are not. FRESH_TAIL_TIME_LENGTH = 3_600 @@ -55,6 +55,9 @@ # The request path's venue call: no retries, and far inside the response deadline. FRESH_TAIL_TIMEOUT_SECONDS = 1.5 EMPTY_TAIL = pd.DataFrame(columns=["timestamp", *OHLCV_COLUMNS]) +# Built contexts kept for the other validators asking the same prompt: at least one minute's +# prompts across every competition. +CONTEXT_CACHE_SIZE = 32 def wrap_output(raw: Sequence, start_time: datetime, time_increment: int) -> tuple: @@ -87,18 +90,26 @@ def servable_assets() -> list[str]: return assets -def warm_up(assets: Sequence[str], days: int = WARMUP_DAYS) -> None: - """Fill the local minute store from each asset's own venue before serving. +def warm_up(assets: Sequence[str], context_days: int) -> None: + """Fill the local minute store from each asset's own venue before serving, and log how much of + the champion's context holds a real close. - Idempotent across restarts: `ingest_range` skips complete final partitions, so only genuinely - missing days are fetched. Hyperliquid's ~3.5-day retention caps how far back HL-routed assets - can reach (see the module docstring); those days simply come back empty and accumulate forward. + A `context_days` context touches context_days + 1 date partitions. Like the background refresh, + the fetch stops at the venue's retention: Hyperliquid's ~3.5 days cap how far back HL-routed + assets reach, and their older days accumulate forward (see the module docstring). Idempotent + across restarts: settled partitions are not fetched again. """ - today = datetime.now(tz=UTC).date() for asset in assets: try: - venue_store(asset).ingest_range(today - timedelta(days=days), today, verbose=False) - logger.info("warm-up complete for %s", asset) + store = venue_store(asset) + store.refresh_recent(days=context_days + 1) + context = build_context(store, datetime.now(tz=UTC).replace(second=0, microsecond=0), context_days) + logger.info( + "warm-up complete for %s: %.1f%% of its %d-day context holds a real close", + asset, + 100 * context["close"].notna().mean(), + context_days, + ) except Exception as exc: # a venue outage must not stop the miner from starting logger.warning("warm-up incomplete for %s: %s", asset, exc) @@ -139,10 +150,10 @@ def bars(self, start_time: datetime, now: datetime | None = None) -> pd.DataFram def build_context( store: MinutePriceStore, start_time: datetime, - window_minutes: int = CONTEXT_MINUTES, + context_days: int = DEFAULT_CONTEXT_DAYS, tail: pd.DataFrame | None = None, ) -> pd.DataFrame: - """The 7-day minute OHLCV context ending at `start_time`, lenient enough to serve from. + """The `context_days` minute OHLCV context ending at `start_time`, lenient enough to serve from. Deliberately not `MinutePriceStore.get_context_window`, which is strict (it raises on a missing day or a short window) — correct for backtesting, fatal on the request path, where a single @@ -154,7 +165,7 @@ def build_context( `tail` (FreshTail's bars) wins over the store for the same minute: today's partition holds a NaN row for every minute its last refresh had not settled yet. """ - context_start = start_time - timedelta(minutes=window_minutes) + context_start = start_time - timedelta(days=context_days) frames = [] day = context_start.date() while day <= start_time.date(): @@ -187,6 +198,22 @@ def build_context( return raw if len(trimmed) < MIN_REAL_BARS else trimmed +class ContextCache: + """Built contexts by (asset, time_length, start_time), so that the validators sending the same + prompt share one. Keeps the newest CONTEXT_CACHE_SIZE.""" + + def __init__(self, size: int = CONTEXT_CACHE_SIZE): + self._size = size + self._contexts: OrderedDict[tuple[str, int, datetime], pd.DataFrame] = OrderedDict() + + def get(self, key: tuple[str, int, datetime], build: Callable[[], pd.DataFrame]) -> pd.DataFrame: + if key not in self._contexts: + self._contexts[key] = build() + if len(self._contexts) > self._size: + self._contexts.popitem(last=False) + return self._contexts[key] + + def start_time_of(simulation_input: SimulationInput) -> datetime: start_time = datetime.fromisoformat(simulation_input.start_time) return start_time if start_time.tzinfo is not None else start_time.replace(tzinfo=UTC) @@ -197,16 +224,23 @@ def serve_request( store: MinutePriceStore, simulation_input: SimulationInput, tail: pd.DataFrame | None = None, + context_days: int = DEFAULT_CONTEXT_DAYS, + contexts: ContextCache | None = None, ) -> tuple: """Build the live context, run the champion, adapt the output to the live contract.""" start_time = start_time_of(simulation_input) - context = build_context(store, start_time, tail=tail) + build = partial(build_context, store, start_time, context_days, tail) + if contexts is None: + context = build() + else: + context = contexts.get((store.asset, simulation_input.time_length, start_time), build) raw = simulate_fn( asset=simulation_input.asset, start_time=simulation_input.start_time, time_increment=simulation_input.time_increment, time_length=simulation_input.time_length, num_simulations=simulation_input.num_simulations, - context_prices=context, + # A copy: the cached frame serves every validator asking this prompt. + context_prices=context.copy(), ) return wrap_output(raw, start_time, simulation_input.time_increment) diff --git a/synth_lib/serving/unpack_champion.py b/synth_lib/serving/unpack_champion.py index 73ece4e..0d680a3 100644 --- a/synth_lib/serving/unpack_champion.py +++ b/synth_lib/serving/unpack_champion.py @@ -43,15 +43,14 @@ from synth.simulation_input import SimulationInput # type: ignore[import-untyped] from synth.validator import response_validation_v2 # type: ignore[import-untyped] -from synth_lib.benchmark.generate_predictions import load_simulate +from synth_lib.benchmark.generate_predictions import context_days_of, load_module from synth_lib.serving.serve import wrap_output from synth_lib.benchmark.nomination import Champion, parse_champion # (time_increment, time_length) per nominated profile — the two live formats. PROFILE_SHAPES = {"low": (60, 3_600), "high": (300, 86_400)} -MINER_TEMPLATE = Template( - '''"""$name: serves the $campaign/$leg champion on SN50. +MINER_TEMPLATE = Template('''"""$name: serves the $campaign/$leg champion on SN50. Generated by synth_lib/serving/unpack_champion.py — provenance in PROVENANCE.md. Run via entrypoint.sh, not directly. @@ -62,15 +61,17 @@ import logging import time +from synth_lib.benchmark.generate_predictions import context_days_of from synth_lib.serving.champion_miner import ChampionMiner # Absolute, not relative: entrypoint.sh runs this file as a script, so there is no parent package -# and `from .modeling import ...` cannot resolve. As a script, its own directory is sys.path[0]. -from modeling import simulate +# and `from . import modeling` cannot resolve. As a script, its own directory is sys.path[0]. +import modeling class $class_name(ChampionMiner): - simulate_fn = staticmethod(simulate) + simulate_fn = staticmethod(modeling.simulate) + context_days = context_days_of(modeling) if __name__ == "__main__": @@ -79,11 +80,9 @@ class $class_name(ChampionMiner): while True: miner.print_info() time.sleep(60) -''' -) +''') -ENTRYPOINT_TEMPLATE = Template( - """#!/usr/bin/env bash +ENTRYPOINT_TEMPLATE = Template("""#!/usr/bin/env bash # Launches the $name miner on Bittensor Subnet 50. # Configurable via env vars. Defaults target mainnet finney. @@ -112,11 +111,9 @@ class $class_name(ChampionMiner): --axon.port "$$PORT" \\ --blacklist.force_validator_permit true \\ --blacklist.validator_min_stake "$$VALIDATOR_MIN_STAKE" -""" -) +""") -PROVENANCE_TEMPLATE = Template( - """# $name — provenance +PROVENANCE_TEMPLATE = Template("""# $name — provenance - Campaign: `$campaign`, leg `$leg` - Champion sha: `$sha` (from `campaign_results/$campaign/$leg/CHAMPION`) @@ -127,13 +124,13 @@ class $class_name(ChampionMiner): - Nomination: `CHAMPION` is the archive's, the sha above. $agent_champion - History: $bundle - Profiles: $profiles +- CONTEXT_DAYS: $context_days (days of minute history the champion is served; `modeling.py` at that sha) - Unpacked: $date - Contract gate at unpack: PASSED (`validate_responses` == CORRECT per profile, synthetic context) The champion's research report, journal and full nomination notes live in `campaign_results/$campaign/$leg/`. -""" -) +""") def run(cmd: list[str], what: str) -> None: @@ -143,15 +140,23 @@ def run(cmd: list[str], what: str) -> None: raise RuntimeError(f"{what} failed (rc={result.returncode}):\n{tail}") -def contract_gate(modeling_path: Path, profiles: tuple[str, ...]) -> None: - """Run the unpacked module through the LIVE validator contract, once per profile.""" - simulate = load_simulate(modeling_path) - idx = pd.date_range("2026-01-01", periods=8 * 24 * 60, freq="1min", tz="UTC") +def contract_gate(modeling_path: Path, profiles: tuple[str, ...]) -> int: + """Run the unpacked module through the LIVE validator contract, once per profile, on a synthetic + context of CONTEXT_DAYS + 1 days. Returns its CONTEXT_DAYS.""" + module = load_module(modeling_path) + days = context_days_of(module) + idx = pd.date_range("2026-01-01", periods=(days + 1) * 24 * 60, freq="1min", tz="UTC") rng = np.random.default_rng(7) closes = 100.0 * np.exp(np.cumsum(rng.normal(0, 1e-4, len(idx)))) context = pd.DataFrame( - {"open": closes, "high": closes * 1.0001, "low": closes * 0.9999, "close": closes, - "volume": np.ones(len(idx)), "trade_count": np.ones(len(idx))}, + { + "open": closes, + "high": closes * 1.0001, + "low": closes * 0.9999, + "close": closes, + "volume": np.ones(len(idx)), + "trade_count": np.ones(len(idx)), + }, index=idx, ) t = idx[-1] @@ -164,7 +169,7 @@ def contract_gate(modeling_path: Path, profiles: tuple[str, ...]) -> None: time_length=time_length, num_simulations=4, ) - raw = simulate( + raw = module.simulate( asset="BTC", start_time=t.isoformat(), time_increment=time_increment, @@ -176,6 +181,7 @@ def contract_gate(modeling_path: Path, profiles: tuple[str, ...]) -> None: verdict = response_validation_v2.validate_responses(wrapped, simulation_input, process_time_str="1.0") if verdict != "CORRECT": raise RuntimeError(f"contract gate failed for profile {profile}: {verdict}") + return days # What the unpacker writes into the tree. Never overwritten: if a champion happens to carry a file @@ -261,7 +267,7 @@ def unpack( with tempfile.TemporaryDirectory(prefix="unpack-") as tmp: source = champion_source(leg_dir, champion, Path(tmp)) - contract_gate(source / "modeling.py", champion.profiles) # gate BEFORE writing anything + context_days = contract_gate(source / "modeling.py", champion.profiles) # gate BEFORE writing anything # The champion's whole agent tree, verbatim: the same content the results repo publishes as # `champion_source/`, so what serves is what was scored, and the fitting scripts and journal # that explain a constant travel with it. @@ -305,6 +311,7 @@ def unpack( "sha": champion.sha, "agent_dir": champion.agent_dir, "profiles": ", ".join(champion.profiles), + "context_days": context_days, "runtime_data": ", ".join(f"`{p}`" for p in runtime_data) if runtime_data else "none (code-only champion)", # Static for the files always written (they do not exist yet at substitution time) plus the # two that depend on the leg. diff --git a/tests/serving/test_champion_miner_calendar.py b/tests/serving/test_champion_miner_calendar.py index 646a12e..c4ec2b5 100644 --- a/tests/serving/test_champion_miner_calendar.py +++ b/tests/serving/test_champion_miner_calendar.py @@ -33,7 +33,7 @@ def miner(monkeypatch, tmp_path): """A miner whose first calendar refresh is held until the test releases it.""" release = threading.Event() monkeypatch.setattr(cm, "MARKET_DATA_DIR", tmp_path / "market_data") - monkeypatch.setattr(cm, "warm_up", lambda assets: None) + monkeypatch.setattr(cm, "warm_up", lambda assets, context_days: None) monkeypatch.setattr(cm, "venue_store", lambda asset: f"store:{asset}") monkeypatch.setattr(cm, "FreshTail", lambda asset: None) monkeypatch.setattr(ChampionMiner, "_background_refresh", lambda self: None) diff --git a/tests/serving/test_champion_miner_refresh.py b/tests/serving/test_champion_miner_refresh.py index 7ca8cd2..c903656 100644 --- a/tests/serving/test_champion_miner_refresh.py +++ b/tests/serving/test_champion_miner_refresh.py @@ -8,13 +8,16 @@ from datetime import datetime, timedelta, timezone +import logging + import pandas as pd import pytest +import synth_lib.serving.serve as serve +from synth_lib.benchmark.generate_predictions import DEFAULT_CONTEXT_DAYS from synth_lib.preparation.config import OHLCV_COLUMNS from synth_lib.preparation.minute_price_store import MinutePriceStore from synth_lib.serving.champion_miner import REFRESH_SECONDS, ChampionMiner -from synth_lib.serving.serve import WARMUP_DAYS class _Store: @@ -27,9 +30,9 @@ def refresh_recent(self, days): class _Venue: source_name = "test" - retention_minutes = None - def __init__(self): + def __init__(self, retention_minutes=None): + self.retention_minutes = retention_minutes self.days: list = [] def fetch_range(self, asset, start_time, end_time): @@ -49,19 +52,29 @@ def now(cls, tz=None): return now monkeypatch.setattr("synth_lib.preparation.minute_price_store.datetime", FrozenDatetime) + monkeypatch.setattr("synth_lib.serving.serve.datetime", FrozenDatetime) return now -def _miner(stores): - miner = object.__new__(ChampionMiner) # Miner.__init__ needs a wallet, a subtensor and a chain +def _miner(stores, miner_class=ChampionMiner): + miner = object.__new__(miner_class) # Miner.__init__ needs a wallet, a subtensor and a chain miner._stores = stores return miner -def test_each_pass_refreshes_the_warm_up_window(): +def test_each_pass_refreshes_the_contexts_date_partitions(): + """A D-day context touches D + 1 date partitions; a miner.py that does not set context_days + serves the default.""" store = _Store() _miner({"BTC": store})._refresh_once() - assert store.refreshed == [WARMUP_DAYS] + assert store.refreshed == [DEFAULT_CONTEXT_DAYS + 1] + + class FourteenDays(ChampionMiner): + context_days = 14 + + store = _Store() + _miner({"BTC": store}, FourteenDays)._refresh_once() + assert store.refreshed == [15] def test_one_venue_outage_does_not_stop_the_others(): @@ -71,7 +84,7 @@ def refresh_recent(self, days): ok = _Store() _miner({"HYPE": _Broken(), "BTC": ok})._refresh_once() - assert ok.refreshed == [WARMUP_DAYS] + assert ok.refreshed == [DEFAULT_CONTEXT_DAYS + 1] def test_after_the_warm_up_a_pass_fetches_only_today(tmp_path, midday): @@ -79,12 +92,23 @@ def test_after_the_warm_up_a_pass_fetches_only_today(tmp_path, midday): venue = _Venue() store = MinutePriceStore("BTC", root=tmp_path, client=venue) today = midday.date() - store.ingest_range(today - timedelta(days=WARMUP_DAYS), today, verbose=False) + store.ingest_range(today - timedelta(days=DEFAULT_CONTEXT_DAYS + 1), today, verbose=False) venue.days.clear() _miner({"BTC": store})._refresh_once() assert venue.days == [today] +@pytest.mark.parametrize("retention_minutes, oldest", [(5_000, 3), (None, 31)]) +def test_the_warm_up_stops_at_the_venues_retention(tmp_path, midday, monkeypatch, caplog, retention_minutes, oldest): + """A 30-day context on a venue keeping ~5000 minutes: asking past them would only be told nothing.""" + venue = _Venue(retention_minutes) + monkeypatch.setattr(serve, "venue_store", lambda asset: MinutePriceStore(asset, root=tmp_path, client=venue)) + with caplog.at_level(logging.INFO, logger=serve.__name__): + serve.warm_up(["XAU"], 30) + assert min(venue.days) == midday.date() - timedelta(days=oldest) + assert "XAU" in caplog.text and "of its 30-day context holds a real close" in caplog.text + + def test_the_store_is_topped_up_every_minute(): assert REFRESH_SECONDS <= 60 diff --git a/tests/serving/test_champion_miner_startup.py b/tests/serving/test_champion_miner_startup.py index bbcaf03..afb13e7 100644 --- a/tests/serving/test_champion_miner_startup.py +++ b/tests/serving/test_champion_miner_startup.py @@ -24,7 +24,7 @@ def miner(monkeypatch): release_xau = threading.Event() warmed: list[str] = [] - def warm_up(assets): + def warm_up(assets, context_days): if assets == ["XAU"]: release_xau.wait(timeout=10) warmed.extend(assets) @@ -35,7 +35,7 @@ def warm_up(assets): monkeypatch.setattr(ChampionMiner, "_background_refresh", lambda self: None) monkeypatch.setattr(ChampionMiner, "_refresh_calendar", lambda self: None) monkeypatch.setenv(cm.CALENDAR_ENV, "") # _start sets it; monkeypatch removes it after the test - monkeypatch.setattr(cm, "serve_request", lambda fn, store, sim_input, tail: (store,)) + monkeypatch.setattr(cm, "serve_request", lambda fn, store, sim_input, **kw: (store,)) miner = object.__new__(ChampionMiner) # Miner.__init__ needs a wallet, a subtensor and a chain miner.release_xau, miner.warmed = release_xau, warmed return miner diff --git a/tests/serving/test_champion_serving.py b/tests/serving/test_champion_serving.py index 58b0c9e..f78c26b 100644 --- a/tests/serving/test_champion_serving.py +++ b/tests/serving/test_champion_serving.py @@ -1,8 +1,11 @@ """Serving wrapper: live-contract adaptation, venue routing (never Pyth), and the unpacker.""" +import asyncio import os import subprocess +import sys from pathlib import Path +from types import ModuleType, SimpleNamespace import numpy as np import pandas as pd @@ -10,10 +13,18 @@ from synth.simulation_input import SimulationInput # type: ignore[import-untyped] from synth.validator import response_validation_v2 # type: ignore[import-untyped] +import synth_lib.serving.serve as serve from synth_lib.benchmark.generate_predictions import load_simulate from synth_lib.benchmark.verdict.run_verdict import DEFAULT_SEED, HASH_SEED_ENV, SEED_ENV, generation_env -from synth_lib.serving.serve import build_context, serve_request, servable_assets, venue_store, wrap_output -from synth_lib.serving.unpack_champion import class_name_for, unpack +from synth_lib.serving.serve import ( + ContextCache, + build_context, + serve_request, + servable_assets, + venue_store, + wrap_output, +) +from synth_lib.serving.unpack_champion import MINER_TEMPLATE, class_name_for, contract_gate, unpack from synth_lib.preparation.binance_client import BinanceClient from synth_lib.preparation.minute_price_store import MinutePriceStore from synth_lib.preparation.hyperliquid_client import HyperliquidClient @@ -46,9 +57,7 @@ def _store_with(tmp_path, series: pd.Series, asset: str = "BTC"): "volume": 1.0, "trade_count": 1.0, } - ).to_parquet( - root / f"date={day.isoformat()}.parquet" - ) + ).to_parquet(root / f"date={day.isoformat()}.parquet") return MinutePriceStore(asset, root=root) @@ -56,8 +65,7 @@ def _context(series: pd.Series) -> pd.DataFrame: """The OHLCV frame simulate() is handed, from a close series.""" closes = series.to_numpy() return pd.DataFrame( - {"open": closes, "high": closes, "low": closes, "close": closes, - "volume": 1.0, "trade_count": 1.0}, + {"open": closes, "high": closes, "low": closes, "close": closes, "volume": 1.0, "trade_count": 1.0}, index=series.index, ) @@ -118,6 +126,118 @@ def test_serve_request_end_to_end_on_a_context_with_holes(tmp_path): assert response_validation_v2.validate_responses(out, sim_input, process_time_str="1.0") == "CORRECT" +def _recording(seen: list): + """A simulate() that records each context it is handed, then mutates it.""" + + def simulate(asset, start_time, time_increment, time_length, num_simulations, context_prices): + seen.append(context_prices.copy()) + context_prices["mine"] = 1.0 + last = float(context_prices["close"].iloc[-1]) + return (start_time, time_increment, *[[last] * (time_length // time_increment + 1)] * num_simulations) + + return simulate + + +def _input(t: pd.Timestamp, time_length: int = 86_400) -> SimulationInput: + increment = 300 if time_length == 86_400 else 60 + return SimulationInput( + asset="BTC", start_time=t.isoformat(), time_increment=increment, time_length=time_length, num_simulations=2 + ) + + +@pytest.mark.parametrize("days", [14, 7]) +def test_serve_request_hands_the_champion_its_context_days(tmp_path, days): + series = _series(16) + t = series.index[-1] + seen: list = [] + serve_request(_recording(seen), _store_with(tmp_path, series), _input(t), context_days=days) + assert seen[0].index[0] == t - pd.Timedelta(days=days) + + +def test_validators_asking_the_same_prompt_share_one_context(tmp_path, monkeypatch): + """Built once per (asset, format, start_time), and each call gets its own copy to change.""" + series = _series() + t = series.index[-1] + store = _store_with(tmp_path, series) + builds: list = [] + monkeypatch.setattr(serve, "build_context", lambda *a: builds.append(a) or build_context(*a)) + seen: list = [] + contexts = ContextCache() + for _ in range(3): + serve_request(_recording(seen), store, _input(t), contexts=contexts) + assert len(builds) == 1 and "mine" not in seen[-1].columns + serve_request(_recording(seen), store, _input(t, time_length=3_600), contexts=contexts) + assert len(builds) == 2, "a crypto-1h prompt with the same start_time has its own context" + + +def test_the_context_cache_returns_the_same_frame_and_keeps_the_newest(): + t = pd.Timestamp("2026-09-30 12:00", tz="UTC").to_pydatetime() + cache = ContextCache(size=2) + frame = cache.get(("BTC", 3_600, t), pd.DataFrame) + assert cache.get(("BTC", 3_600, t), lambda: pytest.fail("built twice")) is frame + for minutes in (1, 2): + cache.get(("BTC", 3_600, t + pd.Timedelta(minutes=minutes)), pd.DataFrame) + assert cache.get(("BTC", 3_600, t), pd.DataFrame) is not frame, "the oldest key is evicted" + + +def _generated_miner(monkeypatch, **attributes) -> type: + """The miner class unpack_champion generates, over a stand-in modeling module.""" + seen: list = [] + modeling = ModuleType("modeling") + modeling.simulate = _recording(seen) + modeling.seen = seen + for name, value in attributes.items(): + setattr(modeling, name, value) + monkeypatch.setitem(sys.modules, "modeling", modeling) + source = MINER_TEMPLATE.substitute(name="test", campaign="c", leg="l", class_name="TestMiner") + namespace: dict = {"__name__": "generated_miner"} + exec(compile(source, "miner.py", "exec"), namespace) + return namespace["TestMiner"] + + +@pytest.mark.parametrize("attributes, days", [({"CONTEXT_DAYS": 14}, 14), ({}, 7)]) +def test_the_generated_miner_serves_the_champions_context_days(tmp_path, monkeypatch, attributes, days): + miner_class = _generated_miner(monkeypatch, **attributes) + assert miner_class.context_days == days + series = _series(16) + t = series.index[-1] + miner = object.__new__(miner_class) # Miner.__init__ needs a wallet, a subtensor and a chain + miner._stores, miner._tails, miner._ready = {"BTC": _store_with(tmp_path, series)}, {}, {"BTC"} + miner._contexts = ContextCache() + asyncio.run(miner.forward_miner(SimpleNamespace(simulation_input=_input(t), simulation_output=None))) + assert sys.modules["modeling"].seen[0].index[0] == t - pd.Timedelta(days=days) + + +@pytest.mark.parametrize("value", [2, 31, True, 14.0]) +def test_the_generated_miner_refuses_context_days_outside_the_bounds(monkeypatch, value): + with pytest.raises(ValueError, match="from 3 to 30"): + _generated_miner(monkeypatch, CONTEXT_DAYS=value) + + +# Passes only on a synthetic context of exactly `days` days. +GATED_MODELING = """ +{constant} + +def simulate(asset, start_time, time_increment, time_length, num_simulations, context_prices=None): + assert len(context_prices) == {days} * 24 * 60, len(context_prices) + last = float(context_prices["close"].iloc[-1]) + return (start_time, time_increment, *[[last] * (time_length // time_increment + 1)] * num_simulations) +""" + + +@pytest.mark.parametrize("constant, days", [("CONTEXT_DAYS = 14", 14), ("", 7)]) +def test_the_contract_gate_runs_on_context_days_plus_one(tmp_path, constant, days): + (tmp_path / "modeling.py").write_text(GATED_MODELING.format(constant=constant, days=days + 1)) + assert contract_gate(tmp_path / "modeling.py", ("low", "high")) == days + + +@pytest.mark.parametrize("value", ["2", "31", "True", "14.0"]) +def test_the_contract_gate_refuses_context_days_outside_the_bounds(tmp_path, value): + (tmp_path / "modeling.py").write_text(GATED_MODELING.format(constant=f"CONTEXT_DAYS = {value}", days=8)) + with pytest.raises(ValueError, match="from 3 to 30"): + contract_gate(tmp_path / "modeling.py", ("low", "high")) + + def test_venue_routing_never_pyth(): assert isinstance(venue_store("BTC").client, BinanceClient) assert isinstance(venue_store("XAU").client, HyperliquidClient) @@ -172,12 +292,13 @@ def test_unpack_generates_a_servable_agent(tmp_path): assert (dest / "modeling.py").read_text() == SCAFFOLD_MODELING.read_text() miner = (dest / "miner.py").read_text() # Absolute: entrypoint.sh runs miner.py as a script, so there is no package for a relative import. - assert "from modeling import simulate" in miner + assert "\nimport modeling\n" in miner and "context_days = context_days_of(modeling)" in miner assert not any(line.startswith("from .") for line in miner.splitlines()) assert "class TestCampFakeMiner(ChampionMiner):" in miner assert (dest / "entrypoint.sh").stat().st_mode & 0o111, "entrypoint must be executable" provenance = (dest / "PROVENANCE.md").read_text() assert "test-camp" in provenance and "sha" in provenance + assert "- CONTEXT_DAYS: 7 " in provenance # unpack() gates through validate_responses before writing; existing dest must refuse with pytest.raises(FileExistsError): unpack("test-camp", "fake", None, results_dir, dest_root) diff --git a/tests/serving/test_fresh_tail.py b/tests/serving/test_fresh_tail.py index 9040e12..60ed067 100644 --- a/tests/serving/test_fresh_tail.py +++ b/tests/serving/test_fresh_tail.py @@ -109,9 +109,9 @@ def bars(self, start_time): return EMPTY_TAIL served = [] - monkeypatch.setattr(cm, "serve_request", lambda fn, store, sim_input, tail: served.append(tail) or ("ok",)) + monkeypatch.setattr(cm, "serve_request", lambda fn, store, sim_input, tail, **kw: served.append(tail) or ("ok",)) miner = object.__new__(ChampionMiner) # Miner.__init__ needs a wallet, a subtensor and a chain - miner._stores, miner._tails, miner._ready = {"BTC": None}, {"BTC": _Tail()}, {"BTC"} + miner._stores, miner._tails, miner._ready, miner._contexts = {"BTC": None}, {"BTC": _Tail()}, {"BTC"}, None asyncio.run(miner.forward_miner(_synapse(86_400))) assert miner._tails["BTC"].calls == 0 and served == [None] From 1b298bd5653fedf1ae9bbe3f441bf0cad4121d2d Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 18:55:58 +0100 Subject: [PATCH 04/13] Constitution: CONTEXT_DAYS in rule 7 and the lessons, and in both starter models Rule 7 says the context spans CONTEXT_DAYS, a plain int from 3 to 30 at module level in modeling.py (7 without it), which predict.py, the interface check, the scoring and the live miner all read, and that whole days before an asset's first trade are NaN. The lessons that said 7 days and "contexts rougher than the snapshot" follow. Both starter modeling.py files set CONTEXT_DAYS = 7. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- .../benchmark/constitution/CAMPAIGN.md.tmpl | 28 ++++++++++++------- .../benchmark/scaffold/modeling_starter.py | 3 ++ .../scaffold/workspace/agent/modeling.py | 3 ++ 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/synth_lib/benchmark/constitution/CAMPAIGN.md.tmpl b/synth_lib/benchmark/constitution/CAMPAIGN.md.tmpl index 8010a1d..76c4f88 100644 --- a/synth_lib/benchmark/constitution/CAMPAIGN.md.tmpl +++ b/synth_lib/benchmark/constitution/CAMPAIGN.md.tmpl @@ -64,8 +64,9 @@ are deployed as miners on the subnet, and emission rewards two things a mean CRP - **Every prompt answered.** A request that times out, raises, or returns a wrong path count is a miss, and a miss is scored like the field's 95th percentile on that prompt. -- **Contexts rougher than the snapshot.** A live context can hold longer gaps, and a shorter real - history after the miner restarts. +- **Contexts rougher than the snapshot.** A live context can hold longer gaps, and fewer real days + than your `CONTEXT_DAYS`: some venues serve only their last 3 days of minutes, so on a new miner + the oldest days of a longer context can be NaN. None of this makes position a target: emission is earned by a low CRPS on every prompt, with no exceptions. @@ -264,11 +265,18 @@ horizon before the window and yours is never treated as late. `simulate(asset, start_time, time_increment, time_length, num_simulations, context_prices)` -> tuple `(start_timestamp, time_increment, *paths)` — each path: a list of `time_length//time_increment + 1` floats starting at the last price in the context. - **The `context_prices` contract, exactly**: a minute-indexed pd.DataFrame over the 7 days - preceding `start_time`, with columns `open, high, low, close, volume, trade_count`. `close` - is the series you are scored against; the rest are context. It MAY CONTAIN NaN in every - column — your `simulate()` must tolerate them (drop, never interpolate). Evaluation passes - the raw frame: handle gaps inside your model, not in a loader you hope evaluation shares. + **The `context_prices` contract, exactly**: a minute-indexed pd.DataFrame over the + `CONTEXT_DAYS` days preceding `start_time`, with columns `open, high, low, close, volume, + trade_count`. `close` is the series you are scored against; the rest are context. It MAY + CONTAIN NaN in every column, on whole days before an asset's first trade too — your + `simulate()` must tolerate them (drop, never interpolate). Evaluation passes the raw frame: + handle gaps inside your model, not in a loader you hope evaluation shares. + **`CONTEXT_DAYS` is yours to choose**, and to measure like any other parameter: a plain + integer at module level in `agent/modeling.py`, from 3 to 30, not computed (a module without + it gets 7). `agent/predict.py`, the interface check, the scoring and the live miner all read + it, and any other value fails every one of them. One value serves every competition: slice a + shorter context inside `simulate()` where you want one. The interface check reports the wall + time of one call at your length. Runnable OFFLINE (weights/artifacts committed or included in the workspace). 8. **If your model uses randomness at all, seed it from `SYNTH_BENCHMARK_SEED`.** Nothing here asks you to introduce randomness — a deterministic champion ignores this rule. If you @@ -321,10 +329,10 @@ by the snapshot. Lessons earlier agents paid for: - **Test a feature by what it adds to a fast EWMA of realized volatility**, not by its own predictive power. Range estimators, volume, trade counts and the leverage effect act on future volatility through realized volatility, which the EWMA has already priced. What it cannot hold - is information outside the context: the calendar, and anything older than the 7 days + is information outside the context: the calendar, and anything older than the `CONTEXT_DAYS` `simulate()` receives.$calendar_method -- **Check what `simulate()` can see before building on it**: one asset's 7-day context per - call, nothing else. A feature that needs another asset's current prices cannot be served. +- **Check what `simulate()` can see before building on it**: one asset's `CONTEXT_DAYS` context + per call, nothing else. A feature that needs another asset's current prices cannot be served. - **Count the asset-window cells where a change helps**, across disjoint windows, not only its average: a small gain signed the same way in most cells survives; a larger one that flips sign between windows is noise. diff --git a/synth_lib/benchmark/scaffold/modeling_starter.py b/synth_lib/benchmark/scaffold/modeling_starter.py index 2c1e30e..93533ac 100644 --- a/synth_lib/benchmark/scaffold/modeling_starter.py +++ b/synth_lib/benchmark/scaffold/modeling_starter.py @@ -9,6 +9,9 @@ import numpy as np +# Days of minute history simulate() receives as context_prices: a plain int from 3 to 30. +CONTEXT_DAYS = 7 + def simulate( asset: str, diff --git a/synth_lib/benchmark/scaffold/workspace/agent/modeling.py b/synth_lib/benchmark/scaffold/workspace/agent/modeling.py index 2c1e30e..93533ac 100644 --- a/synth_lib/benchmark/scaffold/workspace/agent/modeling.py +++ b/synth_lib/benchmark/scaffold/workspace/agent/modeling.py @@ -9,6 +9,9 @@ import numpy as np +# Days of minute history simulate() receives as context_prices: a plain int from 3 to 30. +CONTEXT_DAYS = 7 + def simulate( asset: str, From ad5b02096677eace4824e6db6ae5f11749ebca7e Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 19:41:45 +0100 Subject: [PATCH 05/13] predict.py: only the context's head is exempt from the coverage check check_snapshot_coverage takes the first prompt's time. A missing day is a hole when it is on or after the asset's first partition, or on or after the first prompt's day, so a window whose prompts start before the snapshot's first day stops with the coverage error again, before any generation, instead of handing those prompts an empty context. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- .../benchmark/scaffold/workspace/agent/predict.py | 10 ++++++---- tests/benchmark/test_workspace.py | 10 ++++++++-- tests/evaluation/test_scaffold_predict.py | 13 +++++++++++-- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/synth_lib/benchmark/scaffold/workspace/agent/predict.py b/synth_lib/benchmark/scaffold/workspace/agent/predict.py index 5edfb8c..bb09e6f 100644 --- a/synth_lib/benchmark/scaffold/workspace/agent/predict.py +++ b/synth_lib/benchmark/scaffold/workspace/agent/predict.py @@ -110,19 +110,21 @@ def store_root(asset: str) -> Path: return WORKSPACE / "market_data" / STORE_SUBDIR / asset / "1m" -def check_snapshot_coverage(asset: str, start: datetime, end: datetime) -> None: +def check_snapshot_coverage(asset: str, start: datetime, end: datetime, first_prompt: datetime) -> None: """Fails EARLY and clearly if the requested window falls outside the frozen snapshot. - The days before the asset's first partition are not missing: the context holds them as NaN. + The context's days before the asset's first partition are not missing: the context holds them + as NaN. The days from the first prompt on are never exempt. Without this guard, synth-lib would try to ingest the missing days (network + write) and the error would surface in an unreadable form.""" root = store_root(asset) first = min((p.name for p in root.glob("date=*.parquet")), default="") + holes_from = min(first, f"date={first_prompt.date().isoformat()}.parquet") missing = [] cursor = start.date() while cursor <= end.date(): name = f"date={cursor.isoformat()}.parquet" - if name > first and not (root / name).exists(): + if name >= holes_from and not (root / name).exists(): missing.append(cursor.isoformat()) cursor += timedelta(days=1) if missing: @@ -276,7 +278,7 @@ def main() -> None: cadence = cadence or CRPS_ONLY_CADENCE_MINUTES[args.time_length] times = prompt_times(args.asset, end, args.days, args.time_length, args.time_increment, cadence_minutes=cadence) # the context reaches back CONTEXT_DAYS before the first prompt: coverage must extend that far - check_snapshot_coverage(args.asset, times[0] - timedelta(days=CONTEXT_DAYS), end) + check_snapshot_coverage(args.asset, times[0] - timedelta(days=CONTEXT_DAYS), end, times[0]) PREDICTIONS_ROOT.mkdir(parents=True, exist_ok=True) out_dir = Path(tempfile.mkdtemp(prefix="run-", dir=PREDICTIONS_ROOT)) try: diff --git a/tests/benchmark/test_workspace.py b/tests/benchmark/test_workspace.py index aa9b24f..b5a39db 100644 --- a/tests/benchmark/test_workspace.py +++ b/tests/benchmark/test_workspace.py @@ -267,7 +267,10 @@ def test_predict_fails_clearly_outside_snapshot(tmp_path, monkeypatch): ws, predict = _scaffold_and_import(tmp_path, monkeypatch) with pytest.raises(SystemExit) as exc: predict.check_snapshot_coverage( - "BTC", datetime(2026, 7, 10, tzinfo=timezone.utc), datetime(2026, 7, 12, tzinfo=timezone.utc) + "BTC", + datetime(2026, 7, 10, tzinfo=timezone.utc), + datetime(2026, 7, 12, tzinfo=timezone.utc), + datetime(2026, 7, 11, tzinfo=timezone.utc), ) msg = str(exc.value) assert "outside snapshot" in msg and "2026-07-10" in msg and "data_cutoff" in msg @@ -282,7 +285,10 @@ def test_predict_coverage_passes_when_partitions_present(tmp_path, monkeypatch): for day in ("2026-07-10", "2026-07-11", "2026-07-12"): (root / f"date={day}.parquet").write_bytes(b"x") predict.check_snapshot_coverage( - "BTC", datetime(2026, 7, 10, tzinfo=timezone.utc), datetime(2026, 7, 12, tzinfo=timezone.utc) + "BTC", + datetime(2026, 7, 10, tzinfo=timezone.utc), + datetime(2026, 7, 12, tzinfo=timezone.utc), + datetime(2026, 7, 11, tzinfo=timezone.utc), ) diff --git a/tests/evaluation/test_scaffold_predict.py b/tests/evaluation/test_scaffold_predict.py index 5fe4250..e626717 100644 --- a/tests/evaluation/test_scaffold_predict.py +++ b/tests/evaluation/test_scaffold_predict.py @@ -91,10 +91,19 @@ def test_a_listing_inside_the_context_is_not_missing_but_a_hole_is(predict, tmp_ is a hole the run must not paper over.""" root = _snapshot(tmp_path, "SPCX", "2026-08-10", "2026-08-20") start, end = datetime(2026, 8, 5, tzinfo=timezone.utc), datetime(2026, 8, 20, tzinfo=timezone.utc) - predict.check_snapshot_coverage("SPCX", start, end) + first_prompt = start + timedelta(days=7) + predict.check_snapshot_coverage("SPCX", start, end, first_prompt) (root / "date=2026-08-12.parquet").unlink() with pytest.raises(SystemExit, match=r"for SPCX: missing partitions \['2026-08-12'\]"): - predict.check_snapshot_coverage("SPCX", start, end) + predict.check_snapshot_coverage("SPCX", start, end, first_prompt) + + +def test_prompts_before_the_first_partition_are_missing(predict, tmp_path): + """Only the context's head is exempt: a prompt on a day the snapshot does not hold stops the run.""" + _snapshot(tmp_path, "SPCX", "2026-08-10", "2026-08-20") + first_prompt, end = datetime(2026, 8, 4, tzinfo=timezone.utc), datetime(2026, 8, 20, tzinfo=timezone.utc) + with pytest.raises(SystemExit, match=r"for SPCX: missing partitions \['2026-08-04', '2026-08-05'"): + predict.check_snapshot_coverage("SPCX", first_prompt - timedelta(days=30), end, first_prompt) def test_refuses_to_run_without_the_offline_bundle(predict, monkeypatch): From 924bb92c09d33f0de8275599bda26e2277f52d52 Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 19:45:10 +0100 Subject: [PATCH 06/13] Context length: a context's NaN head is recorded, printed and warned about A context that reaches back before an asset's first partition starts with NaN, and the store's first day is not always the market's first trade: a snapshot or a host store that starts late shortens every context the same way. That is now visible. - Each day index records context_head_nan_minutes, the most minutes one of the day's contexts runs before its first real close (generate_predictions.nan_head_minutes). - run_verdict prints, per asset, how long its contexts start with NaN, and verdict.json records it as context_head_nan_minutes (assets whose contexts all start on a real close are left out). - run_verdict warns when an asset's first partition in the data root is after the day its first context begins, and names the cache directory to delete once the missing days are ingested: the day files keep the contexts they were generated with. - predict.py prints one warning line when a context starts with NaN. - DATA.md, the constitution and predict.py's docstring say a missing file is before the market's first trade, or before the snapshot's first day. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- .../benchmark/constitution/CAMPAIGN.md.tmpl | 11 ++-- synth_lib/benchmark/generate_predictions.py | 24 +++++--- .../scaffold/workspace/agent/predict.py | 16 ++++- synth_lib/benchmark/snapshot.py | 3 +- synth_lib/benchmark/verdict/run_verdict.py | 59 +++++++++++++++++++ tests/benchmark/test_verdict.py | 44 +++++++++++++- .../test_generate_predictions_and_gate.py | 9 +++ tests/evaluation/test_scaffold_predict.py | 10 ++++ 8 files changed, 159 insertions(+), 17 deletions(-) diff --git a/synth_lib/benchmark/constitution/CAMPAIGN.md.tmpl b/synth_lib/benchmark/constitution/CAMPAIGN.md.tmpl index 76c4f88..870163f 100644 --- a/synth_lib/benchmark/constitution/CAMPAIGN.md.tmpl +++ b/synth_lib/benchmark/constitution/CAMPAIGN.md.tmpl @@ -125,8 +125,8 @@ horizon before the window and yours is never treated as late. **Read `DATA.md` at the workspace root FIRST**: it is generated from this exact snapshot at setup and carries the measured truth — the partition schema, the minute semantics (a minute without a trade is a flat candle with `trade_count` 0, NOT market hours; NaN close = no price - at all; missing file = before the market's first trade), and a - per-asset coverage table with density by era. Two caveats it quantifies that will bite if + at all; missing file = before the market's first trade, or before the snapshot's first day), + and a per-asset coverage table with density by era. Two caveats it quantifies that will bite if ignored: the early months of the newer Hyperliquid listings are genuinely THIN (low realized volatility there is real, not smoothing), and these are venue-specific perpetual prices with their own basis (xyz:SP500 trades around 7,500 — not the cash index). @@ -268,9 +268,10 @@ horizon before the window and yours is never treated as late. **The `context_prices` contract, exactly**: a minute-indexed pd.DataFrame over the `CONTEXT_DAYS` days preceding `start_time`, with columns `open, high, low, close, volume, trade_count`. `close` is the series you are scored against; the rest are context. It MAY - CONTAIN NaN in every column, on whole days before an asset's first trade too — your - `simulate()` must tolerate them (drop, never interpolate). Evaluation passes the raw frame: - handle gaps inside your model, not in a loader you hope evaluation shares. + CONTAIN NaN in every column, on whole days before an asset's first trade or before the + snapshot's first day too — your `simulate()` must tolerate them (drop, never interpolate). + Evaluation passes the raw frame: handle gaps inside your model, not in a loader you hope + evaluation shares. **`CONTEXT_DAYS` is yours to choose**, and to measure like any other parameter: a plain integer at module level in `agent/modeling.py`, from 3 to 30, not computed (a module without it gets 7). `agent/predict.py`, the interface check, the scoring and the live miner all read diff --git a/synth_lib/benchmark/generate_predictions.py b/synth_lib/benchmark/generate_predictions.py index 18b77c3..f8d9dd6 100644 --- a/synth_lib/benchmark/generate_predictions.py +++ b/synth_lib/benchmark/generate_predictions.py @@ -18,10 +18,11 @@ Predictions land one file per day, under `//_/date=.npy` — float32, shaped -(prompts, simulations, steps) — beside a `.json` naming the prompt start_times it holds and the -champion's CONTEXT_DAYS. A day whose file already covers every requested start_time is skipped, so -pointing --out-dir at a directory that survives the run makes a re-score reuse the paths instead of -regenerating them. +(prompts, simulations, steps) — beside a `.json` naming the prompt start_times it holds, the +champion's CONTEXT_DAYS and `context_head_nan_minutes`, the most minutes one of the day's contexts +runs before its first real close. A day whose file already covers every requested start_time is +skipped, so pointing --out-dir at a directory that survives the run makes a re-score reuse the paths +instead of regenerating them, with the contexts they were generated from. Usage: python generate_predictions.py --modeling agent/modeling.py --asset BTC \\ @@ -77,6 +78,12 @@ def context_slice( return frame.loc[start - pd.Timedelta(days=context_days) : context_end(start, time_length, issuance_lead)] +def nan_head_minutes(context: pd.DataFrame) -> int: + """How many minutes the context runs before its first real close.""" + real = context["close"].notna().to_numpy() + return int(real.argmax()) if real.any() else len(real) + + def context_days_of(module: ModuleType) -> int: """The champion's `CONTEXT_DAYS`, or the default when its module does not set it.""" value = getattr(module, "CONTEXT_DAYS", DEFAULT_CONTEXT_DAYS) @@ -142,9 +149,9 @@ def store_root(data_root: Path, asset: str) -> Path: def load_minute_prices(data_root: Path, asset: str, start: pd.Timestamp, end: pd.Timestamp) -> pd.DataFrame: """Minute OHLCV over [start, end] from daily partitions. - The days before the asset's first partition, which the store does not have before a market's - first trade, are NaN on every minute, as in a live context. A missing partition after it raises: - a silent hole here would shrink every context that spans it without anyone noticing.""" + The days before the asset's first partition, before the market's first trade or the store's first + day, are NaN on every minute, as in a live context. A missing partition after it raises: a silent + hole here would shrink every context that spans it without anyone noticing.""" root = store_root(data_root, asset) stored = (date.fromisoformat(p.stem.removeprefix("date=")) for p in root.glob("date=*.parquet")) first_day = min(stored, default=start.date()) @@ -222,10 +229,12 @@ def generate_day( return 0 runs = [] + head = 0 for t in times: # THE no-lookahead line: the context stops at the last bar the prompt's issuance allows, # never at t — the bar labelled t closes after the prompt starts. context = context_slice(price_frame, t, time_length, context_days, issuance_lead) + head = max(head, nan_head_minutes(context)) if context_mode == "close": context = context["close"] out = simulate_fn( @@ -251,6 +260,7 @@ def generate_day( "num_simulations": int(stacked.shape[1]), "num_steps": int(stacked.shape[2]), "context_days": context_days, + "context_head_nan_minutes": head, } ) ) diff --git a/synth_lib/benchmark/scaffold/workspace/agent/predict.py b/synth_lib/benchmark/scaffold/workspace/agent/predict.py index bb09e6f..752a582 100644 --- a/synth_lib/benchmark/scaffold/workspace/agent/predict.py +++ b/synth_lib/benchmark/scaffold/workspace/agent/predict.py @@ -17,7 +17,7 @@ through the same lenient path the scored evaluation uses, so a NaN close (no price at all) reaches simulate() rather than stopping the run — your model must tolerate them (drop, never interpolate). The context spans the CONTEXT_DAYS your modeling.py sets (7 without it), and the days before an -asset's first trade are NaN. +asset's first trade, or before the snapshot's first day, are NaN. When the snapshot carries an event calendar ($SYNTH_BENCHMARK_CALENDAR), the summary is followed by your gap to each prompt's best miner, over the prompts whose horizon holds an event of each type and @@ -52,7 +52,12 @@ from synth_lib.backtester.config import PREDICTION_MATCH_TOLERANCE_MINUTES from synth_lib.backtester.loading import get_miner_scores from synth_lib.backtester.orchestration import backtest, score_crps -from synth_lib.benchmark.generate_predictions import context_days_of, context_slice, load_minute_prices +from synth_lib.benchmark.generate_predictions import ( + context_days_of, + context_slice, + load_minute_prices, + nan_head_minutes, +) from synth_lib.preparation.config import STORE_SUBDIR from synth_lib.preparation.event_calendar.events import CALENDAR_ENV, concerns @@ -174,9 +179,11 @@ def generate( # interpolate it away. prices = load_minute_prices(WORKSPACE / "market_data", asset, times[0] - pd.Timedelta(days=CONTEXT_DAYS), times[-1]) out_dir.mkdir(parents=True, exist_ok=True) + head = 0 for t in times: # Never `prices.loc[:t]` — see the module docstring. Same cut the scored evaluation uses. context = context_slice(prices, t, time_length, CONTEXT_DAYS) + head = max(head, nan_head_minutes(context)) out = modeling.simulate( asset=asset, start_time=t.isoformat(), @@ -197,6 +204,11 @@ def generate( } name = t.strftime("%Y-%m-%d_%H:%M:%SZ") + f"_{asset}_{time_length}.json" (out_dir / name).write_text(json.dumps(record)) + if head: + print( + f"warning: {asset} contexts start with up to {head / 1440:.1f} days of NaN, before its first price " + f"in the snapshot" + ) return len(times) diff --git a/synth_lib/benchmark/snapshot.py b/synth_lib/benchmark/snapshot.py index 12390cf..7c84049 100644 --- a/synth_lib/benchmark/snapshot.py +++ b/synth_lib/benchmark/snapshot.py @@ -83,7 +83,8 @@ def render_data_md(snapshot_root: Path) -> str: " and close at the previous close, volume 0), and the validator scores against it. NOT market", " hours: these venues trade 24/7. Trade density varies by asset and era — see the table.", "- **NaN close** = no price at all: before the market's first trade, or a hole in the feed.", - "- **Missing file** = before that market's first trade. Nothing existed; nothing is hidden.", + "- **Missing file** = before that market's first trade, or before the snapshot's first day. A", + " context reaching back before an asset's first file starts with NaN days.", "- Scoring drops NaN per prompt. Never interpolate: inventing prices in the gaps is how", " you fool your own validation.", "", diff --git a/synth_lib/benchmark/verdict/run_verdict.py b/synth_lib/benchmark/verdict/run_verdict.py index 9bc878b..8eb9a97 100644 --- a/synth_lib/benchmark/verdict/run_verdict.py +++ b/synth_lib/benchmark/verdict/run_verdict.py @@ -57,6 +57,7 @@ from synth_lib.benchmark.sandbox.run_sandbox import DEFAULT_IMAGE, image_identity, sandbox_cmd from synth_lib.benchmark.manifest import sha256_file from synth_lib.benchmark.verdict.evaluate import COMPETITIONS, evaluate_candidate, prepare_offline_bundle +from synth_lib.preparation.config import STORE_SUBDIR from synth_lib.preparation.event_calendar.events import CALENDAR_DIR, CALENDAR_ENV, VIEW_FILE GENERATE_SCRIPT = Path(__file__).resolve().parents[1] / "generate_predictions.py" @@ -308,6 +309,44 @@ def reported_context_days(predictions: Path) -> int | None: return reported.pop() if reported else None +def context_heads(predictions: Path, window: tuple[str, str], lead_in: bool) -> dict[str, int]: + """Per asset with a context of the window that did not start on a real close, the most minutes one + ran before its first, from the day indexes of the window's generation days. An index written + before the field reads as 0.""" + first = min(pd.Timestamp(generation_start(window[0], c.time_length, lead_in)) for c in COMPETITIONS) + end = pd.Timestamp(window[1], tz="UTC") + heads: dict[str, int] = {} + for index in predictions.glob("*/*/date=*.json"): + day = pd.Timestamp(index.stem.removeprefix("date="), tz="UTC") + if day < first.normalize() or day >= end: + continue + minutes = json.loads(index.read_text()).get("context_head_nan_minutes", 0) + asset = index.parent.parent.name + if minutes: + heads[asset] = max(heads.get(asset, 0), minutes) + return dict(sorted(heads.items())) + + +def stores_starting_late( + data_root: Path, window_start: str, context_days: int, lead_in: bool +) -> dict[str, tuple[date, date]]: + """Per asset whose first partition in `data_root` is after the day its first context begins: that + day, and the first partition's.""" + begins: dict[str, date] = {} + for comp in COMPETITIONS: + start = pd.Timestamp(generation_start(window_start, comp.time_length, lead_in)) + day = (start - pd.Timedelta(days=context_days)).date() + for asset in comp.asset_list: + begins[asset] = min(begins.get(asset, day), day) + late = {} + for asset, day in begins.items(): + stored = (data_root / STORE_SUBDIR / asset / "1m").glob("date=*.parquet") + first = min((date.fromisoformat(p.stem.removeprefix("date=")) for p in stored), default=None) + if first is not None and first > day: + late[asset] = (day, first) + return late + + def generate_baseline( modeling: Path, data_root: Path, @@ -479,6 +518,7 @@ def verdict_payload( context_mode: str = "ohlcv", calendar: dict | None = None, context_days: int | None = None, + context_head_nan_minutes: dict[str, int] | None = None, ) -> dict: return { "score": result["score"], @@ -519,6 +559,8 @@ def verdict_payload( "context_prices": context_mode, # How many days of it: the champion's CONTEXT_DAYS, as its generation runs reported it. "context_days": context_days, + # Per asset whose contexts did not all start on a real close, the most minutes one ran before it. + "context_head_nan_minutes": context_head_nan_minutes, # The event calendar the champion was handed: the one its campaign's agents saw, or None. "calendar": calendar, } @@ -677,6 +719,21 @@ def main() -> None: # noqa: C901 — a linear operator script; splitting it wou calendar=calendar if calendar_sha is not None else None, ) context_days = reported_context_days(cache) + heads = context_heads(cache, window, not args.no_lead_in) + for asset, minutes in heads.items(): + print( + f"[{leg}] {asset}: contexts start with up to {minutes} NaN minutes ({minutes / 1440:.1f} days)", + flush=True, + ) + if context_days is not None: + late = stores_starting_late(data_root, args.window_start, context_days, not args.no_lead_in) + for asset, (begins, first) in late.items(): + print( + f"[{leg}] WARNING: {asset}'s first partition in {data_root} is {first}, after {begins}, where " + f"its first context begins. If the market traded before {first}, ingest those days and delete " + f"{cache / asset}, whose day files keep the truncated contexts they were generated with.", + flush=True, + ) print(f"[{leg}] scoring (CONTEXT_DAYS {context_days})", flush=True) result = score(leg, cache, window_end, window_days, verdict_dir, sim_reg, registrations, save_order) payload = verdict_payload( @@ -691,6 +748,7 @@ def main() -> None: # noqa: C901 — a linear operator script; splitting it wou args.context, calendar_record, context_days, + heads, ) verdict_dir.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(payload, indent=2) + "\n") @@ -730,6 +788,7 @@ def main() -> None: # noqa: C901 — a linear operator script; splitting it wou record, engine, context_days=reported_context_days(predictions), + context_head_nan_minutes=context_heads(predictions, window, not args.no_lead_in), ) baseline_dir.mkdir(parents=True, exist_ok=True) baseline_out.write_text(json.dumps(payload, indent=2) + "\n") diff --git a/tests/benchmark/test_verdict.py b/tests/benchmark/test_verdict.py index 0a1457f..a724e3e 100644 --- a/tests/benchmark/test_verdict.py +++ b/tests/benchmark/test_verdict.py @@ -1,5 +1,6 @@ import json import sys +from datetime import date import numpy as np import pandas as pd @@ -1094,8 +1095,8 @@ def test_the_verdict_records_the_calendar(tmp_path): assert rv.verdict_payload(result, "abc1234", window, 7, None, True, calendar=calendar)["calendar"] == calendar -def _day_index(root, asset: str, prompt_format: str, **fields) -> None: - path = root / asset / prompt_format / "date=2026-08-30.json" +def _day_index(root, asset: str, prompt_format: str, day: str = "2026-08-30", **fields) -> None: + path = root / asset / prompt_format / f"date={day}.json" path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps({"start_times": [], **fields})) @@ -1126,3 +1127,42 @@ def test_an_index_written_before_the_field_reads_as_the_default(tmp_path): _day_index(tmp_path, "BTC", "86400_300") assert rv.reported_context_days(tmp_path) == 7 assert rv.reported_context_days(tmp_path / "nothing generated") is None + + +def test_the_verdict_records_each_assets_nan_context_head(tmp_path): + _day_index(tmp_path, "BTC", "86400_300", context_head_nan_minutes=0) + _day_index(tmp_path, "XAU", "86400_300", context_head_nan_minutes=2880) + _day_index(tmp_path, "SOL", "86400_300", context_head_nan_minutes=1440) + _day_index(tmp_path, "SOL", "3600_60", context_head_nan_minutes=1500) + _day_index(tmp_path, "ETH", "86400_300") # written before the field + _day_index(tmp_path, "ETH", "86400_300", day="2026-08-10", context_head_nan_minutes=60) # another window + _day_index(tmp_path, "ETH", "86400_300", day="2026-09-09", context_head_nan_minutes=60) + window = ("2026-08-30", "2026-09-09") + heads = rv.context_heads(tmp_path, window, lead_in=True) + assert heads == {"SOL": 1500, "XAU": 2880} + result = { + "score": 12.3, + "mean_competition_rank": 40, + "mean_reward_vs_top": 0.123, + "mean_competition_percentile": 0.8, + "per_competition": {}, + } + payload = rv.verdict_payload(result, "abc1234", window, 7, None, True, context_head_nan_minutes=heads) + assert payload["context_head_nan_minutes"] == heads + + +def _partitions_from(data_root, asset: str, first: str) -> None: + root = data_root / "prices" / asset / "1m" + root.mkdir(parents=True) + (root / f"date={first}.parquet").touch() + (root / "date=2026-09-09.parquet").touch() + + +def test_a_store_starting_after_the_first_context_is_reported(tmp_path): + """BTC's 24h generation starts on 08-29, so a 14-day context begins on 08-15; XAU's store covers it, + and an asset without a store is the generator's to report.""" + _partitions_from(tmp_path, "BTC", "2026-08-20") + _partitions_from(tmp_path, "XAU", "2026-08-15") + late = rv.stores_starting_late(tmp_path, "2026-08-30", 14, lead_in=True) + assert late == {"BTC": (date(2026, 8, 15), date(2026, 8, 20))} + assert rv.stores_starting_late(tmp_path, "2026-08-30", 7, lead_in=True) == {} diff --git a/tests/evaluation/test_generate_predictions_and_gate.py b/tests/evaluation/test_generate_predictions_and_gate.py index 4bd3e90..9262fb3 100644 --- a/tests/evaluation/test_generate_predictions_and_gate.py +++ b/tests/evaluation/test_generate_predictions_and_gate.py @@ -296,6 +296,15 @@ def test_the_generator_hands_the_champion_its_context_days(tmp_path, monkeypatch assert rows == days * 1440 - 2 # up to the bar labelled t - 3 min (issuance lead + bar width) index = json.loads((tmp_path / "out" / "BTC" / "86400_300" / "date=2026-08-20.json").read_text()) assert index["context_days"] == days + assert index["context_head_nan_minutes"] == 0 + + +def test_the_generator_records_how_long_a_context_runs_before_the_stores_first_day(tmp_path, monkeypatch): + """A 25-day context of 08-20 reaches back to 07-26; the store starts on 08-01.""" + seen = _run_generator(tmp_path, monkeypatch, "CONTEXT_DAYS = 25") + assert json.loads(seen.read_text())[1] == 25 * 1440 - 2 + index = json.loads((tmp_path / "out" / "BTC" / "86400_300" / "date=2026-08-20.json").read_text()) + assert index["context_head_nan_minutes"] == 6 * 1440 @pytest.mark.parametrize("value", ["2", "31", "True", "14.0"]) diff --git a/tests/evaluation/test_scaffold_predict.py b/tests/evaluation/test_scaffold_predict.py index e626717..aa8f869 100644 --- a/tests/evaluation/test_scaffold_predict.py +++ b/tests/evaluation/test_scaffold_predict.py @@ -79,6 +79,16 @@ def test_backtests_hand_the_champion_its_context_days(tmp_path, monkeypatch, att assert len(seen[0]) == days * 1440 - 2 # the verdict's slice: up to the bar labelled t - 3 min +def test_a_context_reaching_before_the_snapshot_is_reported(tmp_path, monkeypatch, capsys): + seen = _with_modeling(monkeypatch) + predict = _load_predict(tmp_path, monkeypatch) + _snapshot(tmp_path, "SPCX", "2026-08-10", "2026-08-20") + times = [pd.Timestamp("2026-08-12", tz="UTC"), pd.Timestamp("2026-08-20", tz="UTC")] + predict.generate("SPCX", times, tmp_path / "out", num_simulations=2) + assert seen[0].loc[:"2026-08-09 23:59", "close"].isna().all() + assert "warning: SPCX contexts start with up to 5.0 days of NaN" in capsys.readouterr().out + + @pytest.mark.parametrize("value", [2, 31, True, 14.0]) def test_backtests_refuse_context_days_outside_the_bounds(tmp_path, monkeypatch, value): _with_modeling(monkeypatch, CONTEXT_DAYS=value) From 2c3dd9f9c081153fbcc4789b2bd3f5158663ed78 Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 19:46:08 +0100 Subject: [PATCH 07/13] Interface probe: one import of the champion, and the verdict's context shape probe_all_shapes loads the champion's module once and hands it to probe_simulate for both formats, so a module that does heavy work at import pays for it once inside the probe's timeout. The synthetic context spans [start - CONTEXT_DAYS, start] and goes through context_slice, so the champion gets the shape predict.py and the verdict hand it: CONTEXT_DAYS * 1440 - 2 rows for 24h and - 1 for 1h, ending before the bar labelled start_time. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- synth_lib/benchmark/nomination.py | 25 +++++++++++++++---------- tests/benchmark/test_nomination.py | 29 +++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/synth_lib/benchmark/nomination.py b/synth_lib/benchmark/nomination.py index e72036d..ad7459e 100644 --- a/synth_lib/benchmark/nomination.py +++ b/synth_lib/benchmark/nomination.py @@ -10,11 +10,12 @@ import time from dataclasses import dataclass from pathlib import Path +from types import ModuleType import pandas as pd import yaml -from synth_lib.benchmark.generate_predictions import context_days_of, load_module +from synth_lib.benchmark.generate_predictions import context_days_of, context_slice, load_module REPO_ROOT = Path(__file__).resolve().parent.parent @@ -87,15 +88,16 @@ def load_simulate(modeling_path: Path): def probe_simulate( - modeling_path: Path, + module: ModuleType, *, asset: str = "BTC", time_increment: int = 300, time_length: int = 86_400, num_simulations: int = 1000, ) -> tuple[bool, str]: - """Checks the shape of the contract's return value. Synthetic context => no network needed here; - real network isolation is provided by the calling sandbox (--network none). + """Checks the shape of the contract's return value, for the champion's loaded modeling module. + Synthetic context => no network needed here; real network isolation is provided by the calling + sandbox (--network none). num_simulations defaults to the validator's real serving size (synth.validator.prompt_config.PromptConfig.num_simulations = 1000) rather than a smaller @@ -103,15 +105,17 @@ def probe_simulate( fail here, at nomination time, instead of later at verdict/live time where synth.validator.response_validation_v2 rejects any response with a different path count. - The context spans the champion's CONTEXT_DAYS. On success the message is the call's wall time.""" + The context has the shape the verdict hands: the champion's CONTEXT_DAYS, cut by context_slice. On + success the message is the call's wall time.""" try: - module = load_module(modeling_path) + days = context_days_of(module) start = pd.Timestamp("2026-01-05T00:00:00Z") - idx = pd.date_range(end=start, periods=context_days_of(module) * 24 * 60, freq="1min", tz="UTC") - context = pd.DataFrame( + idx = pd.date_range(start - pd.Timedelta(days=days), start, freq="1min") + frame = pd.DataFrame( {c: 100.0 for c in ("open", "high", "low", "close")} | {"volume": 1.0, "trade_count": 1.0}, index=idx, ) + context = context_slice(frame, start, time_length, days) began = time.monotonic() out = module.simulate( asset=asset, @@ -192,12 +196,13 @@ def probe_all_shapes(modeling_path: Path) -> tuple[bool, str]: crypto-1h — the check must therefore cover both, otherwise the failure only shows up at verdict time, too late for the agent to fix it.""" try: - days = context_days_of(load_module(modeling_path)) + module = load_module(modeling_path) + days = context_days_of(module) except Exception as exc: # an import that fails, or a CONTEXT_DAYS out of bounds return False, f"{type(exc).__name__}: {exc}" seconds = {} for label, ti, tl in (("24h", 300, 86_400), ("1h", 60, 3_600)): - ok, msg = probe_simulate(modeling_path, time_increment=ti, time_length=tl) + ok, msg = probe_simulate(module, time_increment=ti, time_length=tl) if not ok: return False, f"[{label}] {msg}" seconds[label] = msg diff --git a/tests/benchmark/test_nomination.py b/tests/benchmark/test_nomination.py index ba3670f..2213bc8 100644 --- a/tests/benchmark/test_nomination.py +++ b/tests/benchmark/test_nomination.py @@ -2,6 +2,7 @@ import pytest +from synth_lib.benchmark.generate_predictions import load_module from synth_lib.benchmark.nomination import ( Champion, parse_champion, @@ -17,6 +18,14 @@ def simulate(asset, start_time, time_increment=300, time_length=86400, num_simul return (start_time, time_increment, *[[100.0] * steps for _ in range(num_simulations)]) """ +# Writes one "x" per import. +COUNTING_MODELING = """ +from pathlib import Path + +with open(Path(__file__).with_name("imports.log"), "a") as log: + log.write("x") +""" + TOY_MODELING + BROKEN_MODELING = "def simulate(*a, **k):\n return (0,)\n" UNSIZED_MODELING = "def simulate(*a, **k):\n return (0, 300, *[1] * 5)\n" @@ -42,13 +51,14 @@ def simulate(asset, start_time, time_increment=300, time_length=86400, num_simul SIBLING_MODULE = "def flat_paths(steps, n):\n return [[100.0] * steps for _ in range(n)]\n" -# Passes only on a synthetic context of exactly `days` days. +# Passes only on the context the verdict hands at `days` days: up to the bar labelled start_time - 3 min +# for 24h, - 2 min for 1h. LENGTH_MODELING = """ {constant} def simulate(asset, start_time, time_increment=300, time_length=86400, num_simulations=100, context_prices=None): - assert len(context_prices) == {days} * 24 * 60, len(context_prices) + assert len(context_prices) == {days} * 24 * 60 - (2 if time_length == 86400 else 1), len(context_prices) steps = time_length // time_increment + 1 return (start_time, time_increment, *[[100.0] * steps for _ in range(num_simulations)]) """ @@ -108,13 +118,13 @@ def test_parse_champion_lenient_handles_block_profiles(tmp_path): def test_probe_accepts_conforming_simulate(tmp_path): (tmp_path / "modeling.py").write_text(TOY_MODELING) - ok, msg = probe_simulate(tmp_path / "modeling.py", num_simulations=5) + ok, msg = probe_simulate(load_module(tmp_path / "modeling.py"), num_simulations=5) assert ok is True, msg def test_probe_rejects_bad_shape(tmp_path): (tmp_path / "modeling.py").write_text(BROKEN_MODELING) - ok, msg = probe_simulate(tmp_path / "modeling.py", num_simulations=5) + ok, msg = probe_simulate(load_module(tmp_path / "modeling.py"), num_simulations=5) assert ok is False and "tuple" in msg @@ -122,13 +132,13 @@ def test_probe_rejects_unsized_paths(tmp_path): # tuple of the right length (2 + num_simulations) but whose "paths" are ints: # len(path) must raise a TypeError that stays internal to the probe (no driver crash). (tmp_path / "modeling.py").write_text(UNSIZED_MODELING) - ok, msg = probe_simulate(tmp_path / "modeling.py", num_simulations=5) + ok, msg = probe_simulate(load_module(tmp_path / "modeling.py"), num_simulations=5) assert ok is False, msg def test_starter_modeling_passes_probe(): starter = Path(__file__).resolve().parents[2] / "synth_lib" / "benchmark" / "scaffold" / "modeling_starter.py" - ok, msg = probe_simulate(starter, num_simulations=5) + ok, msg = probe_simulate(load_module(starter), num_simulations=5) assert ok is True, msg @@ -202,6 +212,13 @@ def test_the_probe_hands_the_champion_its_context_days(tmp_path, constant, days) assert f"CONTEXT_DAYS {days}," in msg and "(24h) and" in msg +def test_the_probe_imports_the_champion_once(tmp_path): + (tmp_path / "modeling.py").write_text(COUNTING_MODELING) + ok, msg = probe_all_shapes(tmp_path / "modeling.py") + assert ok is True, msg + assert (tmp_path / "imports.log").read_text() == "x" + + @pytest.mark.parametrize("value", ["2", "31", "True", "14.0"]) def test_the_probe_fails_context_days_outside_the_bounds(tmp_path, value): (tmp_path / "modeling.py").write_text(LENGTH_MODELING.format(constant=f"CONTEXT_DAYS = {value}", days=7)) From 7c65d2de313cdcf2dcaf244732b89f32f4ee7d97 Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 19:47:39 +0100 Subject: [PATCH 08/13] Serving: a shared context is built again once the store or the tail moves on ContextCache kept the context the first validator's request built, so a store refresh, or a fresh tail holding a newer bar, that landed before a later validator asked the same prompt never reached it. The key now holds the tail's newest bar (None when the tail is empty), and the cache adds the number of store refreshes marked before the build: ChampionMiner marks one after each refresh pass. Validators asking a prompt within one snapshot of the store and the tail still share one build. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- synth_lib/serving/champion_miner.py | 1 + synth_lib/serving/serve.py | 22 +++++++++++++----- tests/serving/test_champion_miner_refresh.py | 10 ++++++++ tests/serving/test_champion_serving.py | 24 ++++++++++++++++++++ 4 files changed, 51 insertions(+), 6 deletions(-) diff --git a/synth_lib/serving/champion_miner.py b/synth_lib/serving/champion_miner.py index 87813c2..484fd89 100644 --- a/synth_lib/serving/champion_miner.py +++ b/synth_lib/serving/champion_miner.py @@ -117,6 +117,7 @@ def _refresh_once(self) -> None: store.refresh_recent(days=self.context_days + 1) except Exception as exc: logger.warning("refresh failed for %s: %s", asset, exc) + self._contexts.store_refreshed() def _refresh_calendar(self) -> None: """One update of the calendar: a source that fails keeps its rows, and the champion keeps the last view.""" diff --git a/synth_lib/serving/serve.py b/synth_lib/serving/serve.py index 7c60b21..a73d837 100644 --- a/synth_lib/serving/serve.py +++ b/synth_lib/serving/serve.py @@ -199,14 +199,21 @@ def build_context( class ContextCache: - """Built contexts by (asset, time_length, start_time), so that the validators sending the same - prompt share one. Keeps the newest CONTEXT_CACHE_SIZE.""" + """Built contexts, so that the validators sending the same prompt share one while the data under it + stays the same: keyed by the caller's key and the store refreshes marked before the build. Keeps + the newest CONTEXT_CACHE_SIZE.""" def __init__(self, size: int = CONTEXT_CACHE_SIZE): self._size = size - self._contexts: OrderedDict[tuple[str, int, datetime], pd.DataFrame] = OrderedDict() + self._refreshes = 0 + self._contexts: OrderedDict[tuple, pd.DataFrame] = OrderedDict() - def get(self, key: tuple[str, int, datetime], build: Callable[[], pd.DataFrame]) -> pd.DataFrame: + def store_refreshed(self) -> None: + """Marks a refresh of the stores: a context built before it is built again.""" + self._refreshes += 1 + + def get(self, key: tuple, build: Callable[[], pd.DataFrame]) -> pd.DataFrame: + key = (*key, self._refreshes) if key not in self._contexts: self._contexts[key] = build() if len(self._contexts) > self._size: @@ -227,13 +234,16 @@ def serve_request( context_days: int = DEFAULT_CONTEXT_DAYS, contexts: ContextCache | None = None, ) -> tuple: - """Build the live context, run the champion, adapt the output to the live contract.""" + """Build the live context, run the champion, adapt the output to the live contract. With `contexts`, + the validators asking the same prompt share its context while the store and the tail's newest bar + stay the same.""" start_time = start_time_of(simulation_input) build = partial(build_context, store, start_time, context_days, tail) if contexts is None: context = build() else: - context = contexts.get((store.asset, simulation_input.time_length, start_time), build) + tail_end = None if tail is None or tail.empty else tail["timestamp"].max() + context = contexts.get((store.asset, simulation_input.time_length, start_time, tail_end), build) raw = simulate_fn( asset=simulation_input.asset, start_time=simulation_input.start_time, diff --git a/tests/serving/test_champion_miner_refresh.py b/tests/serving/test_champion_miner_refresh.py index c903656..88cbb46 100644 --- a/tests/serving/test_champion_miner_refresh.py +++ b/tests/serving/test_champion_miner_refresh.py @@ -18,6 +18,7 @@ from synth_lib.preparation.config import OHLCV_COLUMNS from synth_lib.preparation.minute_price_store import MinutePriceStore from synth_lib.serving.champion_miner import REFRESH_SECONDS, ChampionMiner +from synth_lib.serving.serve import ContextCache class _Store: @@ -59,6 +60,7 @@ def now(cls, tz=None): def _miner(stores, miner_class=ChampionMiner): miner = object.__new__(miner_class) # Miner.__init__ needs a wallet, a subtensor and a chain miner._stores = stores + miner._contexts = ContextCache() return miner @@ -77,6 +79,14 @@ class FourteenDays(ChampionMiner): assert store.refreshed == [15] +def test_each_pass_rebuilds_the_cached_contexts(): + miner = _miner({"BTC": _Store()}) + t = datetime(2026, 10, 29, 9, 30, tzinfo=timezone.utc) + frame = miner._contexts.get(("BTC", 3_600, t), pd.DataFrame) + miner._refresh_once() + assert miner._contexts.get(("BTC", 3_600, t), pd.DataFrame) is not frame + + def test_one_venue_outage_does_not_stop_the_others(): class _Broken(_Store): def refresh_recent(self, days): diff --git a/tests/serving/test_champion_serving.py b/tests/serving/test_champion_serving.py index f78c26b..77c0bf9 100644 --- a/tests/serving/test_champion_serving.py +++ b/tests/serving/test_champion_serving.py @@ -26,6 +26,7 @@ ) from synth_lib.serving.unpack_champion import MINER_TEMPLATE, class_name_for, contract_gate, unpack from synth_lib.preparation.binance_client import BinanceClient +from synth_lib.preparation.config import OHLCV_COLUMNS from synth_lib.preparation.minute_price_store import MinutePriceStore from synth_lib.preparation.hyperliquid_client import HyperliquidClient @@ -170,6 +171,29 @@ def test_validators_asking_the_same_prompt_share_one_context(tmp_path, monkeypat assert len(builds) == 2, "a crypto-1h prompt with the same start_time has its own context" +def test_a_shared_context_is_built_again_once_the_store_or_the_tail_moves_on(tmp_path): + """A store refresh, or a tail holding a newer bar, between two validators asking the same prompt + reaches the second one.""" + series = _series() + t = series.index[-1] + pd.Timedelta(minutes=3) + early = _store_with(tmp_path / "early", series.iloc[:-5]) + later = _store_with(tmp_path / "later", series) # the same BTC store, refreshed + seen: list = [] + contexts = ContextCache() + serve_request(_recording(seen), early, _input(t), contexts=contexts) + serve_request(_recording(seen), later, _input(t), contexts=contexts) + contexts.store_refreshed() + serve_request(_recording(seen), later, _input(t), contexts=contexts) + assert [c.index[-1] for c in seen] == [t - pd.Timedelta(minutes=8)] * 2 + [t - pd.Timedelta(minutes=3)] + + seen.clear() + for minutes in (3, 2, 2): + index = pd.date_range(end=t - pd.Timedelta(minutes=minutes), periods=15, freq="1min") + tail = pd.DataFrame({"timestamp": index, **{c: 1.0 for c in OHLCV_COLUMNS}}) + serve_request(_recording(seen), early, _input(t, time_length=3_600), tail=tail, contexts=contexts) + assert [c.index[-1] for c in seen] == [t - pd.Timedelta(minutes=m) for m in (3, 2, 2)] + + def test_the_context_cache_returns_the_same_frame_and_keeps_the_newest(): t = pd.Timestamp("2026-09-30 12:00", tz="UTC").to_pydatetime() cache = ContextCache(size=2) From c97136cfe62f553d93f52cc834e8e2ad36f31a64 Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 19:48:38 +0100 Subject: [PATCH 09/13] Serving: the assets already served stay fresh while the others warm up On a cold volume the background refresh started only once every asset had warmed up, and a long context makes each Binance asset's warm-up fetch a month of partitions: the assets served first aged meanwhile. After each asset's warm-up, the warm-up thread now refreshes today's partition of the assets already ready (refresh_recent(days=1) on a snapshot of the ready set, each in its own try), never one still warming up, and marks the refresh for the cached contexts like the background pass does. The refresh test's stdlib imports form one group. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- synth_lib/serving/champion_miner.py | 27 ++++++++++++-------- tests/serving/test_champion_miner_refresh.py | 26 +++++++++++++++++-- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/synth_lib/serving/champion_miner.py b/synth_lib/serving/champion_miner.py index 484fd89..06448d2 100644 --- a/synth_lib/serving/champion_miner.py +++ b/synth_lib/serving/champion_miner.py @@ -3,12 +3,13 @@ Subclasses set `simulate_fn` and `context_days`, the champion's CONTEXT_DAYS — `unpack_champion.py` generates that subclass, and one that leaves `context_days` unset serves the default. Startup does not wait for data: a background thread warms each venue-routed minute store up, asset -by asset, then keeps them all fresh, and an asset is served as soon as its own warm-up is done. Each -request slices the trailing `context_days` context — a crypto-1h one topped up with the bars that -closed since the last refresh — and adapts the champion's output to the live validator contract. A -second thread refreshes the event calendar at startup and daily, and the champion finds it at -`$SYNTH_BENCHMARK_CALENDAR`, as in its campaign. A calendar already stored on the volume, no older -than `CALENDAR_MAX_AGE_SECONDS`, is served until the first refresh replaces it. +by asset, then keeps them all fresh, and an asset is served as soon as its own warm-up is done; its +today's partition is refreshed after each later asset's warm-up. Each request slices the trailing +`context_days` context — a crypto-1h one topped up with the bars that closed since the last refresh — +and adapts the champion's output to the live validator contract. A second thread refreshes the event +calendar at startup and daily, and the champion finds it at `$SYNTH_BENCHMARK_CALENDAR`, as in its +campaign. A calendar already stored on the volume, no older than `CALENDAR_MAX_AGE_SECONDS`, is +served until the first refresh replaces it. No guards: an unservable asset, a data hole, or an exploding path crashes the request so it shows up in monitoring instead of silently degrading. @@ -105,16 +106,22 @@ def _warm_up_then_refresh(self) -> None: for asset in self._stores: warm_up([asset], self.context_days) self._calendar_ready.wait(max(0.0, calendar_deadline - time.monotonic())) + # The assets already served stay fresh while the others warm up. + self._refresh(list(self._ready), days=1) self._ready.add(asset) logger.info("warm-up complete; refreshing every %d s", REFRESH_SECONDS) self._background_refresh() def _refresh_once(self) -> None: - """One pass over every store, over the context's date partitions. A venue outage must not stop - the others being refreshed.""" - for asset, store in self._stores.items(): + """One pass over every store, over the context's date partitions.""" + self._refresh(list(self._stores), self.context_days + 1) + + def _refresh(self, assets: list[str], days: int) -> None: + """refresh_recent(days) on each asset's store, then marks the refresh for the cached contexts. A + venue outage must not stop the others being refreshed.""" + for asset in assets: try: - store.refresh_recent(days=self.context_days + 1) + self._stores[asset].refresh_recent(days=days) except Exception as exc: logger.warning("refresh failed for %s: %s", asset, exc) self._contexts.store_refreshed() diff --git a/tests/serving/test_champion_miner_refresh.py b/tests/serving/test_champion_miner_refresh.py index 88cbb46..8781b0b 100644 --- a/tests/serving/test_champion_miner_refresh.py +++ b/tests/serving/test_champion_miner_refresh.py @@ -6,9 +6,9 @@ refresh interval. This pins the cadence and what each pass asks the venue for. """ -from datetime import datetime, timedelta, timezone - import logging +import threading +from datetime import datetime, timedelta, timezone import pandas as pd import pytest @@ -97,6 +97,28 @@ def refresh_recent(self, days): assert ok.refreshed == [DEFAULT_CONTEXT_DAYS + 1] +def test_the_assets_already_served_are_refreshed_while_the_others_warm_up(monkeypatch): + """Each asset's warm-up is followed by a refresh of today's partition of the assets ready before it, + never of one still warming up.""" + warmed: list[str] = [] + monkeypatch.setattr("synth_lib.serving.champion_miner.warm_up", lambda assets, context_days: warmed.extend(assets)) + monkeypatch.setattr(ChampionMiner, "_background_refresh", lambda self: None) + + class _Broken(_Store): + def refresh_recent(self, days): + raise RuntimeError("venue down") + + stores = {"XAU": _Broken(), "NVDAX": _Store(), "BTC": _Store(), "HYPE": _Store()} + miner = _miner(stores) + miner._ready = set() + miner._calendar_ready = threading.Event() + miner._calendar_ready.set() + miner._warm_up_then_refresh() + assert warmed == ["XAU", "NVDAX", "BTC", "HYPE"] + assert (stores["NVDAX"].refreshed, stores["BTC"].refreshed, stores["HYPE"].refreshed) == ([1, 1], [1], []) + assert miner._ready == set(stores) + + def test_after_the_warm_up_a_pass_fetches_only_today(tmp_path, midday): """Settled days are never asked for again, so the pass is cheap enough to run every minute.""" venue = _Venue() From 552aa22d382610930a289dda7346ebaabaefe1bf Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 19:49:26 +0100 Subject: [PATCH 10/13] Price store: the day a venue holds only in part is stored unsettled, once Hyperliquid serves its last 5000 minutes, so the oldest day a refresh asks it for can start before that edge and come back partial. ingest_day stored it as settled, so neither the miner nor a later archive ingest, which skips settled partitions, ever completed it. A past day that starts before the client's retention_minutes is now stored with is_final False, and once a partition for it exists it is not fetched again (unless force_refresh), so the per-minute refresh does not ask for it every pass. The stored > fresh guard is unchanged, and a client without retention (Binance) is too. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- README.md | 4 +++- synth_lib/preparation/minute_price_store.py | 14 ++++++++--- .../test_store_never_loses_prices.py | 23 +++++++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7bc5545..2769084 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,9 @@ uv run synth_lib/preparation/market_data.py --days 7 `--days N` downloads today and the N days before it (N + 1 daily partitions). Today's partition is marked `is_final=False` and is re-downloaded on every -subsequent run. +subsequent run. A past day that starts before the venue's retention (Hyperliquid +keeps its last 5000 minutes) comes back partial: it is stored once with +`is_final=False`, for an archive ingest to complete, and not downloaded again. ### Re-download existing partitions diff --git a/synth_lib/preparation/minute_price_store.py b/synth_lib/preparation/minute_price_store.py index a739f46..ac65a19 100644 --- a/synth_lib/preparation/minute_price_store.py +++ b/synth_lib/preparation/minute_price_store.py @@ -90,15 +90,23 @@ def ingest_range( return paths def ingest_day(self, day: date, force_refresh: bool = False) -> Path: - """Fetch and persist one day of minute prices.""" + """Fetch and persist one day of minute prices. + + A past day that starts before the client's retention comes back partial: it is stored unsettled, + for an archive ingest to complete, and not fetched again once stored.""" self.ensure_root() path = self.day_path(day) - today = datetime.now(tz=UTC).date() + now = datetime.now(tz=UTC) + today = now.date() is_final = day < today if path.exists() and is_final and not force_refresh and _settled(path): return path day_start = datetime.combine(day, time.min, tzinfo=UTC) + retention = getattr(self.client, "retention_minutes", None) + partly_retained = is_final and retention is not None and day_start < now - timedelta(minutes=retention) + if partly_retained and path.exists() and not force_refresh: + return path day_end = datetime.combine(day, time.max, tzinfo=UTC).replace(second=0, microsecond=0) if not is_final: # The current day is still running: clamp the window to the last settled minute rather @@ -127,7 +135,7 @@ def ingest_day(self, day: date, force_refresh: bool = False) -> Path: frame["source"] = getattr(self.client, "source_name", "unknown") frame["ingested_at"] = datetime.now(tz=UTC).replace(microsecond=0) # An empty fetch of a past day stays unsettled, so the next ingest asks again. - frame["is_final"] = bool(is_final) and not fetched.empty + frame["is_final"] = bool(is_final) and not fetched.empty and not partly_retained frame = frame.reset_index(names="timestamp") # A force-refresh must never trade real prices for nothing. Venues serve a limited window # of history — Hyperliquid about 5000 minutes — so re-fetching a day that has aged out diff --git a/tests/preparation/test_store_never_loses_prices.py b/tests/preparation/test_store_never_loses_prices.py index 7d7a818..e30229e 100644 --- a/tests/preparation/test_store_never_loses_prices.py +++ b/tests/preparation/test_store_never_loses_prices.py @@ -101,6 +101,29 @@ def now(cls, tz=None): assert client.calls == [DAY, DAY], "a settled day was fetched again" +@pytest.mark.parametrize("retention_minutes, settled", [(5000, False), (None, True)]) +def test_a_day_the_venue_holds_only_in_part_is_stored_unsettled_and_fetched_once( + tmp_path, monkeypatch, retention_minutes, settled +): + """At 15:00 Hyperliquid's ~5000 minutes start at 03:40 three days back: that day comes back partial + and stays unsettled, without a refetch on every refresh. Binance keeps everything and settles it.""" + + class Clock(datetime): + @classmethod + def now(cls, tz=None): + return datetime(2026, 9, 28, 15, 0, tzinfo=timezone.utc) + + monkeypatch.setattr("synth_lib.preparation.minute_price_store.datetime", Clock) + client = _Client(rows=1220) + client.retention_minutes = retention_minutes + store = _store(tmp_path, client) + store.refresh_recent(days=3) + assert bool(pd.read_parquet(store.day_path(DAY))["is_final"].all()) is settled + assert bool(pd.read_parquet(store.day_path(DAY + timedelta(days=1)))["is_final"].all()) + store.refresh_recent(days=3) + assert client.calls.count(DAY) == 1 + + def test_an_empty_fetch_of_a_past_day_is_asked_again(tmp_path): client = _Client(rows=0) store = _store(tmp_path, client) From 8c5471c34a79da6232ab5a9ac0bb3b8fc553777b Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 19:49:52 +0100 Subject: [PATCH 11/13] Price store: each day partition is written atomically ingest_day writes the partition through write_atomic, a temporary file in the same directory replaced into place, so a write cut short (a killed pod mid-refresh) leaves the previous partition whole instead of a truncated file that later reads take for the day. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- synth_lib/preparation/minute_price_store.py | 3 ++- .../preparation/test_store_never_loses_prices.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/synth_lib/preparation/minute_price_store.py b/synth_lib/preparation/minute_price_store.py index ac65a19..cc00f30 100644 --- a/synth_lib/preparation/minute_price_store.py +++ b/synth_lib/preparation/minute_price_store.py @@ -18,6 +18,7 @@ legacy_partition_error, utc_datetime, ) +from synth_lib.preparation.event_calendar.events import write_atomic from synth_lib.preparation.price_client import PriceClient, build_price_client logger = logging.getLogger(__name__) @@ -152,7 +153,7 @@ def ingest_day(self, day: date, force_refresh: bool = False) -> Path: fresh, ) return path - frame.to_parquet(path, index=False) + write_atomic(frame, path) return path def refresh_recent(self, days: int = 8) -> list[Path]: diff --git a/tests/preparation/test_store_never_loses_prices.py b/tests/preparation/test_store_never_loses_prices.py index e30229e..1e9de70 100644 --- a/tests/preparation/test_store_never_loses_prices.py +++ b/tests/preparation/test_store_never_loses_prices.py @@ -10,6 +10,7 @@ """ from datetime import date, datetime, timedelta, timezone +from pathlib import Path import pandas as pd import pytest @@ -132,6 +133,21 @@ def test_an_empty_fetch_of_a_past_day_is_asked_again(tmp_path): assert _closes(store.ingest_day(DAY)) == MINUTES_PER_DAY +def test_an_interrupted_write_leaves_the_stored_partition_whole(tmp_path, monkeypatch): + store = _store(tmp_path, _Client(rows=100)) + path = store.ingest_day(DAY, force_refresh=True) + + def cut_short(self, target, **kwargs): + Path(target).write_bytes(b"PAR1") + raise KeyboardInterrupt + + monkeypatch.setattr(pd.DataFrame, "to_parquet", cut_short) + with pytest.raises(KeyboardInterrupt): + _store(tmp_path, _Client(rows=MINUTES_PER_DAY)).ingest_day(DAY, force_refresh=True) + assert _closes(path) == 100 + assert [p.name for p in path.parent.iterdir()] == [path.name] + + def test_a_day_of_zero_trade_candles_is_not_replaced_by_a_shorter_refetch(tmp_path): """A flat zero-trade candle is a price: it counts on both sides of the comparison.""" path = _store(tmp_path, _Client(rows=MINUTES_PER_DAY)).ingest_day(DAY, force_refresh=True) From 24745ea70cb22423140a01ad9b23c34c76105c0a Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 19:50:19 +0100 Subject: [PATCH 12/13] Serving: say which days a warm-up and a refresh pass cover refresh_recent(days=D + 1) covers today-(D + 1) through today, the context's D + 1 date partitions and the day before them. The warm_up and _refresh_once docstrings and the refresh test's name said D + 1 partitions. warm_up's docstring also says the oldest day Hyperliquid holds in part is stored unsettled. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- synth_lib/serving/champion_miner.py | 3 ++- synth_lib/serving/serve.py | 9 +++++---- tests/serving/test_champion_miner_refresh.py | 6 +++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/synth_lib/serving/champion_miner.py b/synth_lib/serving/champion_miner.py index 06448d2..cd4bfce 100644 --- a/synth_lib/serving/champion_miner.py +++ b/synth_lib/serving/champion_miner.py @@ -113,7 +113,8 @@ def _warm_up_then_refresh(self) -> None: self._background_refresh() def _refresh_once(self) -> None: - """One pass over every store, over the context's date partitions.""" + """One pass over every store: refresh_recent(days=context_days + 1) covers today-(context_days + 1) + through today, and fetches only today and the days not yet settled.""" self._refresh(list(self._stores), self.context_days + 1) def _refresh(self, assets: list[str], days: int) -> None: diff --git a/synth_lib/serving/serve.py b/synth_lib/serving/serve.py index a73d837..348e0b5 100644 --- a/synth_lib/serving/serve.py +++ b/synth_lib/serving/serve.py @@ -94,10 +94,11 @@ def warm_up(assets: Sequence[str], context_days: int) -> None: """Fill the local minute store from each asset's own venue before serving, and log how much of the champion's context holds a real close. - A `context_days` context touches context_days + 1 date partitions. Like the background refresh, - the fetch stops at the venue's retention: Hyperliquid's ~3.5 days cap how far back HL-routed - assets reach, and their older days accumulate forward (see the module docstring). Idempotent - across restarts: settled partitions are not fetched again. + refresh_recent(days=context_days + 1) covers today-(context_days + 1) through today: the context's + context_days + 1 date partitions and the day before them. Like the background refresh, the fetch + stops at the venue's retention: Hyperliquid's ~3.5 days cap how far back HL-routed assets reach, + the oldest day it holds in part is stored unsettled, and their older days accumulate forward (see + the module docstring). Idempotent across restarts: settled partitions are not fetched again. """ for asset in assets: try: diff --git a/tests/serving/test_champion_miner_refresh.py b/tests/serving/test_champion_miner_refresh.py index 8781b0b..fb38f67 100644 --- a/tests/serving/test_champion_miner_refresh.py +++ b/tests/serving/test_champion_miner_refresh.py @@ -64,9 +64,9 @@ def _miner(stores, miner_class=ChampionMiner): return miner -def test_each_pass_refreshes_the_contexts_date_partitions(): - """A D-day context touches D + 1 date partitions; a miner.py that does not set context_days - serves the default.""" +def test_each_pass_covers_today_and_the_d_plus_one_days_before_it(): + """refresh_recent(days=D + 1) covers today-(D + 1) through today; a miner.py that does not set + context_days serves the default.""" store = _Store() _miner({"BTC": store})._refresh_once() assert store.refreshed == [DEFAULT_CONTEXT_DAYS + 1] From d1a92d2766c6a22481f1566bfca648a949bd097f Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 20:16:49 +0100 Subject: [PATCH 13/13] write_atomic: the written file is readable by others, as a plain write is mkstemp creates its file owner-only, so every atomically written partition was 0600. It is now 0644. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- synth_lib/preparation/event_calendar/events.py | 2 ++ tests/preparation/test_store_never_loses_prices.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/synth_lib/preparation/event_calendar/events.py b/synth_lib/preparation/event_calendar/events.py index 378e6fa..b150a29 100644 --- a/synth_lib/preparation/event_calendar/events.py +++ b/synth_lib/preparation/event_calendar/events.py @@ -88,6 +88,8 @@ def write_atomic(frame: pd.DataFrame, path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp") os.close(fd) + # mkstemp creates the file owner-only; the written file is readable like any other. + os.chmod(tmp, 0o644) try: frame.to_parquet(tmp, index=False) os.replace(tmp, path) diff --git a/tests/preparation/test_store_never_loses_prices.py b/tests/preparation/test_store_never_loses_prices.py index 1e9de70..a936a53 100644 --- a/tests/preparation/test_store_never_loses_prices.py +++ b/tests/preparation/test_store_never_loses_prices.py @@ -148,6 +148,11 @@ def cut_short(self, target, **kwargs): assert [p.name for p in path.parent.iterdir()] == [path.name] +def test_a_partition_is_readable_by_others(tmp_path): + path = _store(tmp_path, _Client(rows=100)).ingest_day(DAY, force_refresh=True) + assert path.stat().st_mode & 0o777 == 0o644 + + def test_a_day_of_zero_trade_candles_is_not_replaced_by_a_shorter_refetch(tmp_path): """A flat zero-trade candle is a price: it counts on both sides of the comparison.""" path = _store(tmp_path, _Client(rows=MINUTES_PER_DAY)).ingest_day(DAY, force_refresh=True)