from __future__ import annotations

from datetime import datetime
from typing import Any, cast

import numpy as np
import polars as pl

from uforia.analytics.options_alpha import (
    build_hedge_route_frame,
    rolling_corr_expr,
    session_bucket_expr,
)
from uforia.analytics.surface import (
    TARGET_MATURITIES,
    _closest_maturity,
    _ssvi_total_variance,
    compute_surface_features,
    fit_surface,
)
from uforia.config import get_config
from uforia.storage import DATASET_SPECS, DatasetName, DatasetStore
from uforia.storage.datasets import empty_frame

SURFACE_SOURCE = "surface_monitor"
CARRY_SOURCE = "carry_vrp"
SKEW_SOURCE = "skew_crash_premium"
DISLOCATION_SOURCE = "surface_dislocation"
RELATIVE_VALUE_SOURCE = "btc_eth"
POSITIONING_SOURCE = "positioning_flow"
SURFACE_FACTOR_SOURCE = "surface_factor_rv"
ML_CARRY_SOURCE = "ml_carry_toxicity"
EXPIRY_FLOW_SOURCE = "expiry_flow_map"
ADAPTIVE_HEDGE_SOURCE = "adaptive_hedging"
SESSIONALITY_SOURCE = "sessionality_clocks"
DISPERSION_SOURCE = "btc_eth_dispersion"
FUNDING_SKEW_SOURCE = "funding_basis_skew"
OPTIONS_SIGNAL_SOURCE = "options_composite"
VOL_CONE_SOURCE = "vol_cone"
TERM_MOMENTUM_SOURCE = "term_structure_momentum"
GAMMA_EXPOSURE_SOURCE = "gamma_exposure"
FUNDING_CONTEXT_SOURCE = "funding_context"
STRADDLE_BREAKEVEN_SOURCE = "straddle_breakeven"
VOL_REGIME_TRANSITION_SOURCE = "vol_regime_transitions"
_STORAGE_METADATA_COLUMNS = {"run_id", "run_ts", "logical_ts", "write_mode"}


def build_options_surface_snapshot(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    options = _read_options(store, start=start, end=end, underlying=underlying)
    if options.is_empty():
        return empty_frame(DatasetName.OPTIONS_SURFACE_SNAPSHOT)
    frame = (
        options.filter(pl.col("forward").is_not_null() & pl.col("mark_iv").is_not_null())
        .with_columns(
            _tenor_bucket_expr().alias("tenor_bucket"),
            ((pl.col("strike") / pl.col("forward")) - 1.0).alias("moneyness"),
            (pl.col("ask") - pl.col("bid")).alias("spread"),
        )
        .with_columns(_moneyness_bucket_expr(pl.col("moneyness")).alias("moneyness_bucket"))
        .group_by(["ts", "date", "underlying", "tenor_bucket", "moneyness_bucket"])
        .agg(
            pl.col("mark_iv").mean().alias("mean_mark_iv"),
            pl.col("mid").mean().alias("mean_mid"),
            pl.col("spread").mean().alias("mean_spread"),
            pl.len().alias("option_count"),
            pl.col("open_interest").sum().alias("open_interest"),
            pl.col("volume").sum().alias("volume"),
            pl.col("quality_flag").sort().last().alias("quality_flag"),
        )
        .with_columns(pl.lit(SURFACE_SOURCE).alias("source"))
        .sort(["underlying", "tenor_bucket", "moneyness_bucket", "ts"])
        .with_columns(
            pl.col("mean_mark_iv")
            .diff()
            .over(["underlying", "source", "tenor_bucket", "moneyness_bucket"])
            .alias("iv_change_1h"),
            pl.col("mean_mark_iv")
            .diff(24)
            .over(["underlying", "source", "tenor_bucket", "moneyness_bucket"])
            .alias("iv_change_24h"),
            pl.col("mean_mark_iv")
            .diff(24 * 7)
            .over(["underlying", "source", "tenor_bucket", "moneyness_bucket"])
            .alias("iv_change_7d"),
        )
    )
    return frame.select(_business_columns(DatasetName.OPTIONS_SURFACE_SNAPSHOT))


def build_options_carry_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    surface = _read_surface(store, start=start, end=end, underlying=underlying)
    if surface.is_empty():
        return empty_frame(DatasetName.OPTIONS_CARRY_SERIES)
    realized = _realized_vol_frame(store, start=start, end=end, underlying=underlying)
    frame = (
        surface.join(realized, on="ts", how="left", coalesce=True)
        .sort("ts")
        .with_columns(
            (pl.col("atm_iv_7d") - pl.col("realized_vol_7d")).alias("carry_spread_7d"),
            (pl.col("atm_iv_30d") - pl.col("realized_vol_30d")).alias("carry_spread_30d"),
            (pl.col("atm_iv_7d") - pl.col("atm_iv_30d")).alias("term_carry_7_30"),
            (pl.col("atm_iv_14d") - pl.col("atm_iv_7d")).alias("roll_down_carry"),
            _rolling_zscore_expr("atm_iv_7d", "realized_vol_7d").alias("carry_zscore"),
        )
        .with_columns(
            pl.when(pl.col("carry_zscore") >= 1.0)
            .then(pl.lit("rich"))
            .when(pl.col("carry_zscore") <= -1.0)
            .then(pl.lit("cheap"))
            .otherwise(pl.lit("neutral"))
            .alias("carry_regime"),
            pl.lit(CARRY_SOURCE).alias("source"),
            pl.when(pl.col("realized_vol_7d").is_null())
            .then(pl.lit("insufficient_data"))
            .otherwise(pl.lit("ok"))
            .alias("quality_flag"),
        )
    )
    return frame.select(_business_columns(DatasetName.OPTIONS_CARRY_SERIES))


def build_options_skew_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    surface = _read_surface(store, start=start, end=end, underlying=underlying)
    if surface.is_empty():
        return empty_frame(DatasetName.OPTIONS_SKEW_SERIES)
    spot = _spot_frame(store, start=start, end=end, underlying=underlying)
    frame = (
        surface.join(spot.select(["ts", "spot_move_24h"]), on="ts", how="left")
        .sort("ts")
        .with_columns(
            (pl.col("rr_25_7d") - pl.col("rr_25_30d")).alias("skew_slope"),
            pl.col("rr_25_7d").diff().alias("skew_change_1h"),
            _rolling_single_zscore_expr("rr_25_7d").alias("skew_zscore"),
        )
        .with_columns(
            pl.when(pl.col("skew_zscore") <= -1.0)
            .then(pl.lit("panic"))
            .when(pl.col("skew_zscore") >= 1.0)
            .then(pl.lit("unwind"))
            .otherwise(pl.lit("neutral"))
            .alias("crash_premium_regime"),
            pl.lit(SKEW_SOURCE).alias("source"),
            pl.when(pl.col("rr_25_7d").is_null())
            .then(pl.lit("insufficient_data"))
            .otherwise(pl.lit("ok"))
            .alias("quality_flag"),
        )
    )
    return frame.select(_business_columns(DatasetName.OPTIONS_SKEW_SERIES))


def build_options_dislocation_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    options = _read_options(store, start=start, end=end, underlying=underlying)
    if options.is_empty():
        return empty_frame(DatasetName.OPTIONS_DISLOCATION_SERIES)
    rows: list[dict[str, Any]] = []
    for ts_key, snapshot in options.partition_by("ts", maintain_order=True, as_dict=True).items():
        ts = ts_key[0] if isinstance(ts_key, tuple) else ts_key
        try:
            fit = fit_surface(snapshot)
        except ValueError:
            continue
        expiries = (
            snapshot.select(
                [
                    "expiry",
                    ((pl.col("expiry") - pl.lit(ts)).dt.total_seconds() / 86400.0).alias(
                        "days_to_expiry"
                    ),
                ]
            )
            .unique("expiry")
            .filter(pl.col("days_to_expiry") > 0)
            .sort("days_to_expiry")
        )
        for target in TARGET_MATURITIES:
            params = fit.parameters.get(target)
            if params is None:
                continue
            maturity_row = _closest_maturity(expiries, target)
            if maturity_row is None:
                continue
            tenor_snapshot = snapshot.filter(pl.col("expiry") == maturity_row["expiry"]).drop_nulls(
                ["mark_iv", "strike", "forward"]
            )
            if tenor_snapshot.height < 5:
                continue
            maturity = max(float(cast(float, maturity_row["days_to_expiry"])) / 365.0, 1 / 365.0)
            forward = float(cast(float, tenor_snapshot["forward"].median()))
            moneyness = (tenor_snapshot["strike"].to_numpy() / forward) - 1.0
            log_moneyness = np.log(tenor_snapshot["strike"].to_numpy() / forward)
            fitted_total = _ssvi_total_variance(log_moneyness, *params)
            fitted_iv = np.sqrt(np.maximum(fitted_total / maturity, 0.0))
            residual = tenor_snapshot["mark_iv"].to_numpy() - fitted_iv
            abs_residual = np.abs(residual)
            atm_mask = np.abs(moneyness) <= 0.05
            wing_mask = np.abs(moneyness) >= 0.10
            rows.append(
                {
                    "ts": ts,
                    "date": ts.date(),
                    "underlying": underlying,
                    "source": DISLOCATION_SOURCE,
                    "tenor_bucket": f"{target}d",
                    "atm_residual": (
                        float(abs_residual[atm_mask].mean()) if atm_mask.any() else None
                    ),
                    "wing_residual": (
                        float(abs_residual[wing_mask].mean()) if wing_mask.any() else None
                    ),
                    "max_residual": float(abs_residual.max()) if len(abs_residual) else None,
                    "mean_abs_residual": float(abs_residual.mean()) if len(abs_residual) else None,
                    "dispersion_score": float(abs_residual.std()) if len(abs_residual) else None,
                    "residual_p90": (
                        float(np.quantile(abs_residual, 0.9)) if len(abs_residual) else None
                    ),
                    "rich_strike_count": int((residual > 0.02).sum()),
                    "cheap_strike_count": int((residual < -0.02).sum()),
                    "fit_validity_flag": fit.fit_validity_flag,
                    "quality_flag": (
                        "bad_fit"
                        if fit.fit_validity_flag != "ok"
                        else ("ok" if fit.coverage_ratio >= 0.25 else "low_coverage")
                    ),
                }
            )
    if not rows:
        return empty_frame(DatasetName.OPTIONS_DISLOCATION_SERIES)
    frame = (
        pl.DataFrame(rows)
        .sort(["underlying", "source", "tenor_bucket", "ts"])
        .with_columns(
            _rolling_single_zscore_expr(
                "max_residual",
                group_cols=["underlying", "source", "tenor_bucket"],
            ).alias("dislocation_zscore")
        )
    )
    return frame.select(_business_columns(DatasetName.OPTIONS_DISLOCATION_SERIES))


def build_options_relative_value_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
) -> pl.DataFrame:
    btc_carry = store.read(
        DatasetName.OPTIONS_CARRY_SERIES,
        start=start,
        end=end,
        filters={"underlying": "BTC", "source": CARRY_SOURCE},
    ).select(
        [
            "ts",
            pl.col("atm_iv_7d").alias("btc_atm_iv_7d"),
            pl.col("atm_iv_30d").alias("btc_atm_iv_30d"),
            pl.col("carry_spread_7d").alias("btc_carry_spread_7d"),
        ]
    )
    eth_carry = store.read(
        DatasetName.OPTIONS_CARRY_SERIES,
        start=start,
        end=end,
        filters={"underlying": "ETH", "source": CARRY_SOURCE},
    ).select(
        [
            "ts",
            pl.col("atm_iv_7d").alias("eth_atm_iv_7d"),
            pl.col("atm_iv_30d").alias("eth_atm_iv_30d"),
            pl.col("carry_spread_7d").alias("eth_carry_spread_7d"),
        ]
    )
    btc_skew = store.read(
        DatasetName.OPTIONS_SKEW_SERIES,
        start=start,
        end=end,
        filters={"underlying": "BTC", "source": SKEW_SOURCE},
    ).select(["ts", pl.col("rr_25_30d").alias("btc_rr_25_30d")])
    eth_skew = store.read(
        DatasetName.OPTIONS_SKEW_SERIES,
        start=start,
        end=end,
        filters={"underlying": "ETH", "source": SKEW_SOURCE},
    ).select(["ts", pl.col("rr_25_30d").alias("eth_rr_25_30d")])
    btc_vvix = store.read(
        DatasetName.VVIX_SERIES,
        start=start,
        end=end,
        filters={"underlying": "BTC", "source": "dvol"},
    ).select(["ts", pl.col("rvvix_14d").alias("btc_rvvix_14d")])
    eth_vvix = store.read(
        DatasetName.VVIX_SERIES,
        start=start,
        end=end,
        filters={"underlying": "ETH", "source": "dvol"},
    ).select(["ts", pl.col("rvvix_14d").alias("eth_rvvix_14d")])
    btc_razor = store.read(
        DatasetName.RAZOR_SERIES,
        start=start,
        end=end,
        filters={"underlying": "BTC"},
    ).select(["ts", pl.col("razor_vrp").alias("btc_razor_vrp")])
    eth_razor = store.read(
        DatasetName.RAZOR_SERIES,
        start=start,
        end=end,
        filters={"underlying": "ETH"},
    ).select(["ts", pl.col("razor_vrp").alias("eth_razor_vrp")])
    btc_funding = store.read(
        DatasetName.FUNDING_CONTEXT_SERIES,
        start=start,
        end=end,
        filters={"underlying": "BTC", "source": FUNDING_CONTEXT_SOURCE},
    ).select(["ts", pl.col("funding_rate").alias("btc_funding_rate")])
    eth_funding = store.read(
        DatasetName.FUNDING_CONTEXT_SERIES,
        start=start,
        end=end,
        filters={"underlying": "ETH", "source": FUNDING_CONTEXT_SOURCE},
    ).select(["ts", pl.col("funding_rate").alias("eth_funding_rate")])
    mandatory_frames = [
        btc_carry,
        eth_carry,
        btc_skew,
        eth_skew,
        btc_vvix,
        eth_vvix,
        btc_razor,
        eth_razor,
    ]
    optional_frames = [btc_funding, eth_funding]
    if any(frame.is_empty() for frame in (btc_carry, eth_carry)):
        return empty_frame(DatasetName.OPTIONS_RELATIVE_VALUE_SERIES)
    joined = mandatory_frames[0]
    for frame in mandatory_frames[1:]:
        joined = joined.join(frame, on="ts", how="inner")
    for frame in optional_frames:
        if not frame.is_empty():
            joined = joined.join(frame, on="ts", how="left")
    if "btc_funding_rate" not in joined.columns:
        joined = joined.with_columns(pl.lit(None, dtype=pl.Float64).alias("btc_funding_rate"))
    if "eth_funding_rate" not in joined.columns:
        joined = joined.with_columns(pl.lit(None, dtype=pl.Float64).alias("eth_funding_rate"))
    if joined.is_empty():
        return empty_frame(DatasetName.OPTIONS_RELATIVE_VALUE_SERIES)
    result = (
        joined.sort("ts")
        .with_columns(
            (pl.col("btc_atm_iv_7d") - pl.col("eth_atm_iv_7d")).alias("atm_iv_spread_7d"),
            (pl.col("btc_atm_iv_30d") - pl.col("eth_atm_iv_30d")).alias("atm_iv_spread_30d"),
            (pl.col("btc_rr_25_30d") - pl.col("eth_rr_25_30d")).alias("skew_spread_30d"),
            (pl.col("btc_rvvix_14d") - pl.col("eth_rvvix_14d")).alias("vvix_spread"),
            (pl.col("btc_razor_vrp") - pl.col("eth_razor_vrp")).alias("razor_spread"),
            (pl.col("btc_carry_spread_7d") - pl.col("eth_carry_spread_7d")).alias("carry_spread"),
            (pl.col("btc_funding_rate") - pl.col("eth_funding_rate")).alias("funding_spread"),
        )
        .with_columns(
            _rolling_single_zscore_expr("atm_iv_spread_7d").alias("spread_zscore"),
            pl.lit(RELATIVE_VALUE_SOURCE).alias("source"),
            pl.lit("BTC_ETH").alias("pair_id"),
        )
        .with_columns(
            pl.when(pl.col("spread_zscore") >= 1.0)
            .then(pl.lit("btc_rich"))
            .when(pl.col("spread_zscore") <= -1.0)
            .then(pl.lit("eth_rich"))
            .otherwise(pl.lit("neutral"))
            .alias("relative_regime"),
            pl.lit("ok").alias("quality_flag"),
            pl.col("ts").dt.date().alias("date"),
        )
    )
    return result.select(_business_columns(DatasetName.OPTIONS_RELATIVE_VALUE_SERIES))


def build_options_positioning_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    options = _read_options(store, start=start, end=end, underlying=underlying)
    if options.is_empty():
        return empty_frame(DatasetName.OPTIONS_POSITIONING_SERIES)
    frame = (
        options.filter(pl.col("forward").is_not_null())
        .with_columns(
            _expiry_bucket_expr().alias("expiry_bucket"),
            ((pl.col("strike") / pl.col("forward")) - 1.0).alias("moneyness"),
        )
        .with_columns(_moneyness_bucket_expr(pl.col("moneyness")).alias("moneyness_bucket"))
        .group_by(["ts", "date", "underlying", "expiry_bucket", "moneyness_bucket"])
        .agg(
            pl.col("open_interest").sum().alias("open_interest"),
            pl.col("volume").sum().alias("volume"),
            pl.len().alias("option_count"),
        )
        .sort(["underlying", "expiry_bucket", "moneyness_bucket", "ts"])
        .with_columns(
            pl.col("open_interest")
            .diff()
            .over(["underlying", "expiry_bucket", "moneyness_bucket"])
            .alias("open_interest_change")
        )
    )
    totals = frame.group_by(["ts", "underlying"]).agg(
        pl.col("open_interest").sum().alias("total_open_interest"),
        pl.col("volume").sum().alias("total_volume"),
        pl.when(pl.col("moneyness_bucket") == "atm")
        .then(pl.col("open_interest"))
        .otherwise(0.0)
        .sum()
        .alias("atm_open_interest"),
        pl.when(
            pl.col("moneyness_bucket").is_in(
                ["put_wing", "call_wing", "deep_put", "deep_call"]
            )
        )
        .then(pl.col("open_interest"))
        .otherwise(0.0)
        .sum()
        .alias("wing_open_interest"),
    )
    result = (
        frame.join(totals, on=["ts", "underlying"], how="left")
        .with_columns(
            (pl.col("open_interest") / pl.col("total_open_interest")).alias("open_interest_share"),
            (pl.col("volume") / pl.col("total_volume")).alias("volume_share"),
            (pl.col("atm_open_interest") / pl.col("total_open_interest")).alias(
                "atm_pinning_concentration"
            ),
            (pl.col("wing_open_interest") / pl.col("total_open_interest")).alias(
                "wing_concentration"
            ),
            pl.lit(POSITIONING_SOURCE).alias("source"),
            pl.lit("ok").alias("quality_flag"),
        )
    )
    return result.select(_business_columns(DatasetName.OPTIONS_POSITIONING_SERIES))


def build_options_surface_factor_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    surface_snapshot = store.read(
        DatasetName.OPTIONS_SURFACE_SNAPSHOT,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": SURFACE_SOURCE},
    )
    if surface_snapshot.is_empty():
        surface_snapshot = build_options_surface_snapshot(
            store,
            start=start,
            end=end,
            underlying=underlying,
        )
    if surface_snapshot.is_empty():
        return empty_frame(DatasetName.OPTIONS_SURFACE_FACTOR_SERIES)
    frame = (
        surface_snapshot.sort(["tenor_bucket", "moneyness_bucket", "ts"])
        .with_columns(
            (pl.col("mean_mark_iv") / pl.col("mean_mark_iv").shift(1) - 1.0)
            .over(["tenor_bucket", "moneyness_bucket"])
            .alias("mean_node_return_1h"),
            (
                (pl.col("open_interest").fill_null(0.0) + pl.col("volume").fill_null(0.0))
                / (1.0 + pl.col("mean_spread").fill_null(0.0) * 100.0)
            ).alias("liquidity_weight_raw"),
        )
    )
    factor = (
        frame.group_by("ts")
        .agg(
            (
                (pl.col("mean_node_return_1h").fill_null(0.0) * pl.col("liquidity_weight_raw"))
                .sum()
                / (pl.col("liquidity_weight_raw").sum() + 1e-9)
            ).alias("factor_return_1h"),
        )
        .sort("ts")
        .with_columns(
            pl.col("factor_return_1h")
            .rolling_mean(window_size=24, min_samples=6)
            .alias("factor_momentum_24h")
        )
    )
    result = (
        frame.join(factor, on="ts", how="left")
        .with_columns(
            (
                pl.col("liquidity_weight_raw")
                / (pl.col("liquidity_weight_raw").sum().over("ts") + 1e-9)
            ).alias("liquidity_weight"),
            (pl.col("mean_node_return_1h") - pl.col("factor_return_1h")).alias("node_residual"),
        )
        .sort(["underlying", "tenor_bucket", "moneyness_bucket", "ts"])
        .with_columns(
            _rolling_single_zscore_expr(
                "node_residual",
                group_cols=["underlying", "tenor_bucket", "moneyness_bucket"],
                window=24 * 14,
            ).alias("node_residual_zscore")
        )
        .with_columns(
            (-(pl.col("node_residual_zscore")) * (0.5 + pl.col("liquidity_weight"))).alias(
                "residual_value_score"
            ),
            pl.when(pl.col("factor_momentum_24h") >= 0.005)
            .then(pl.lit("momentum_up"))
            .when(pl.col("factor_momentum_24h") <= -0.005)
            .then(pl.lit("momentum_down"))
            .otherwise(pl.lit("neutral"))
            .alias("factor_regime"),
            pl.lit(SURFACE_FACTOR_SOURCE).alias("source"),
            pl.when(pl.col("mean_node_return_1h").is_null())
            .then(pl.lit("insufficient_data"))
            .otherwise(pl.lit("ok"))
            .alias("quality_flag"),
        )
    )
    return result.select(_business_columns(DatasetName.OPTIONS_SURFACE_FACTOR_SERIES))


def build_options_ml_carry_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    carry = store.read(
        DatasetName.OPTIONS_CARRY_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": CARRY_SOURCE},
    )
    if carry.is_empty():
        carry = build_options_carry_series(store, start=start, end=end, underlying=underlying)
    if carry.is_empty():
        return empty_frame(DatasetName.OPTIONS_ML_CARRY_SERIES)
    surface = _read_surface(store, start=start, end=end, underlying=underlying).select(
        ["ts", "fit_quality", "rr_25_7d", "bf_25_7d"]
    )
    spot = _spot_frame(store, start=start, end=end, underlying=underlying).select(
        ["ts", "spot_move_24h", "underlying_price"]
    )
    funding = store.read(
        DatasetName.FUNDING_CONTEXT_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": FUNDING_CONTEXT_SOURCE},
    ).select(["ts", "funding_rate", "funding_rate_zscore"])
    perps_composite = store.read(
        DatasetName.PERPS_COMPOSITE_SERIES,
        start=start,
        end=end,
        filters={"asset": underlying, "venue_scope": "all"},
    ).select(["ts", "toxicity_score"])
    surface_snapshot = store.read(
        DatasetName.OPTIONS_SURFACE_SNAPSHOT,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": SURFACE_SOURCE},
    )
    spreads = (
        surface_snapshot.group_by("ts")
        .agg(pl.col("mean_spread").mean().alias("spread_cost"))
        .sort("ts")
    )
    realized = _realized_vol_frame(store, start=start, end=end, underlying=underlying)
    forecast = realized.with_columns(
        (
            0.6 * pl.col("realized_vol_7d")
            + 0.3 * pl.col("realized_vol_14d")
            + 0.1 * pl.col("realized_vol_30d")
        ).alias("forecast_rv_7d")
    ).select(["ts", "forecast_rv_7d"])
    frame = carry.select(
        ["ts", "date", "underlying", "atm_iv_7d", "carry_zscore", "carry_spread_7d"]
    )
    for next_frame in (surface, spot, funding, perps_composite, spreads, forecast):
        if not next_frame.is_empty():
            frame = frame.join(next_frame, on="ts", how="left")
    ml_missing_defaults: dict[str, Any] = {
        "funding_rate": 0.0,
        "funding_rate_zscore": 0.0,
        "toxicity_score": None,
        "spread_cost": 0.0,
        "spot_move_24h": 0.0,
        "rr_25_7d": 0.0,
        "bf_25_7d": 0.0,
        "fit_quality": 0.002,
        "forecast_rv_7d": None,
    }
    absent_ml = [column for column in ml_missing_defaults if column not in frame.columns]
    if absent_ml:
        frame = frame.with_columns(
            [
                pl.lit(ml_missing_defaults[column], dtype=pl.Float64).alias(column)
                for column in absent_ml
            ]
        )
    frame = frame.with_columns(
        pl.col("toxicity_score")
        .fill_null(pl.col("funding_rate_zscore").abs())
        .alias("toxicity_score"),
        (pl.col("atm_iv_7d") - pl.col("forecast_rv_7d")).alias("gross_carry_spread"),
        (
            (pl.col("spot_move_24h").abs() * 100.0)
            + pl.col("rr_25_7d").abs().fill_null(0.0) * 10.0
            + pl.col("bf_25_7d").fill_null(0.0) * 5.0
        ).alias("jump_risk_score"),
        (
            pl.col("spread_cost").fill_null(0.0) * 10_000.0
            + pl.col("atm_iv_7d").fill_null(0.0) * 50.0
        ).alias("expected_hedge_cost"),
        (pl.col("funding_rate").fill_null(0.0).abs() * 10_000.0).alias("funding_drag"),
    ).with_columns(
        _rolling_single_zscore_expr("gross_carry_spread", window=24 * 14).alias(
            "gross_carry_score"
        )
    ).with_columns(
        (
            pl.col("gross_carry_score")
            - (pl.col("toxicity_score").fill_null(0.0) * 0.75)
            - (pl.col("jump_risk_score").fill_null(0.0) * 0.25)
            - (pl.col("expected_hedge_cost").fill_null(0.0) / 100.0)
            - (pl.col("spread_cost").fill_null(0.0) * 100.0)
            - (pl.col("funding_drag").fill_null(0.0) / 100.0)
        ).alias("net_carry_score"),
        (
            1.0
            - pl.col("fit_quality").fill_null(0.002).clip(0.0, 0.02) / 0.02
        ).clip(0.0, 1.0).alias("carry_confidence"),
    ).with_columns(
        (
            (pl.col("toxicity_score").fill_null(0.0) > 1.5)
            | (pl.col("jump_risk_score").fill_null(0.0) > 4.0)
            | (pl.col("net_carry_score").fill_null(-999.0) < 0.0)
        ).alias("carry_veto"),
        pl.lit(ML_CARRY_SOURCE).alias("source"),
        pl.when(pl.col("forecast_rv_7d").is_null())
        .then(pl.lit("insufficient_data"))
        .otherwise(pl.lit("ok"))
        .alias("quality_flag"),
    )
    return frame.select(_business_columns(DatasetName.OPTIONS_ML_CARRY_SERIES))


def build_options_expiry_flow_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    options = _read_options(store, start=start, end=end, underlying=underlying)
    if options.is_empty():
        return empty_frame(DatasetName.OPTIONS_EXPIRY_FLOW_SERIES)
    enriched = (
        options.filter(pl.col("forward").is_not_null())
        .with_columns(
            _expiry_bucket_expr().alias("expiry_bucket"),
            (((pl.col("strike") / pl.col("forward")) - 1.0)).alias("moneyness"),
        )
        .with_columns(_strike_bucket_expr(pl.col("moneyness")).alias("strike_bucket"))
        .with_columns(
            (
                pl.when(pl.col("gamma").is_not_null())
                .then(
                    pl.col("gamma")
                    * pl.col("open_interest").fill_null(0.0)
                    * pl.col("underlying_price").fill_null(0.0) ** 2
                )
                .otherwise(None)
            ).alias("gamma_exposure_raw"),
            (
                pl.when(pl.col("vega").is_not_null())
                .then(
                    pl.col("vega")
                    * pl.col("open_interest").fill_null(0.0)
                    * pl.col("moneyness").fill_null(0.0)
                )
                .otherwise(None)
            ).alias("vanna_exposure_raw"),
            (
                pl.when(pl.col("delta").is_not_null())
                .then(
                    pl.col("delta")
                    * pl.col("open_interest").fill_null(0.0)
                    / (
                        ((pl.col("expiry") - pl.col("ts")).dt.total_seconds() / 86400.0)
                        .clip(1.0, None)
                    )
                )
                .otherwise(None)
            ).alias("charm_exposure_raw"),
            pl.col("gamma").is_not_null().cast(pl.Int64).alias("gamma_present"),
            pl.col("vega").is_not_null().cast(pl.Int64).alias("vega_present"),
            pl.col("delta").is_not_null().cast(pl.Int64).alias("delta_present"),
        )
    )
    pain = (
        enriched.group_by(["ts", "underlying"])
        .agg(
            (
                (pl.col("strike") * pl.col("open_interest").fill_null(0.0)).sum()
                / (pl.col("open_interest").fill_null(0.0).sum() + 1e-9)
            ).alias("pain_strike"),
            pl.col("underlying_price").median().alias("underlying_price"),
        )
        .sort("ts")
    )
    dominant = (
        enriched.group_by(["ts", "expiry_bucket", "strike_bucket", "strike"])
        .agg(pl.col("open_interest").sum().alias("bucket_oi"))
        .sort(["ts", "expiry_bucket", "strike_bucket", "bucket_oi", "strike"])
        .group_by(["ts", "expiry_bucket", "strike_bucket"])
        .agg(pl.col("strike").tail(1).alias("dominant_strike"))
        .with_columns(pl.col("dominant_strike").list.first().alias("dominant_strike"))
    )
    grouped = (
        enriched.group_by(["ts", "date", "underlying", "expiry_bucket", "strike_bucket"])
        .agg(
            pl.len().alias("option_count"),
            pl.col("open_interest").sum().alias("bucket_open_interest"),
            pl.col("gamma_exposure_raw").sum().alias("gamma_exposure"),
            pl.col("vanna_exposure_raw").sum().alias("vanna_exposure"),
            pl.col("charm_exposure_raw").sum().alias("charm_exposure"),
            pl.col("gamma_present").sum().alias("gamma_count"),
            pl.col("vega_present").sum().alias("vega_count"),
            pl.col("delta_present").sum().alias("delta_count"),
        )
        .join(pain, on=["ts", "underlying"], how="left")
        .join(dominant, on=["ts", "expiry_bucket", "strike_bucket"], how="left")
    )
    totals = grouped.group_by(["ts", "underlying"]).agg(
        pl.col("bucket_open_interest").sum().alias("total_open_interest")
    )
    result = (
        grouped.join(totals, on=["ts", "underlying"], how="left")
        .with_columns(
            (pl.col("bucket_open_interest") / (pl.col("total_open_interest") + 1e-9)).alias(
                "local_oi_concentration"
            ),
            (
                (pl.col("underlying_price") - pl.col("pain_strike")).abs()
                / pl.col("underlying_price").clip(1e-9, None)
            ).alias("distance_to_pain_pct"),
            (
                pl.min_horizontal(
                    [
                        pl.col("gamma_count"),
                        pl.col("vega_count"),
                        pl.col("delta_count"),
                    ]
                )
                / pl.col("option_count").clip(1, None)
            ).alias("greek_coverage_ratio"),
        )
        .with_columns(
            pl.when(pl.col("gamma_count") > 0)
            .then(pl.col("gamma_exposure").fill_null(0.0).abs() * pl.col("local_oi_concentration"))
            .otherwise(None)
            .alias("pinning_score"),
            pl.when((pl.col("vega_count") > 0) & (pl.col("delta_count") > 0))
            .then(
                pl.col("vanna_exposure").fill_null(0.0).abs()
                + pl.col("charm_exposure").fill_null(0.0).abs()
                + (pl.col("distance_to_pain_pct").fill_null(0.0) * 100.0)
            )
            .otherwise(None)
            .alias("breakout_score"),
        )
        .with_columns(
            pl.when(pl.col("greek_coverage_ratio") <= 0.0)
            .then(pl.lit("insufficient_context"))
            .when(pl.col("pinning_score").is_not_null() & (pl.col("gamma_exposure") >= 0))
            .then(pl.lit("positive_gamma_pin"))
            .when(
                pl.col("breakout_score").is_not_null()
                & (
                    pl.col("pinning_score").is_null()
                    | (pl.col("breakout_score") > pl.col("pinning_score"))
                )
            )
            .then(pl.lit("negative_gamma_breakout"))
            .otherwise(pl.lit("neutral"))
            .alias("flow_regime"),
            pl.lit(EXPIRY_FLOW_SOURCE).alias("source"),
            pl.when(pl.col("greek_coverage_ratio") <= 0.0)
            .then(pl.lit("insufficient_data"))
            .when(pl.col("greek_coverage_ratio") < 1.0)
            .then(pl.lit("partial_context"))
            .otherwise(pl.lit("ok"))
            .alias("quality_flag"),
        )
    )
    return result.select(_business_columns(DatasetName.OPTIONS_EXPIRY_FLOW_SERIES))


def build_options_adaptive_hedge_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    options = _read_options(store, start=start, end=end, underlying=underlying)
    surface = _read_surface(store, start=start, end=end, underlying=underlying)
    realized = _realized_vol_frame(store, start=start, end=end, underlying=underlying)
    funding = store.read(
        DatasetName.FUNDING_CONTEXT_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": FUNDING_CONTEXT_SOURCE},
    ).select(["ts", "funding_rate"])
    perps = store.read(
        DatasetName.PERPS_SIGNAL_SERIES,
        start=start,
        end=end,
        filters={"asset": underlying, "venue_scope": "all"},
    ).select(["ts", "basis_bps"])
    route_frame = build_hedge_route_frame(store, start=start, end=end, asset=underlying)
    if options.is_empty() or surface.is_empty():
        return empty_frame(DatasetName.OPTIONS_ADAPTIVE_HEDGE_SERIES)
    atm = (
        options.filter(
            pl.col("forward").is_not_null()
            & (((pl.col("strike") / pl.col("forward")) - 1.0).abs() < 0.05)
        )
        .group_by(["ts", "date", "underlying"])
        .agg(
            pl.col("delta").mean().alias("atm_delta"),
            pl.col("gamma").mean().alias("atm_gamma"),
            pl.col("vega").mean().alias("atm_vega"),
            (pl.col("ask") - pl.col("bid")).mean().alias("mean_spread"),
        )
        .sort("ts")
    )
    frame = atm.join(
        surface.select(["ts", "rr_25_7d", "atm_iv_7d"]),
        on="ts",
        how="left",
    ).join(
        realized.select(["ts", "realized_vol_7d"]),
        on="ts",
        how="left",
    )
    for next_frame in (funding, perps):
        if not next_frame.is_empty():
            frame = frame.join(next_frame, on="ts", how="left")
    if not route_frame.is_empty():
        frame = frame.sort("ts").join_asof(route_frame.sort("ts"), on="ts", strategy="backward")
    if "funding_rate" not in frame.columns:
        frame = frame.with_columns(pl.lit(0.0).alias("funding_rate"))
    if "basis_bps" not in frame.columns:
        frame = frame.with_columns(pl.lit(0.0).alias("basis_bps"))
    route_defaults: dict[str, Any] = {
        "route_basis_bps": None,
        "micro_route_score": None,
        "route_preference": None,
    }
    missing_route = [column for column in route_defaults if column not in frame.columns]
    if missing_route:
        frame = frame.with_columns(
            [
                pl.lit(route_defaults[column], dtype=pl.Float64 if column != "route_preference" else pl.String).alias(column)
                for column in missing_route
            ]
        )
    frame = frame.with_columns(
        (
            pl.col("atm_delta").fill_null(0.0)
            + (pl.col("atm_vega").fill_null(0.0) * pl.col("rr_25_7d").fill_null(0.0))
        ).alias("smile_aware_delta"),
        (
            pl.col("atm_delta").fill_null(0.0)
            * (
                pl.col("realized_vol_7d").fill_null(pl.col("atm_iv_7d").fill_null(0.0))
                / (pl.col("atm_iv_7d").fill_null(1.0) + 1e-9)
            )
        ).alias("min_var_delta"),
        (
            (
                (pl.col("mean_spread").fill_null(0.001) * 10_000.0)
                / (
                    (pl.col("atm_gamma").fill_null(0.001).abs() ** 2)
                    * pl.col("realized_vol_7d").fill_null(0.25).clip(0.05, None)
                    * 10_000.0
                )
            )
            ** (1.0 / 3.0)
        ).alias("hedge_band_width"),
        pl.coalesce(
            [
                pl.col("route_basis_bps"),
                pl.col("basis_bps").abs(),
                pl.lit(0.0),
            ]
        ).alias("basis_mismatch_bps"),
        (pl.col("funding_rate").fill_null(0.0).abs() * 10_000.0).alias("funding_drag"),
        pl.coalesce(
            [
                pl.col("micro_route_score"),
                100.0 - pl.col("basis_bps").fill_null(0.0).abs(),
            ]
        ).alias("route_input_score"),
    ).with_columns(
        (
            pl.col("hedge_band_width")
            * pl.col("realized_vol_7d").fill_null(0.0)
            * 10_000.0
        ).alias("expected_hedge_bleed_bps"),
    ).with_columns(
        (
            pl.col("route_input_score").fill_null(100.0)
            - pl.col("expected_hedge_bleed_bps").fill_null(0.0)
            - (pl.col("funding_drag").fill_null(0.0) / 2.0)
        ).alias("route_score"),
        pl.coalesce(
            [
                pl.col("route_preference"),
                pl.when(pl.col("basis_bps").fill_null(0.0).abs() > 25.0)
                .then(pl.lit("nearby_future"))
                .otherwise(pl.lit("perp")),
            ]
        ).alias("route_preference"),
        pl.lit(ADAPTIVE_HEDGE_SOURCE).alias("source"),
        pl.when(pl.col("atm_gamma").is_null())
        .then(pl.lit("insufficient_data"))
        .otherwise(pl.lit("ok"))
        .alias("quality_flag"),
    )
    return frame.select(_business_columns(DatasetName.OPTIONS_ADAPTIVE_HEDGE_SERIES))


def build_options_sessionality_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    surface = _read_surface(store, start=start, end=end, underlying=underlying)
    carry = store.read(
        DatasetName.OPTIONS_CARRY_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": CARRY_SOURCE},
    )
    spot = _spot_frame(store, start=start, end=end, underlying=underlying)
    if surface.is_empty() or spot.is_empty():
        return empty_frame(DatasetName.OPTIONS_SESSIONALITY_SERIES)
    frame = (
        spot.join(surface.select(["ts", "atm_iv_7d"]), on="ts", how="left")
        .join(carry.select(["ts", "carry_spread_7d"]), on="ts", how="left")
        .sort("ts")
        .with_columns(
            pl.col("ts").dt.date().alias("date"),
            pl.lit(underlying).alias("underlying"),
            session_bucket_expr().alias("session_bucket"),
            pl.col("ts").dt.weekday().is_in([6, 7]).alias("is_weekend"),
            pl.col("ts").dt.hour().is_in([0, 8, 16]).alias("is_funding_window"),
            (
                (pl.col("ts").dt.weekday() == 5)
                & pl.col("ts").dt.hour().is_between(6, 10, closed="both")
            ).alias("is_expiry_window"),
            pl.col("underlying_price").log().diff().abs().alias("session_realized_vol"),
        )
        .with_columns(
            pl.col("atm_iv_7d").alias("session_implied_vol"),
            pl.col("carry_spread_7d").alias("session_carry"),
        )
        .with_columns(
            _rolling_single_zscore_expr(
                "session_realized_vol",
                group_cols=["session_bucket"],
                window=24 * 14,
            ).alias("rv_z"),
            _rolling_single_zscore_expr(
                "session_carry",
                group_cols=["session_bucket"],
                window=24 * 14,
            ).alias("carry_z"),
        )
        .with_columns(
            pl.mean_horizontal(
                [
                    pl.col("rv_z"),
                    pl.col("carry_z"),
                    pl.when(pl.col("is_funding_window")).then(0.5).otherwise(0.0),
                    pl.when(pl.col("is_expiry_window")).then(0.75).otherwise(0.0),
                ]
            ).alias("sessionality_score"),
            pl.when(pl.col("carry_spread_7d") >= 0.0)
            .then(pl.lit("carry_on"))
            .otherwise(pl.lit("carry_off"))
            .alias("carry_switch"),
            pl.lit(SESSIONALITY_SOURCE).alias("source"),
            pl.when(pl.col("session_implied_vol").is_null())
            .then(pl.lit("insufficient_data"))
            .otherwise(pl.lit("ok"))
            .alias("quality_flag"),
        )
    )
    return frame.select(_business_columns(DatasetName.OPTIONS_SESSIONALITY_SERIES))


def build_options_dispersion_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
) -> pl.DataFrame:
    btc = _read_surface(store, start=start, end=end, underlying="BTC").select(
        [
            "ts",
            pl.col("atm_iv_7d").alias("btc_atm_iv_7d"),
            pl.col("atm_iv_30d").alias("btc_atm_iv_30d"),
            pl.col("rr_25_7d").alias("btc_rr_25_7d"),
            pl.col("bf_25_7d").alias("btc_bf_25_7d"),
            pl.col("rr_25_30d").alias("btc_rr_25_30d"),
            pl.col("bf_25_30d").alias("btc_bf_25_30d"),
        ]
    )
    eth = _read_surface(store, start=start, end=end, underlying="ETH").select(
        [
            "ts",
            pl.col("atm_iv_7d").alias("eth_atm_iv_7d"),
            pl.col("atm_iv_30d").alias("eth_atm_iv_30d"),
            pl.col("rr_25_7d").alias("eth_rr_25_7d"),
            pl.col("bf_25_7d").alias("eth_bf_25_7d"),
            pl.col("rr_25_30d").alias("eth_rr_25_30d"),
            pl.col("bf_25_30d").alias("eth_bf_25_30d"),
        ]
    )
    btc_spot = _spot_frame(store, start=start, end=end, underlying="BTC").select(
        ["ts", pl.col("underlying_price").alias("btc_price")]
    )
    eth_spot = _spot_frame(store, start=start, end=end, underlying="ETH").select(
        ["ts", pl.col("underlying_price").alias("eth_price")]
    )
    if btc.is_empty() or eth.is_empty():
        return empty_frame(DatasetName.OPTIONS_DISPERSION_SERIES)
    joined = btc.join(eth, on="ts", how="inner")
    if not btc_spot.is_empty() and not eth_spot.is_empty():
        joined = joined.join(btc_spot, on="ts", how="left").join(eth_spot, on="ts", how="left")
        joined = joined.sort("ts").with_columns(
            (pl.col("btc_price").log().diff()).alias("btc_ret"),
            (pl.col("eth_price").log().diff()).alias("eth_ret"),
            (pl.col("btc_atm_iv_7d").log().diff()).alias("btc_iv_ret_7d"),
            (pl.col("eth_atm_iv_7d").log().diff()).alias("eth_iv_ret_7d"),
            (pl.col("btc_atm_iv_30d").log().diff()).alias("btc_iv_ret_30d"),
            (pl.col("eth_atm_iv_30d").log().diff()).alias("eth_iv_ret_30d"),
        ).with_columns(
            rolling_corr_expr("btc_ret", "eth_ret", 24 * 7).alias("realized_corr_7d"),
            rolling_corr_expr("btc_ret", "eth_ret", 24 * 30).alias("realized_corr_30d"),
            rolling_corr_expr("btc_iv_ret_7d", "eth_iv_ret_7d", 24 * 7).alias(
                "implied_corr_7d_proxy"
            ),
            rolling_corr_expr("btc_iv_ret_30d", "eth_iv_ret_30d", 24 * 30).alias(
                "implied_corr_30d_proxy"
            ),
        )
    else:
        joined = joined.with_columns(
            pl.lit(None, dtype=pl.Float64).alias("realized_corr_7d"),
            pl.lit(None, dtype=pl.Float64).alias("realized_corr_30d"),
            pl.lit(None, dtype=pl.Float64).alias("implied_corr_7d_proxy"),
            pl.lit(None, dtype=pl.Float64).alias("implied_corr_30d_proxy"),
        )
    rows: list[pl.DataFrame] = []
    for tenor in ("7d", "30d"):
        iv_col = f"atm_iv_{tenor}"
        realized_col = f"realized_corr_{tenor}"
        rr_col = f"rr_25_{tenor}"
        bf_col = f"bf_25_{tenor}"
        tenor_frame = joined.select(
            [
                "ts",
                pl.col("ts").dt.date().alias("date"),
                pl.lit("BTC_ETH").alias("pair_id"),
                pl.lit(DISPERSION_SOURCE).alias("source"),
                pl.lit(tenor).alias("tenor"),
                pl.col(f"btc_{iv_col}").alias("btc_atm_iv"),
                pl.col(f"eth_{iv_col}").alias("eth_atm_iv"),
                pl.col(realized_col).alias("realized_correlation"),
                pl.col(f"implied_corr_{tenor}_proxy").alias("implied_correlation"),
                (pl.col(f"btc_{rr_col}") - pl.col(f"eth_{rr_col}")).alias("rr_richness_spread"),
                (pl.col(f"btc_{bf_col}") - pl.col(f"eth_{bf_col}")).alias("bf_richness_spread"),
            ]
        ).with_columns(
            (
                0.7 * pl.col("realized_correlation") + 0.3 * pl.col("realized_correlation").shift(24)
            ).alias("realized_corr_forecast")
        ).with_columns(
            (
                1.0
                - (
                    pl.col("rr_richness_spread").abs() * 5.0
                    + pl.col("bf_richness_spread").abs() * 5.0
                )
            ).clip(-1.0, 1.0).alias("cross_smile_consistency"),
            (pl.col("implied_correlation") - pl.col("realized_corr_forecast")).alias("corr_gap"),
        ).with_columns(
            (
                pl.col("corr_gap").fill_null(0.0)
                + (pl.col("cross_smile_consistency").fill_null(0.0) * 0.5)
            ).alias("dispersion_score"),
            pl.when(pl.col("corr_gap") >= 0.1)
            .then(pl.lit("long_dispersion"))
            .when(pl.col("corr_gap") <= -0.1)
            .then(pl.lit("short_dispersion"))
            .otherwise(pl.lit("neutral"))
            .alias("dispersion_regime"),
            pl.when(
                pl.col("realized_corr_forecast").is_null() | pl.col("implied_correlation").is_null()
            )
            .then(pl.lit("insufficient_data"))
            .otherwise(pl.lit("ok"))
            .alias("quality_flag"),
        )
        rows.append(tenor_frame)
    return pl.concat(rows, how="vertical_relaxed").select(
        _business_columns(DatasetName.OPTIONS_DISPERSION_SERIES)
    )


def build_options_funding_skew_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    surface = _read_surface(store, start=start, end=end, underlying=underlying)
    funding = store.read(
        DatasetName.FUNDING_CONTEXT_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": FUNDING_CONTEXT_SOURCE},
    )
    perps = store.read(
        DatasetName.PERPS_SIGNAL_SERIES,
        start=start,
        end=end,
        filters={"asset": underlying, "venue_scope": "all"},
    ).select(["ts", "basis_bps"])
    if funding.is_empty():
        funding = build_funding_context_series(store, start=start, end=end, underlying=underlying)
    if surface.is_empty() or funding.is_empty():
        return empty_frame(DatasetName.OPTIONS_FUNDING_SKEW_SERIES)
    frame = surface.select(
        ["ts", "date", "underlying", "rr_25_7d", "bf_25_7d"]
    ).join(
        funding.select(["ts", "funding_rate", "funding_rate_zscore"]),
        on="ts",
        how="left",
    )
    if not perps.is_empty():
        frame = frame.join(perps, on="ts", how="left")
    else:
        frame = frame.with_columns(pl.lit(None, dtype=pl.Float64).alias("basis_bps"))
    frame = frame.with_columns(
        ((-pl.col("rr_25_7d")) + pl.col("bf_25_7d").fill_null(0.0)).alias("skew_richness"),
        (
            (pl.col("basis_bps").fill_null(0.0) / 100.0)
            + pl.col("funding_rate_zscore").fill_null(0.0)
        ).alias("basis_pressure"),
    ).with_columns(
        (pl.col("basis_pressure") + pl.col("skew_richness") * 10.0).alias("funding_basis_skew_score"),
    ).with_columns(
        pl.when(pl.col("funding_basis_skew_score") >= 1.0)
        .then(pl.lit("put_rich"))
        .when(pl.col("funding_basis_skew_score") <= -1.0)
        .then(pl.lit("call_rich"))
        .otherwise(pl.lit("neutral"))
        .alias("regime"),
        pl.lit(FUNDING_SKEW_SOURCE).alias("source"),
        pl.lit("ok").alias("quality_flag"),
    )
    return frame.select(_business_columns(DatasetName.OPTIONS_FUNDING_SKEW_SERIES))


def build_vol_cone_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    surface = _read_surface(store, start=start, end=end, underlying=underlying)
    realized = _realized_vol_frame(store, start=start, end=end, underlying=underlying)
    if surface.is_empty() or realized.is_empty():
        return empty_frame(DatasetName.VOL_CONE_SERIES)
    cfg = get_config()
    joined = surface.join(realized, on="ts", how="left", coalesce=True).sort("ts")
    rows: list[dict[str, Any]] = []
    mappings = (
        ("7d", "atm_iv_7d", "realized_vol_7d"),
        ("14d", "atm_iv_14d", "realized_vol_14d"),
        ("30d", "atm_iv_30d", "realized_vol_30d"),
    )
    for tenor, implied_col, realized_col in mappings:
        tenor_frame = joined.select(
            [
                "ts",
                "date",
                "underlying",
                pl.col(implied_col).alias("implied_vol"),
                pl.col(realized_col).alias("realized_vol"),
            ]
        ).with_columns((pl.col("implied_vol") - pl.col("realized_vol")).alias("iv_rv_spread"))
        history = max(4, cfg.modeling.vol_cone_lookback_days * 24 // 3)
        tenor_frame = tenor_frame.with_columns(
            _rolling_percentile_expr("implied_vol", history).alias("implied_percentile"),
            _rolling_percentile_expr("realized_vol", history).alias("realized_percentile"),
            _rolling_percentile_expr("iv_rv_spread", history).alias("iv_rv_spread_percentile"),
            pl.lit(VOL_CONE_SOURCE).alias("source"),
            pl.lit(tenor).alias("tenor"),
            pl.when(pl.col("realized_vol").is_null())
            .then(pl.lit("insufficient_data"))
            .otherwise(pl.lit("ok"))
            .alias("quality_flag"),
        )
        rows.extend(tenor_frame.to_dicts())
    if not rows:
        return empty_frame(DatasetName.VOL_CONE_SERIES)
    return pl.DataFrame(rows).select(_business_columns(DatasetName.VOL_CONE_SERIES))


def build_term_structure_momentum_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    surface = _read_surface(store, start=start, end=end, underlying=underlying)
    if surface.is_empty():
        return empty_frame(DatasetName.TERM_STRUCTURE_MOMENTUM_SERIES)
    return (
        surface.sort("ts")
        .with_columns(
            pl.col("term_slope_7_30").diff(24).alias("term_slope_change_24h"),
            pl.col("term_slope_7_30").diff(72).alias("term_slope_change_72h"),
            pl.lit(TERM_MOMENTUM_SOURCE).alias("source"),
        )
        .with_columns(
            pl.when(pl.col("term_slope_change_24h") > 0)
            .then(pl.lit("steepening"))
            .when(pl.col("term_slope_change_24h") < 0)
            .then(pl.lit("flattening"))
            .otherwise(pl.lit("neutral"))
            .alias("momentum_regime"),
            pl.when(pl.col("term_slope_7_30").is_null())
            .then(pl.lit("insufficient_data"))
            .otherwise(pl.col("quality_flag"))
            .alias("quality_flag"),
        )
        .select(_business_columns(DatasetName.TERM_STRUCTURE_MOMENTUM_SERIES))
    )


def build_gamma_exposure_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    options = _read_options(store, start=start, end=end, underlying=underlying)
    if options.is_empty() or "gamma" not in options.columns:
        return empty_frame(DatasetName.GAMMA_EXPOSURE_SERIES)
    frame = (
        options.filter(pl.col("forward").is_not_null() & pl.col("gamma").is_not_null())
        .with_columns(
            _expiry_bucket_expr().alias("expiry_bucket"),
            ((pl.col("strike") / pl.col("forward")) - 1.0).alias("moneyness"),
        )
        .with_columns(_strike_bucket_expr(pl.col("moneyness")).alias("strike_bucket"))
        .with_columns(
            (
                pl.col("gamma").abs() * pl.col("open_interest") * pl.col("underlying_price") ** 2
            ).alias("gamma_exposure_raw")
        )
        .group_by(["ts", "date", "underlying", "expiry_bucket", "strike_bucket"])
        .agg(pl.col("gamma_exposure_raw").sum().alias("gamma_exposure"))
    )
    totals = frame.group_by(["ts", "underlying"]).agg(
        pl.col("gamma_exposure").sum().alias("total_gamma_exposure"),
        pl.when(pl.col("strike_bucket") == "atm")
        .then(pl.col("gamma_exposure"))
        .otherwise(0.0)
        .sum()
        .alias("atm_gamma_exposure"),
    )
    return (
        frame.join(totals, on=["ts", "underlying"], how="left")
        .with_columns(
            pl.when(pl.col("total_gamma_exposure") > 0)
            .then(pl.col("gamma_exposure") / pl.col("total_gamma_exposure"))
            .otherwise(None)
            .alias("gamma_share"),
            pl.when(pl.col("total_gamma_exposure") > 0)
            .then(pl.col("atm_gamma_exposure") / pl.col("total_gamma_exposure"))
            .otherwise(None)
            .alias("pinning_concentration"),
            pl.lit(GAMMA_EXPOSURE_SOURCE).alias("source"),
            pl.lit("ok").alias("quality_flag"),
        )
        .select(_business_columns(DatasetName.GAMMA_EXPOSURE_SERIES))
    )


def build_funding_context_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    funding = store.read(
        DatasetName.FUNDING_RATE_SNAPSHOT,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": "deribit_perpetual"},
    )
    vvix = store.read(
        DatasetName.VVIX_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": "dvol"},
    )
    if funding.is_empty():
        return empty_frame(DatasetName.FUNDING_CONTEXT_SERIES)
    cfg = get_config()
    history = max(8, cfg.modeling.funding_zscore_lookback_days * 24 // 2)
    return (
        funding.join(vvix.select(["ts", "rvvix_14d", "qvvix_30d"]), on="ts", how="left")
        .sort("ts")
        .with_columns(
            pl.col("interest_8h").alias("funding_rate"),
            pl.col("interest_8h").diff(24).alias("funding_change_24h"),
            _rolling_single_zscore_expr("interest_8h", window=history).alias(
                "funding_rate_zscore"
            ),
            pl.lit(FUNDING_CONTEXT_SOURCE).alias("source"),
        )
        .with_columns(
            pl.when(pl.col("funding_rate_zscore") >= 1.0)
            .then(pl.lit("crowded_long"))
            .when(pl.col("funding_rate_zscore") <= -1.0)
            .then(pl.lit("crowded_short"))
            .otherwise(pl.lit("neutral"))
            .alias("funding_regime"),
            pl.lit("ok").alias("quality_flag"),
        )
        .select(_business_columns(DatasetName.FUNDING_CONTEXT_SERIES))
    )


def build_straddle_breakeven_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    surface = _read_surface(store, start=start, end=end, underlying=underlying)
    spot = _spot_frame(store, start=start, end=end, underlying=underlying)
    if surface.is_empty() or spot.is_empty():
        return empty_frame(DatasetName.STRADDLE_BREAKEVEN_SERIES)
    return (
        surface.join(spot.select(["ts", "spot_move_24h"]), on="ts", how="left")
        .sort("ts")
        .with_columns(
            (pl.col("atm_iv_7d") * float(np.sqrt(7.0 / 365.0))).alias("breakeven_move_pct"),
            (
                pl.col("spot_move_24h")
                .abs()
                .rolling_max(window_size=7 * 24, min_samples=24)
            ).alias("realized_move_7d_pct"),
            pl.lit(STRADDLE_BREAKEVEN_SOURCE).alias("source"),
        )
        .with_columns(
            (pl.col("realized_move_7d_pct") - pl.col("breakeven_move_pct")).alias(
                "breakeven_gap_pct"
            ),
            pl.when(pl.col("realized_move_7d_pct") > pl.col("breakeven_move_pct"))
            .then(pl.lit("gamma_underpriced"))
            .when(pl.col("realized_move_7d_pct") < pl.col("breakeven_move_pct"))
            .then(pl.lit("gamma_overpriced"))
            .otherwise(pl.lit("neutral"))
            .alias("breakeven_regime"),
            pl.when(pl.col("realized_move_7d_pct").is_null())
            .then(pl.lit("insufficient_data"))
            .otherwise(pl.lit("ok"))
            .alias("quality_flag"),
        )
        .select(_business_columns(DatasetName.STRADDLE_BREAKEVEN_SERIES))
    )


def build_vol_regime_transition_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    signals = store.read(
        DatasetName.OPTIONS_SIGNAL_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": OPTIONS_SIGNAL_SOURCE},
    ).sort("ts")
    if signals.is_empty():
        return empty_frame(DatasetName.VOL_REGIME_TRANSITION_SERIES)
    cfg = get_config()
    lookback_hours = max(24, cfg.modeling.regime_transition_lookback_days * 24)
    rows: list[dict[str, Any]] = []
    signal_rows = signals.select(["ts", "date", "underlying", "signal_state"]).to_dicts()
    for idx, row in enumerate(signal_rows):
        current_state_raw = row["signal_state"]
        if current_state_raw is None:
            rows.append(
                {
                    "ts": row["ts"],
                    "date": row["date"],
                    "underlying": row["underlying"],
                    "source": VOL_REGIME_TRANSITION_SOURCE,
                    "signal_state": None,
                    "prior_state": None,
                    "next_state_probability_long_vol_bias": None,
                    "next_state_probability_neutral": None,
                    "next_state_probability_short_vol_bias": None,
                    "median_state_duration_hours": None,
                    "quality_flag": "insufficient_data",
                }
            )
            continue
        window = [
            item
            for item in signal_rows[max(0, idx - lookback_hours + 1) : idx + 1]
            if item["signal_state"] is not None
        ]
        if not window:
            rows.append(
                {
                    "ts": row["ts"],
                    "date": row["date"],
                    "underlying": row["underlying"],
                    "source": VOL_REGIME_TRANSITION_SOURCE,
                    "signal_state": str(current_state_raw),
                    "prior_state": None,
                    "next_state_probability_long_vol_bias": None,
                    "next_state_probability_neutral": None,
                    "next_state_probability_short_vol_bias": None,
                    "median_state_duration_hours": None,
                    "quality_flag": "insufficient_data",
                }
            )
            continue
        transitions = {
            "long_vol_bias": 0,
            "neutral": 0,
            "short_vol_bias": 0,
        }
        durations: dict[str, list[int]] = {
            "long_vol_bias": [],
            "neutral": [],
            "short_vol_bias": [],
        }
        run_state: str | None = None
        run_length = 0
        for pos, item in enumerate(window):
            state = str(item["signal_state"])
            if pos > 0:
                prev_state = str(window[pos - 1]["signal_state"])
                transitions[state] += int(prev_state == str(row["signal_state"]))
            if state == run_state:
                run_length += 1
            else:
                if run_state is not None:
                    durations[run_state].append(run_length)
                run_state = state
                run_length = 1
        if run_state is not None:
            durations[run_state].append(run_length)
        current_state = str(current_state_raw)
        total_from_state = sum(
            1
            for pos in range(1, len(window))
            if str(window[pos - 1]["signal_state"]) == current_state
        )
        prior_state = next(
            (
                str(item["signal_state"])
                for item in reversed(signal_rows[:idx])
                if item["signal_state"] is not None
            ),
            None,
        )
        rows.append(
            {
                "ts": row["ts"],
                "date": row["date"],
                "underlying": row["underlying"],
                "source": VOL_REGIME_TRANSITION_SOURCE,
                "signal_state": current_state,
                "prior_state": prior_state,
                "next_state_probability_long_vol_bias": (
                    transitions["long_vol_bias"] / total_from_state if total_from_state else None
                ),
                "next_state_probability_neutral": (
                    transitions["neutral"] / total_from_state if total_from_state else None
                ),
                "next_state_probability_short_vol_bias": (
                    transitions["short_vol_bias"] / total_from_state if total_from_state else None
                ),
                "median_state_duration_hours": (
                    float(np.median(durations[current_state])) if durations[current_state] else None
                ),
                "quality_flag": "ok" if total_from_state else "insufficient_data",
            }
        )
    return pl.DataFrame(rows).select(_business_columns(DatasetName.VOL_REGIME_TRANSITION_SERIES))


def build_options_signal_series(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    carry = store.read(
        DatasetName.OPTIONS_CARRY_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": CARRY_SOURCE},
    ).select(["ts", "date", "underlying", "carry_zscore"])
    skew = store.read(
        DatasetName.OPTIONS_SKEW_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": SKEW_SOURCE},
    ).select(["ts", pl.col("skew_zscore")])
    dislocation = (
        store.read(
            DatasetName.OPTIONS_DISLOCATION_SERIES,
            start=start,
            end=end,
            filters={"underlying": underlying, "source": DISLOCATION_SOURCE},
        )
        .group_by("ts")
        .agg(
            pl.col("dislocation_zscore").mean().alias("dislocation_zscore"),
            pl.col("max_residual").max().alias("max_residual"),
        )
    )
    vvix = store.read(
        DatasetName.VVIX_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": "dvol"},
    ).select(["ts", "rvvix_14d", "qvvix_30d"])
    razor = store.read(
        DatasetName.RAZOR_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying},
    ).select(["ts", "razor_vrp_zscore", "razor_vrp"])
    funding = store.read(
        DatasetName.FUNDING_CONTEXT_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": FUNDING_CONTEXT_SOURCE},
    ).select(["ts", "funding_rate_zscore"])
    breakeven = store.read(
        DatasetName.STRADDLE_BREAKEVEN_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": STRADDLE_BREAKEVEN_SOURCE},
    ).select(["ts", "breakeven_gap_pct"])
    term_momentum = store.read(
        DatasetName.TERM_STRUCTURE_MOMENTUM_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": TERM_MOMENTUM_SOURCE},
    ).select(["ts", "term_slope_change_24h"])
    relative = store.read(
        DatasetName.OPTIONS_RELATIVE_VALUE_SERIES,
        start=start,
        end=end,
        filters={"pair_id": "BTC_ETH", "source": RELATIVE_VALUE_SOURCE},
    ).select(["ts", "spread_zscore"])
    surface_factor = (
        store.read(
            DatasetName.OPTIONS_SURFACE_FACTOR_SERIES,
            start=start,
            end=end,
            filters={"underlying": underlying, "source": SURFACE_FACTOR_SOURCE},
        )
        .group_by("ts")
        .agg(
            pl.col("factor_momentum_24h").mean().alias("surface_factor_score"),
            pl.col("residual_value_score").mean().alias("residual_surface_rv_score"),
        )
    )
    ml_carry = store.read(
        DatasetName.OPTIONS_ML_CARRY_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": ML_CARRY_SOURCE},
    ).select(["ts", "net_carry_score", "carry_veto"])
    expiry_flow = (
        store.read(
            DatasetName.OPTIONS_EXPIRY_FLOW_SERIES,
            start=start,
            end=end,
            filters={"underlying": underlying, "source": EXPIRY_FLOW_SOURCE},
        )
        .group_by("ts")
        .agg(
            pl.col("pinning_score").mean().alias("expiry_pin_score"),
            pl.col("breakout_score").mean().alias("gamma_flip_score"),
        )
    )
    adaptive_hedge = store.read(
        DatasetName.OPTIONS_ADAPTIVE_HEDGE_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": ADAPTIVE_HEDGE_SOURCE},
    ).select(["ts", "route_score"])
    sessionality = (
        store.read(
            DatasetName.OPTIONS_SESSIONALITY_SERIES,
            start=start,
            end=end,
            filters={"underlying": underlying, "source": SESSIONALITY_SOURCE},
        )
        .group_by("ts")
        .agg(pl.col("sessionality_score").mean().alias("sessionality_score"))
    )
    dispersion = store.read(
        DatasetName.OPTIONS_DISPERSION_SERIES,
        start=start,
        end=end,
        filters={"pair_id": "BTC_ETH", "source": DISPERSION_SOURCE, "tenor": "30d"},
    ).select(["ts", "dispersion_score"])
    funding_skew = store.read(
        DatasetName.OPTIONS_FUNDING_SKEW_SERIES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": FUNDING_SKEW_SOURCE},
    ).select(["ts", "funding_basis_skew_score"])
    if carry.is_empty():
        return empty_frame(DatasetName.OPTIONS_SIGNAL_SERIES)
    cfg = get_config()
    threshold_window = max(24, cfg.modeling.options_signal_threshold_lookback_days * 24)
    min_history = max(12, cfg.modeling.options_signal_threshold_min_history_hours)
    long_quantile = cfg.modeling.options_signal_long_quantile
    short_quantile = cfg.modeling.options_signal_short_quantile
    frame = carry
    for next_frame in (
        skew,
        dislocation,
        vvix,
        razor,
        funding,
        breakeven,
        term_momentum,
        surface_factor,
        ml_carry,
        expiry_flow,
        adaptive_hedge,
        sessionality,
        funding_skew,
    ):
        if not next_frame.is_empty():
            frame = frame.join(next_frame, on="ts", how="left")
    if not dispersion.is_empty():
        dispersion_for_asset = dispersion.with_columns(
            pl.when(pl.lit(underlying) == "BTC")
            .then(pl.col("dispersion_score"))
            .otherwise(-pl.col("dispersion_score"))
            .alias("dispersion_score")
        ).select(["ts", "dispersion_score"])
        frame = frame.join(dispersion_for_asset, on="ts", how="left")
    else:
        frame = frame.with_columns(pl.lit(None, dtype=pl.Float64).alias("dispersion_score"))
    missing_defaults: dict[str, Any] = {
        "skew_zscore": None,
        "dislocation_zscore": None,
        "funding_rate_zscore": None,
        "breakeven_gap_pct": None,
        "term_slope_change_24h": None,
        "rvvix_14d": None,
        "razor_vrp": None,
        "qvvix_30d": None,
        "surface_factor_score": None,
        "residual_surface_rv_score": None,
        "net_carry_score": None,
        "expiry_pin_score": None,
        "gamma_flip_score": None,
        "route_score": None,
        "sessionality_score": None,
        "funding_basis_skew_score": None,
    }
    absent_numeric = [column for column in missing_defaults if column not in frame.columns]
    if absent_numeric:
        frame = frame.with_columns(
            [pl.lit(missing_defaults[column], dtype=pl.Float64).alias(column) for column in absent_numeric]
        )
    if "carry_veto" not in frame.columns:
        frame = frame.with_columns(pl.lit(False).alias("carry_veto"))
    if not relative.is_empty():
        relative_for_asset = relative.with_columns(
            pl.when(pl.lit(underlying) == "BTC")
            .then(pl.col("spread_zscore"))
            .otherwise(-pl.col("spread_zscore"))
            .alias("cross_asset_vol_score")
        ).select(["ts", "cross_asset_vol_score"])
        frame = frame.join(relative_for_asset, on="ts", how="left")
    else:
        frame = frame.with_columns(pl.lit(None, dtype=pl.Float64).alias("cross_asset_vol_score"))
    frame = (
        frame.sort("ts")
        .with_columns(
            pl.col("carry_zscore").alias("vol_carry_score"),
            (-pl.col("skew_zscore")).alias("skew_stress_score"),
            pl.col("dislocation_zscore").alias("surface_dislocation_score"),
            pl.col("funding_rate_zscore").alias("funding_vol_context_score"),
            _rolling_single_zscore_expr("breakeven_gap_pct").alias("breakeven_gap_score"),
            _rolling_single_zscore_expr("term_slope_change_24h").alias(
                "term_structure_momentum_score"
            ),
            _rolling_single_zscore_expr("surface_factor_score").alias("surface_factor_score"),
            _rolling_single_zscore_expr("residual_surface_rv_score").alias(
                "residual_surface_rv_score"
            ),
            _rolling_single_zscore_expr("net_carry_score").alias("ml_carry_toxicity_score"),
            _rolling_single_zscore_expr("expiry_pin_score").alias("expiry_pin_score"),
            _rolling_single_zscore_expr("gamma_flip_score").alias("gamma_flip_score"),
            _rolling_single_zscore_expr("route_score").alias("adaptive_hedge_score"),
            _rolling_single_zscore_expr("sessionality_score").alias("sessionality_score"),
            _rolling_single_zscore_expr("funding_basis_skew_score").alias("funding_skew_score"),
            _rolling_single_zscore_expr("rvvix_14d").alias("vvix_zscore"),
            _rolling_single_zscore_expr("razor_vrp").alias("razor_context_zscore"),
            _rolling_single_zscore_expr("qvvix_30d").alias("qvvix_zscore"),
        )
        .with_columns(
            pl.mean_horizontal(
                [
                    pl.coalesce(pl.col("qvvix_zscore"), pl.col("vvix_zscore")),
                    (-pl.col("carry_zscore")),
                    (-pl.col("skew_zscore")),
                    pl.col("razor_context_zscore"),
                    pl.col("funding_rate_zscore"),
                    pl.col("breakeven_gap_score"),
                    pl.col("surface_factor_score"),
                    pl.col("residual_surface_rv_score"),
                    pl.col("ml_carry_toxicity_score"),
                    pl.col("expiry_pin_score"),
                    pl.col("gamma_flip_score"),
                    pl.col("adaptive_hedge_score"),
                    pl.col("sessionality_score"),
                    pl.col("dispersion_score"),
                    pl.col("funding_skew_score"),
                ]
            ).alias("vol_regime_score")
        )
        .with_columns(
            _rolling_quantile_expr("vol_carry_score", long_quantile, threshold_window, min_history)
            .alias("vol_carry_long_threshold"),
            _rolling_quantile_expr("vol_carry_score", short_quantile, threshold_window, min_history)
            .alias("vol_carry_short_threshold"),
            _rolling_quantile_expr(
                "skew_stress_score", long_quantile, threshold_window, min_history
            ).alias("skew_stress_long_threshold"),
            _rolling_quantile_expr(
                "skew_stress_score", short_quantile, threshold_window, min_history
            ).alias("skew_stress_short_threshold"),
            _rolling_quantile_expr(
                "surface_dislocation_score", long_quantile, threshold_window, min_history
            ).alias("surface_dislocation_long_threshold"),
            _rolling_quantile_expr(
                "surface_dislocation_score", short_quantile, threshold_window, min_history
            ).alias("surface_dislocation_short_threshold"),
            _rolling_quantile_expr(
                "cross_asset_vol_score", long_quantile, threshold_window, min_history
            ).alias("cross_asset_vol_long_threshold"),
            _rolling_quantile_expr(
                "cross_asset_vol_score", short_quantile, threshold_window, min_history
            ).alias("cross_asset_vol_short_threshold"),
            _rolling_quantile_expr(
                "funding_vol_context_score", long_quantile, threshold_window, min_history
            ).alias("funding_vol_context_long_threshold"),
            _rolling_quantile_expr(
                "funding_vol_context_score", short_quantile, threshold_window, min_history
            ).alias("funding_vol_context_short_threshold"),
            _rolling_quantile_expr(
                "breakeven_gap_score", long_quantile, threshold_window, min_history
            ).alias("breakeven_gap_long_threshold"),
            _rolling_quantile_expr(
                "breakeven_gap_score", short_quantile, threshold_window, min_history
            ).alias("breakeven_gap_short_threshold"),
            _rolling_quantile_expr(
                "term_structure_momentum_score", long_quantile, threshold_window, min_history
            ).alias("term_structure_momentum_long_threshold"),
            _rolling_quantile_expr(
                "term_structure_momentum_score", short_quantile, threshold_window, min_history
            ).alias("term_structure_momentum_short_threshold"),
            _rolling_quantile_expr(
                "surface_factor_score", long_quantile, threshold_window, min_history
            ).alias("surface_factor_long_threshold"),
            _rolling_quantile_expr(
                "surface_factor_score", short_quantile, threshold_window, min_history
            ).alias("surface_factor_short_threshold"),
            _rolling_quantile_expr(
                "residual_surface_rv_score", long_quantile, threshold_window, min_history
            ).alias("residual_surface_rv_long_threshold"),
            _rolling_quantile_expr(
                "residual_surface_rv_score", short_quantile, threshold_window, min_history
            ).alias("residual_surface_rv_short_threshold"),
            _rolling_quantile_expr(
                "ml_carry_toxicity_score", long_quantile, threshold_window, min_history
            ).alias("ml_carry_toxicity_long_threshold"),
            _rolling_quantile_expr(
                "ml_carry_toxicity_score", short_quantile, threshold_window, min_history
            ).alias("ml_carry_toxicity_short_threshold"),
            _rolling_quantile_expr(
                "expiry_pin_score", long_quantile, threshold_window, min_history
            ).alias("expiry_pin_long_threshold"),
            _rolling_quantile_expr(
                "expiry_pin_score", short_quantile, threshold_window, min_history
            ).alias("expiry_pin_short_threshold"),
            _rolling_quantile_expr(
                "gamma_flip_score", long_quantile, threshold_window, min_history
            ).alias("gamma_flip_long_threshold"),
            _rolling_quantile_expr(
                "gamma_flip_score", short_quantile, threshold_window, min_history
            ).alias("gamma_flip_short_threshold"),
            _rolling_quantile_expr(
                "adaptive_hedge_score", long_quantile, threshold_window, min_history
            ).alias("adaptive_hedge_long_threshold"),
            _rolling_quantile_expr(
                "adaptive_hedge_score", short_quantile, threshold_window, min_history
            ).alias("adaptive_hedge_short_threshold"),
            _rolling_quantile_expr(
                "sessionality_score", long_quantile, threshold_window, min_history
            ).alias("sessionality_long_threshold"),
            _rolling_quantile_expr(
                "sessionality_score", short_quantile, threshold_window, min_history
            ).alias("sessionality_short_threshold"),
            _rolling_quantile_expr(
                "dispersion_score", long_quantile, threshold_window, min_history
            ).alias("dispersion_long_threshold"),
            _rolling_quantile_expr(
                "dispersion_score", short_quantile, threshold_window, min_history
            ).alias("dispersion_short_threshold"),
            _rolling_quantile_expr(
                "funding_skew_score", long_quantile, threshold_window, min_history
            ).alias("funding_skew_long_threshold"),
            _rolling_quantile_expr(
                "funding_skew_score", short_quantile, threshold_window, min_history
            ).alias("funding_skew_short_threshold"),
            _rolling_quantile_expr("vol_regime_score", long_quantile, threshold_window, min_history)
            .alias("vol_regime_long_threshold"),
            _rolling_quantile_expr(
                "vol_regime_score", short_quantile, threshold_window, min_history
            ).alias("vol_regime_short_threshold"),
        )
        .with_columns(
            pl.when(
                pl.col("vol_regime_long_threshold").is_null()
                | pl.col("vol_regime_short_threshold").is_null()
            )
            .then(pl.lit(None, dtype=pl.String))
            .when(pl.col("vol_regime_score") >= pl.col("vol_regime_long_threshold"))
            .then(pl.lit("long_vol_bias"))
            .when(pl.col("vol_regime_score") <= pl.col("vol_regime_short_threshold"))
            .then(pl.lit("short_vol_bias"))
            .otherwise(pl.lit("neutral"))
            .alias("signal_state"),
            pl.lit(OPTIONS_SIGNAL_SOURCE).alias("source"),
            pl.when(
                pl.col("vol_carry_score").is_null()
                | pl.col("vol_regime_long_threshold").is_null()
                | pl.col("vol_regime_short_threshold").is_null()
            )
            .then(pl.lit("insufficient_data"))
            .when(pl.col("carry_veto").fill_null(False))
            .then(pl.lit("vetoed"))
            .otherwise(pl.lit("ok"))
            .alias("quality_flag"),
        )
    )
    return frame.select(_business_columns(DatasetName.OPTIONS_SIGNAL_SERIES))


def _read_options(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    return store.read(
        DatasetName.OPTION_CHAIN_SNAPSHOT,
        start=start,
        end=end,
        filters={"underlying": underlying},
    ).filter(pl.col("quality_flag").is_in(["ok", "wide_spread"]))


def _read_surface(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    business_columns = _business_columns(DatasetName.SURFACE_FEATURES)
    surface = store.read(
        DatasetName.SURFACE_FEATURES,
        start=start,
        end=end,
        filters={"underlying": underlying, "source": "ssvi"},
    ).select(business_columns).sort("ts")
    options = _read_options(store, start=start, end=end, underlying=underlying)
    if options.is_empty():
        return surface

    stored_timestamps = (
        set(cast(list[datetime], surface.get_column("ts").to_list())) if not surface.is_empty() else set()
    )
    missing_timestamps = [
        ts
        for ts in cast(list[datetime], options.get_column("ts").unique().sort().to_list())
        if ts not in stored_timestamps
    ]
    if not missing_timestamps:
        return surface

    recovered_rows: list[pl.DataFrame] = []
    snapshots = options.filter(pl.col("ts").is_in(missing_timestamps))
    for _, snapshot in snapshots.partition_by("ts", maintain_order=True, as_dict=True).items():
        try:
            recovered_rows.append(compute_surface_features(snapshot))
        except ValueError:
            continue
    if not recovered_rows:
        return surface

    recovered = pl.concat(recovered_rows, how="vertical_relaxed").select(business_columns)
    if surface.is_empty():
        return recovered.sort("ts")
    return (
        pl.concat([surface, recovered], how="vertical_relaxed")
        .sort(["underlying", "source", "ts"])
        .unique(subset=["ts", "underlying", "source"], keep="last", maintain_order=True)
        .sort("ts")
    )


def _spot_frame(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    options = _read_options(store, start=start, end=end, underlying=underlying)
    if options.is_empty():
        return pl.DataFrame(
            schema={
                "ts": pl.Datetime(time_zone="UTC"),
                "underlying_price": pl.Float64,
                "spot_move_24h": pl.Float64,
            }
        )
    return (
        options.group_by("ts")
        .agg(pl.col("underlying_price").median().alias("underlying_price"))
        .sort("ts")
        .with_columns(
            (
                pl.col("underlying_price") / pl.col("underlying_price").shift(24) - 1.0
            ).alias("spot_move_24h")
        )
    )


def _realized_vol_frame(
    store: DatasetStore,
    *,
    start: datetime,
    end: datetime,
    underlying: str,
) -> pl.DataFrame:
    spot = _spot_frame(store, start=start, end=end, underlying=underlying)
    if spot.is_empty():
        return pl.DataFrame(
            schema={
                "ts": pl.Datetime(time_zone="UTC"),
                "realized_vol_7d": pl.Float64,
                "realized_vol_14d": pl.Float64,
                "realized_vol_30d": pl.Float64,
            }
        )
    annualizer = float(np.sqrt(24.0 * 365.0))
    return (
        spot.with_columns(pl.col("underlying_price").log().diff().alias("log_return"))
        .with_columns(
            (
                pl.col("log_return").rolling_std(window_size=7 * 24, min_samples=24)
                * annualizer
            ).alias("realized_vol_7d"),
            (
                pl.col("log_return").rolling_std(window_size=14 * 24, min_samples=24)
                * annualizer
            ).alias("realized_vol_14d"),
            (
                pl.col("log_return").rolling_std(window_size=30 * 24, min_samples=24)
                * annualizer
            ).alias("realized_vol_30d"),
        )
        .select(["ts", "realized_vol_7d", "realized_vol_14d", "realized_vol_30d"])
    )


def _tenor_bucket_expr() -> pl.Expr:
    days = (pl.col("expiry") - pl.col("ts")).dt.total_seconds() / 86400.0
    distance = pl.min_horizontal(
        [(days - target).abs() for target in TARGET_MATURITIES]
    )
    return (
        pl.when(distance == (days - 7).abs())
        .then(pl.lit("7d"))
        .when(distance == (days - 14).abs())
        .then(pl.lit("14d"))
        .when(distance == (days - 30).abs())
        .then(pl.lit("30d"))
        .otherwise(pl.lit("60d"))
    )


def _expiry_bucket_expr() -> pl.Expr:
    days = (pl.col("expiry") - pl.col("ts")).dt.total_seconds() / 86400.0
    return (
        pl.when(days <= 7.0)
        .then(pl.lit("0_7d"))
        .when(days <= 30.0)
        .then(pl.lit("8_30d"))
        .when(days <= 90.0)
        .then(pl.lit("31_90d"))
        .otherwise(pl.lit("90d_plus"))
    )


def _moneyness_bucket_expr(expr: pl.Expr) -> pl.Expr:
    return (
        pl.when(expr <= -0.20)
        .then(pl.lit("deep_put"))
        .when(expr <= -0.05)
        .then(pl.lit("put_wing"))
        .when(expr < 0.05)
        .then(pl.lit("atm"))
        .when(expr < 0.20)
        .then(pl.lit("call_wing"))
        .otherwise(pl.lit("deep_call"))
    )


def _strike_bucket_expr(expr: pl.Expr) -> pl.Expr:
    return (
        pl.when(expr <= -0.10)
        .then(pl.lit("downside"))
        .when(expr < 0.10)
        .then(pl.lit("atm"))
        .otherwise(pl.lit("upside"))
    )


def _rolling_zscore_expr(numerator_col: str, denominator_col: str) -> pl.Expr:
    spread = pl.col(numerator_col) - pl.col(denominator_col)
    mean = spread.rolling_mean(window_size=24 * 30, min_samples=24)
    std = spread.rolling_std(window_size=24 * 30, min_samples=24)
    return ((spread - mean) / std).fill_nan(None)


def _rolling_single_zscore_expr(
    column: str,
    *,
    group_cols: list[str] | None = None,
    window: int = 24 * 30,
) -> pl.Expr:
    base = pl.col(column)
    if group_cols:
        mean = base.rolling_mean(window_size=window, min_samples=24).over(group_cols)
        std = base.rolling_std(window_size=window, min_samples=24).over(group_cols)
    else:
        mean = base.rolling_mean(window_size=window, min_samples=24)
        std = base.rolling_std(window_size=window, min_samples=24)
    return ((base - mean) / std).fill_nan(None)


def _rolling_percentile_expr(column: str, window: int) -> pl.Expr:
    return (
        ((pl.col(column) - pl.col(column).rolling_min(window_size=window, min_samples=12))
        / (
            pl.col(column).rolling_max(window_size=window, min_samples=12)
            - pl.col(column).rolling_min(window_size=window, min_samples=12)
        ))
        .clip(0.0, 1.0)
        .fill_nan(None)
    )


def _rolling_quantile_expr(column: str, quantile: float, window: int, min_samples: int) -> pl.Expr:
    return pl.col(column).rolling_quantile(
        quantile=quantile,
        window_size=window,
        min_samples=min_samples,
    )


def _business_columns(dataset: DatasetName) -> list[str]:
    return [
        column
        for column in DATASET_SPECS[dataset].schema.keys()
        if column not in _STORAGE_METADATA_COLUMNS
    ]
