from __future__ import annotations

from dataclasses import dataclass
from enum import Enum
from typing import Any

import polars as pl


class DatasetName(str, Enum):
    MICROSTRUCTURE_RAW_EVENT = "microstructure_raw_event"
    MICROSTRUCTURE_BOOK_STATE = "microstructure_book_state"
    MICROSTRUCTURE_BAR = "microstructure_bar"
    MICROSTRUCTURE_FEATURE_SERIES = "microstructure_feature_series"
    MICROSTRUCTURE_LABEL_SERIES = "microstructure_label_series"
    MICROSTRUCTURE_MODEL_OUTPUT = "microstructure_model_output"
    MICROSTRUCTURE_REGIME_SERIES = "microstructure_regime_series"
    MICROSTRUCTURE_SIGNAL_SERIES = "microstructure_signal_series"
    MICROSTRUCTURE_TRAINING_RUN = "microstructure_training_run"
    MICROSTRUCTURE_BACKTEST_RUN = "microstructure_backtest_run"
    OPTION_CHAIN_SNAPSHOT = "option_chain_snapshot"
    DVOL_SNAPSHOT = "dvol_snapshot"
    FUNDING_RATE_SNAPSHOT = "funding_rate_snapshot"
    VOL_INDEX_SERIES = "vol_index_series"
    SURFACE_FEATURES = "surface_features"
    VVIX_SERIES = "vvix_series"
    RAZOR_SERIES = "razor_series"
    OPTIONS_SURFACE_SNAPSHOT = "options_surface_snapshot"
    OPTIONS_CARRY_SERIES = "options_carry_series"
    OPTIONS_SKEW_SERIES = "options_skew_series"
    OPTIONS_DISLOCATION_SERIES = "options_dislocation_series"
    OPTIONS_RELATIVE_VALUE_SERIES = "options_relative_value_series"
    OPTIONS_POSITIONING_SERIES = "options_positioning_series"
    OPTIONS_SURFACE_FACTOR_SERIES = "options_surface_factor_series"
    OPTIONS_ML_CARRY_SERIES = "options_ml_carry_series"
    OPTIONS_EXPIRY_FLOW_SERIES = "options_expiry_flow_series"
    OPTIONS_ADAPTIVE_HEDGE_SERIES = "options_adaptive_hedge_series"
    OPTIONS_SESSIONALITY_SERIES = "options_sessionality_series"
    OPTIONS_DISPERSION_SERIES = "options_dispersion_series"
    OPTIONS_FUNDING_SKEW_SERIES = "options_funding_skew_series"
    OPTIONS_SIGNAL_SERIES = "options_signal_series"
    VOL_CONE_SERIES = "vol_cone_series"
    TERM_STRUCTURE_MOMENTUM_SERIES = "term_structure_momentum_series"
    GAMMA_EXPOSURE_SERIES = "gamma_exposure_series"
    FUNDING_CONTEXT_SERIES = "funding_context_series"
    STRADDLE_BREAKEVEN_SERIES = "straddle_breakeven_series"
    VOL_REGIME_TRANSITION_SERIES = "vol_regime_transition_series"
    PERPS_SIGNAL_SERIES = "perps_signal_series"
    PERPS_COMPOSITE_SERIES = "perps_composite_series"
    PERPS_TRANSITION_SERIES = "perps_transition_series"
    COMPOSITE_SIGNAL_SERIES = "composite_signal_series"
    COMPOSITE_COMPONENT_SERIES = "composite_component_series"
    COMPOSITE_TRANSITION_SERIES = "composite_transition_series"
    EXECUTION_ORDER_INTENT = "execution_order_intent"
    EXECUTION_ORDER_EVENT = "execution_order_event"
    EXECUTION_FILL_EVENT = "execution_fill_event"
    EXECUTION_POSITION_SNAPSHOT = "execution_position_snapshot"
    EXECUTION_BALANCE_SNAPSHOT = "execution_balance_snapshot"
    EXECUTION_QUOTE_SNAPSHOT = "execution_quote_snapshot"
    EXECUTION_MARKET_SNAPSHOT = "execution_market_snapshot"
    EXECUTION_HEALTH_SNAPSHOT = "execution_health_snapshot"
    RAW_DERIBIT_OPTION_SNAPSHOT = "raw_deribit_option_snapshot"
    INGESTION_RUNS = "ingestion_runs"


class ReadMode(str, Enum):
    LATEST = "latest"
    ALL_RUNS = "all_runs"


class WriteMode(str, Enum):
    IMMUTABLE = "immutable"


class RunStatus(str, Enum):
    SUCCESS = "success"
    GAP = "gap"
    FAILED = "failed"
    SKIPPED_EXISTING = "skipped_existing"


@dataclass(frozen=True)
class DatasetSpec:
    name: DatasetName
    version: int
    schema: dict[str, Any]
    partition_columns: tuple[str, ...]
    primary_key: tuple[str, ...]
    default_read_mode: ReadMode = ReadMode.LATEST


UTC_DT = pl.Datetime(time_zone="UTC")

STORAGE_METADATA_SCHEMA: dict[str, Any] = {
    "run_id": pl.String,
    "run_ts": UTC_DT,
    "logical_ts": UTC_DT,
    "write_mode": pl.String,
}


def _with_storage_metadata(schema: dict[str, Any]) -> dict[str, Any]:
    return {**schema, **STORAGE_METADATA_SCHEMA}


DATASET_SPECS: dict[DatasetName, DatasetSpec] = {
    DatasetName.MICROSTRUCTURE_RAW_EVENT: DatasetSpec(
        name=DatasetName.MICROSTRUCTURE_RAW_EVENT,
        version=3,
        partition_columns=("asset", "venue", "event_type", "date"),
        primary_key=("ts", "asset", "venue", "event_type", "event_id"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "asset": pl.String,
                "venue": pl.String,
                "event_type": pl.String,
                "event_id": pl.String,
                "exchange_symbol": pl.String,
                "venue_group": pl.String,
                "market_type": pl.String,
                "is_derivatives": pl.Boolean,
                "contract_type": pl.String,
                "exchange_ts": UTC_DT,
                "received_ts": UTC_DT,
                "local_ts": UTC_DT,
                "transaction_ts": UTC_DT,
                "side": pl.String,
                "liquidation_side": pl.String,
                "price": pl.Float64,
                "size": pl.Float64,
                "trade_notional": pl.Float64,
                "bid": pl.Float64,
                "ask": pl.Float64,
                "mark_price": pl.Float64,
                "open_interest": pl.Float64,
                "depth_notional_bid_5": pl.Float64,
                "depth_notional_ask_5": pl.Float64,
                "first_update_id": pl.UInt64,
                "last_update_id": pl.UInt64,
                "payload": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.MICROSTRUCTURE_BOOK_STATE: DatasetSpec(
        name=DatasetName.MICROSTRUCTURE_BOOK_STATE,
        version=3,
        partition_columns=("asset", "venue", "date"),
        primary_key=("ts", "asset", "venue"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "asset": pl.String,
                "venue": pl.String,
                "exchange_symbol": pl.String,
                "venue_group": pl.String,
                "market_type": pl.String,
                "is_derivatives": pl.Boolean,
                "contract_type": pl.String,
                "best_bid": pl.Float64,
                "best_ask": pl.Float64,
                "best_bid_size": pl.Float64,
                "best_ask_size": pl.Float64,
                "mid_price": pl.Float64,
                "microprice": pl.Float64,
                "spread": pl.Float64,
                "spread_rel": pl.Float64,
                "depth_bid_5": pl.Float64,
                "depth_ask_5": pl.Float64,
                "depth_notional_bid_5": pl.Float64,
                "depth_notional_ask_5": pl.Float64,
                "imbalance_5": pl.Float64,
                "is_seeded": pl.Boolean,
                "is_sequence_valid": pl.Boolean,
                "is_gap_detected": pl.Boolean,
                "reseed_count": pl.Int64,
                "book_quality": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.MICROSTRUCTURE_BAR: DatasetSpec(
        name=DatasetName.MICROSTRUCTURE_BAR,
        version=2,
        partition_columns=("asset", "venue", "clock", "date"),
        primary_key=("ts", "asset", "venue", "clock", "bar_size_secs"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "asset": pl.String,
                "venue": pl.String,
                "exchange_symbol": pl.String,
                "venue_group": pl.String,
                "market_type": pl.String,
                "is_derivatives": pl.Boolean,
                "contract_type": pl.String,
                "clock": pl.String,
                "bar_size_secs": pl.Int64,
                "open": pl.Float64,
                "high": pl.Float64,
                "low": pl.Float64,
                "close": pl.Float64,
                "volume": pl.Float64,
                "buy_volume": pl.Float64,
                "sell_volume": pl.Float64,
                "trade_count": pl.Int64,
                "vwap": pl.Float64,
                "microprice": pl.Float64,
                "spread": pl.Float64,
                "depth_bid_5": pl.Float64,
                "depth_ask_5": pl.Float64,
                "depth_notional_bid_5": pl.Float64,
                "depth_notional_ask_5": pl.Float64,
                "is_gap": pl.Boolean,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.MICROSTRUCTURE_FEATURE_SERIES: DatasetSpec(
        name=DatasetName.MICROSTRUCTURE_FEATURE_SERIES,
        version=2,
        partition_columns=("asset", "horizon", "source", "date"),
        primary_key=("ts", "asset", "horizon", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "asset": pl.String,
                "horizon": pl.String,
                "source": pl.String,
                "bar_size_secs": pl.Int64,
                "mid_price": pl.Float64,
                "return_1": pl.Float64,
                "return_5": pl.Float64,
                "realized_vol": pl.Float64,
                "book_imbalance": pl.Float64,
                "microprice_tilt": pl.Float64,
                "order_flow_imbalance": pl.Float64,
                "cumulative_delta": pl.Float64,
                "cum_delta_10": pl.Float64,
                "cum_delta_30": pl.Float64,
                "cum_delta_50": pl.Float64,
                "basis_bps": pl.Float64,
                "funding_rate": pl.Float64,
                "oi_value": pl.Float64,
                "liquidation_volume": pl.Float64,
                "iv_atm": pl.Float64,
                "iv_skew_25d": pl.Float64,
                "cross_venue_spread": pl.Float64,
                "iv_rv_gap": pl.Float64,
                "feature_count": pl.Int64,
                "feature_payload": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.MICROSTRUCTURE_LABEL_SERIES: DatasetSpec(
        name=DatasetName.MICROSTRUCTURE_LABEL_SERIES,
        version=3,
        partition_columns=("asset", "horizon", "source", "date"),
        primary_key=("ts", "asset", "horizon", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "asset": pl.String,
                "horizon": pl.String,
                "source": pl.String,
                "forward_microprice_return_bps": pl.Float64,
                "forward_return_bps": pl.Float64,
                "forward_move_abs_bps": pl.Float64,
                "label": pl.Int64,
                "tau_bps": pl.Float64,
                "bar_size_secs": pl.Int64,
                "label_horizon_steps": pl.Int64,
                "horizon_seconds": pl.Int64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.MICROSTRUCTURE_MODEL_OUTPUT: DatasetSpec(
        name=DatasetName.MICROSTRUCTURE_MODEL_OUTPUT,
        version=2,
        partition_columns=("asset", "horizon", "source", "date"),
        primary_key=("ts", "asset", "horizon", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "asset": pl.String,
                "horizon": pl.String,
                "source": pl.String,
                "prob_down": pl.Float64,
                "prob_flat": pl.Float64,
                "prob_up": pl.Float64,
                "expected_move_bps": pl.Float64,
                "expected_edge_bps": pl.Float64,
                "tradability_score": pl.Float64,
                "confidence": pl.Float64,
                "confidence_bucket": pl.String,
                "regime_state": pl.String,
                "blocked_reason": pl.String,
                "model_version": pl.String,
                "model_family": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.MICROSTRUCTURE_REGIME_SERIES: DatasetSpec(
        name=DatasetName.MICROSTRUCTURE_REGIME_SERIES,
        version=1,
        partition_columns=("asset", "source", "date"),
        primary_key=("ts", "asset", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "asset": pl.String,
                "source": pl.String,
                "regime_state": pl.String,
                "vol_percentile": pl.Float64,
                "flow_percentile": pl.Float64,
                "basis_percentile": pl.Float64,
                "persistence_hours": pl.Float64,
                "hysteresis_state": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.MICROSTRUCTURE_SIGNAL_SERIES: DatasetSpec(
        name=DatasetName.MICROSTRUCTURE_SIGNAL_SERIES,
        version=2,
        partition_columns=("asset", "horizon", "source", "date"),
        primary_key=("ts", "asset", "horizon", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "asset": pl.String,
                "horizon": pl.String,
                "source": pl.String,
                "signal_state": pl.String,
                "paper_position": pl.String,
                "expected_edge_bps": pl.Float64,
                "cost_filter_passed": pl.Boolean,
                "quality_filter_passed": pl.Boolean,
                "blocked_reason": pl.String,
                "confidence": pl.Float64,
                "tradability_score": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.MICROSTRUCTURE_TRAINING_RUN: DatasetSpec(
        name=DatasetName.MICROSTRUCTURE_TRAINING_RUN,
        version=4,
        partition_columns=("asset", "horizon", "source", "date"),
        primary_key=("ts", "asset", "horizon", "source", "model_version"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "asset": pl.String,
                "horizon": pl.String,
                "source": pl.String,
                "model_version": pl.String,
                "train_start": UTC_DT,
                "train_end": UTC_DT,
                "split_count": pl.Int64,
                "cv_accuracy_mean": pl.Float64,
                "cv_accuracy_std": pl.Float64,
                "brier_score": pl.Float64,
                "calibration_error": pl.Float64,
                "lift_vs_tabular": pl.Float64,
                "profitable": pl.Boolean,
                "artifact_uri": pl.String,
                "artifact_payload": pl.String,
                "feature_schema_version": pl.Int64,
                "label_schema_version": pl.Int64,
                "validation_embargo_bars": pl.Int64,
                "tabular_best_iteration": pl.Int64,
                "fusion_best_iteration": pl.Int64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.MICROSTRUCTURE_BACKTEST_RUN: DatasetSpec(
        name=DatasetName.MICROSTRUCTURE_BACKTEST_RUN,
        version=3,
        partition_columns=("asset", "horizon", "source", "date"),
        primary_key=("ts", "asset", "horizon", "source", "model_version"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "asset": pl.String,
                "horizon": pl.String,
                "source": pl.String,
                "model_version": pl.String,
                "train_start": UTC_DT,
                "train_end": UTC_DT,
                "test_start": UTC_DT,
                "test_end": UTC_DT,
                "split_count": pl.Int64,
                "mean_accuracy": pl.Float64,
                "mean_brier_score": pl.Float64,
                "paper_sharpe": pl.Float64,
                "paper_net_edge_bps": pl.Float64,
                "directional_f1": pl.Float64,
                "feature_schema_version": pl.Int64,
                "label_schema_version": pl.Int64,
                "validation_embargo_bars": pl.Int64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTION_CHAIN_SNAPSHOT: DatasetSpec(
        name=DatasetName.OPTION_CHAIN_SNAPSHOT,
        version=3,
        partition_columns=("venue", "underlying", "date"),
        primary_key=("ts", "instrument_name"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "venue": pl.String,
                "underlying": pl.String,
                "expiry": UTC_DT,
                "instrument_name": pl.String,
                "option_type": pl.String,
                "strike": pl.Float64,
                "bid": pl.Float64,
                "ask": pl.Float64,
                "mark": pl.Float64,
                "mid": pl.Float64,
                "mark_iv": pl.Float64,
                "delta": pl.Float64,
                "gamma": pl.Float64,
                "vega": pl.Float64,
                "index_price": pl.Float64,
                "underlying_price": pl.Float64,
                "forward": pl.Float64,
                "interest_rate": pl.Float64,
                "open_interest": pl.Float64,
                "volume": pl.Float64,
                "quote_age_ms": pl.Int64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.DVOL_SNAPSHOT: DatasetSpec(
        name=DatasetName.DVOL_SNAPSHOT,
        version=2,
        partition_columns=("venue", "underlying", "date"),
        primary_key=("ts", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "venue": pl.String,
                "underlying": pl.String,
                "source": pl.String,
                "dvol_open": pl.Float64,
                "dvol_high": pl.Float64,
                "dvol_low": pl.Float64,
                "dvol_close": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.FUNDING_RATE_SNAPSHOT: DatasetSpec(
        name=DatasetName.FUNDING_RATE_SNAPSHOT,
        version=1,
        partition_columns=("venue", "underlying", "date"),
        primary_key=("ts", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "venue": pl.String,
                "underlying": pl.String,
                "source": pl.String,
                "interest_1h": pl.Float64,
                "interest_8h": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.VOL_INDEX_SERIES: DatasetSpec(
        name=DatasetName.VOL_INDEX_SERIES,
        version=3,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "value": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.SURFACE_FEATURES: DatasetSpec(
        name=DatasetName.SURFACE_FEATURES,
        version=4,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "atm_iv_7d": pl.Float64,
                "atm_iv_14d": pl.Float64,
                "atm_iv_30d": pl.Float64,
                "atm_iv_60d": pl.Float64,
                "rr_25_7d": pl.Float64,
                "rr_25_30d": pl.Float64,
                "bf_25_7d": pl.Float64,
                "bf_25_30d": pl.Float64,
                "term_slope_7_30": pl.Float64,
                "term_slope_30_60": pl.Float64,
                "fit_quality": pl.Float64,
                "coverage_ratio": pl.Float64,
                "maturity_violations": pl.Int64,
                "fit_validity_flag": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.VVIX_SERIES: DatasetSpec(
        name=DatasetName.VVIX_SERIES,
        version=3,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "vol_index_value": pl.Float64,
                "rvvix_7d": pl.Float64,
                "rvvix_14d": pl.Float64,
                "rvvix_30d": pl.Float64,
                "qvvix_30d": pl.Float64,
                "model_version": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.RAZOR_SERIES: DatasetSpec(
        name=DatasetName.RAZOR_SERIES,
        version=2,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "reference_tenor_days": pl.Int64,
                "razor_pnl": pl.Float64,
                "razor_vrp": pl.Float64,
                "buyer_increment": pl.Float64,
                "seller_increment": pl.Float64,
                "option_reval_pnl": pl.Float64,
                "hedge_pnl": pl.Float64,
                "roll_pnl": pl.Float64,
                "razor_pnl_zscore": pl.Float64,
                "razor_vrp_zscore": pl.Float64,
                "trade_band": pl.String,
                "long_vol_signal": pl.Boolean,
                "short_vol_signal": pl.Boolean,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTIONS_SURFACE_SNAPSHOT: DatasetSpec(
        name=DatasetName.OPTIONS_SURFACE_SNAPSHOT,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source", "tenor_bucket", "moneyness_bucket"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "tenor_bucket": pl.String,
                "moneyness_bucket": pl.String,
                "mean_mark_iv": pl.Float64,
                "mean_mid": pl.Float64,
                "mean_spread": pl.Float64,
                "option_count": pl.Int64,
                "open_interest": pl.Float64,
                "volume": pl.Float64,
                "iv_change_1h": pl.Float64,
                "iv_change_24h": pl.Float64,
                "iv_change_7d": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTIONS_CARRY_SERIES: DatasetSpec(
        name=DatasetName.OPTIONS_CARRY_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "atm_iv_7d": pl.Float64,
                "atm_iv_14d": pl.Float64,
                "atm_iv_30d": pl.Float64,
                "realized_vol_7d": pl.Float64,
                "realized_vol_14d": pl.Float64,
                "realized_vol_30d": pl.Float64,
                "carry_spread_7d": pl.Float64,
                "carry_spread_30d": pl.Float64,
                "term_carry_7_30": pl.Float64,
                "roll_down_carry": pl.Float64,
                "carry_zscore": pl.Float64,
                "carry_regime": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTIONS_SKEW_SERIES: DatasetSpec(
        name=DatasetName.OPTIONS_SKEW_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "rr_25_7d": pl.Float64,
                "rr_25_30d": pl.Float64,
                "bf_25_7d": pl.Float64,
                "bf_25_30d": pl.Float64,
                "skew_slope": pl.Float64,
                "skew_change_1h": pl.Float64,
                "skew_zscore": pl.Float64,
                "spot_move_24h": pl.Float64,
                "crash_premium_regime": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTIONS_DISLOCATION_SERIES: DatasetSpec(
        name=DatasetName.OPTIONS_DISLOCATION_SERIES,
        version=2,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source", "tenor_bucket"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "tenor_bucket": pl.String,
                "atm_residual": pl.Float64,
                "wing_residual": pl.Float64,
                "max_residual": pl.Float64,
                "mean_abs_residual": pl.Float64,
                "dispersion_score": pl.Float64,
                "residual_p90": pl.Float64,
                "rich_strike_count": pl.Int64,
                "cheap_strike_count": pl.Int64,
                "dislocation_zscore": pl.Float64,
                "fit_validity_flag": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTIONS_RELATIVE_VALUE_SERIES: DatasetSpec(
        name=DatasetName.OPTIONS_RELATIVE_VALUE_SERIES,
        version=1,
        partition_columns=("pair_id", "source", "date"),
        primary_key=("ts", "pair_id", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "pair_id": pl.String,
                "source": pl.String,
                "atm_iv_spread_7d": pl.Float64,
                "atm_iv_spread_30d": pl.Float64,
                "skew_spread_30d": pl.Float64,
                "vvix_spread": pl.Float64,
                "razor_spread": pl.Float64,
                "carry_spread": pl.Float64,
                "funding_spread": pl.Float64,
                "spread_zscore": pl.Float64,
                "relative_regime": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTIONS_POSITIONING_SERIES: DatasetSpec(
        name=DatasetName.OPTIONS_POSITIONING_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source", "expiry_bucket", "moneyness_bucket"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "expiry_bucket": pl.String,
                "moneyness_bucket": pl.String,
                "open_interest": pl.Float64,
                "volume": pl.Float64,
                "open_interest_change": pl.Float64,
                "open_interest_share": pl.Float64,
                "volume_share": pl.Float64,
                "atm_pinning_concentration": pl.Float64,
                "wing_concentration": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTIONS_SURFACE_FACTOR_SERIES: DatasetSpec(
        name=DatasetName.OPTIONS_SURFACE_FACTOR_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source", "tenor_bucket", "moneyness_bucket"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "tenor_bucket": pl.String,
                "moneyness_bucket": pl.String,
                "liquidity_weight": pl.Float64,
                "mean_node_return_1h": pl.Float64,
                "factor_return_1h": pl.Float64,
                "factor_momentum_24h": pl.Float64,
                "node_residual": pl.Float64,
                "node_residual_zscore": pl.Float64,
                "residual_value_score": pl.Float64,
                "factor_regime": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTIONS_ML_CARRY_SERIES: DatasetSpec(
        name=DatasetName.OPTIONS_ML_CARRY_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "forecast_rv_7d": pl.Float64,
                "gross_carry_spread": pl.Float64,
                "jump_risk_score": pl.Float64,
                "toxicity_score": pl.Float64,
                "expected_hedge_cost": pl.Float64,
                "spread_cost": pl.Float64,
                "funding_drag": pl.Float64,
                "gross_carry_score": pl.Float64,
                "net_carry_score": pl.Float64,
                "carry_confidence": pl.Float64,
                "carry_veto": pl.Boolean,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTIONS_EXPIRY_FLOW_SERIES: DatasetSpec(
        name=DatasetName.OPTIONS_EXPIRY_FLOW_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source", "expiry_bucket", "strike_bucket"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "expiry_bucket": pl.String,
                "strike_bucket": pl.String,
                "pain_strike": pl.Float64,
                "dominant_strike": pl.Float64,
                "distance_to_pain_pct": pl.Float64,
                "local_oi_concentration": pl.Float64,
                "gamma_exposure": pl.Float64,
                "vanna_exposure": pl.Float64,
                "charm_exposure": pl.Float64,
                "pinning_score": pl.Float64,
                "breakout_score": pl.Float64,
                "flow_regime": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTIONS_ADAPTIVE_HEDGE_SERIES: DatasetSpec(
        name=DatasetName.OPTIONS_ADAPTIVE_HEDGE_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "smile_aware_delta": pl.Float64,
                "min_var_delta": pl.Float64,
                "hedge_band_width": pl.Float64,
                "expected_hedge_bleed_bps": pl.Float64,
                "basis_mismatch_bps": pl.Float64,
                "funding_drag": pl.Float64,
                "route_score": pl.Float64,
                "route_preference": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTIONS_SESSIONALITY_SERIES: DatasetSpec(
        name=DatasetName.OPTIONS_SESSIONALITY_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source", "session_bucket"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "session_bucket": pl.String,
                "is_weekend": pl.Boolean,
                "is_funding_window": pl.Boolean,
                "is_expiry_window": pl.Boolean,
                "session_realized_vol": pl.Float64,
                "session_implied_vol": pl.Float64,
                "session_carry": pl.Float64,
                "sessionality_score": pl.Float64,
                "carry_switch": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTIONS_DISPERSION_SERIES: DatasetSpec(
        name=DatasetName.OPTIONS_DISPERSION_SERIES,
        version=1,
        partition_columns=("pair_id", "source", "date"),
        primary_key=("ts", "pair_id", "source", "tenor"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "pair_id": pl.String,
                "source": pl.String,
                "tenor": pl.String,
                "btc_atm_iv": pl.Float64,
                "eth_atm_iv": pl.Float64,
                "implied_correlation": pl.Float64,
                "realized_correlation": pl.Float64,
                "realized_corr_forecast": pl.Float64,
                "corr_gap": pl.Float64,
                "rr_richness_spread": pl.Float64,
                "bf_richness_spread": pl.Float64,
                "cross_smile_consistency": pl.Float64,
                "dispersion_score": pl.Float64,
                "dispersion_regime": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTIONS_FUNDING_SKEW_SERIES: DatasetSpec(
        name=DatasetName.OPTIONS_FUNDING_SKEW_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "basis_bps": pl.Float64,
                "funding_rate": pl.Float64,
                "funding_rate_zscore": pl.Float64,
                "rr_25_7d": pl.Float64,
                "bf_25_7d": pl.Float64,
                "skew_richness": pl.Float64,
                "basis_pressure": pl.Float64,
                "funding_basis_skew_score": pl.Float64,
                "regime": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.OPTIONS_SIGNAL_SERIES: DatasetSpec(
        name=DatasetName.OPTIONS_SIGNAL_SERIES,
        version=4,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "vol_carry_score": pl.Float64,
                "skew_stress_score": pl.Float64,
                "surface_dislocation_score": pl.Float64,
                "cross_asset_vol_score": pl.Float64,
                "funding_vol_context_score": pl.Float64,
                "breakeven_gap_score": pl.Float64,
                "term_structure_momentum_score": pl.Float64,
                "surface_factor_score": pl.Float64,
                "residual_surface_rv_score": pl.Float64,
                "ml_carry_toxicity_score": pl.Float64,
                "expiry_pin_score": pl.Float64,
                "gamma_flip_score": pl.Float64,
                "adaptive_hedge_score": pl.Float64,
                "sessionality_score": pl.Float64,
                "dispersion_score": pl.Float64,
                "funding_skew_score": pl.Float64,
                "vol_regime_score": pl.Float64,
                "vol_carry_long_threshold": pl.Float64,
                "vol_carry_short_threshold": pl.Float64,
                "skew_stress_long_threshold": pl.Float64,
                "skew_stress_short_threshold": pl.Float64,
                "surface_dislocation_long_threshold": pl.Float64,
                "surface_dislocation_short_threshold": pl.Float64,
                "cross_asset_vol_long_threshold": pl.Float64,
                "cross_asset_vol_short_threshold": pl.Float64,
                "funding_vol_context_long_threshold": pl.Float64,
                "funding_vol_context_short_threshold": pl.Float64,
                "breakeven_gap_long_threshold": pl.Float64,
                "breakeven_gap_short_threshold": pl.Float64,
                "term_structure_momentum_long_threshold": pl.Float64,
                "term_structure_momentum_short_threshold": pl.Float64,
                "surface_factor_long_threshold": pl.Float64,
                "surface_factor_short_threshold": pl.Float64,
                "residual_surface_rv_long_threshold": pl.Float64,
                "residual_surface_rv_short_threshold": pl.Float64,
                "ml_carry_toxicity_long_threshold": pl.Float64,
                "ml_carry_toxicity_short_threshold": pl.Float64,
                "expiry_pin_long_threshold": pl.Float64,
                "expiry_pin_short_threshold": pl.Float64,
                "gamma_flip_long_threshold": pl.Float64,
                "gamma_flip_short_threshold": pl.Float64,
                "adaptive_hedge_long_threshold": pl.Float64,
                "adaptive_hedge_short_threshold": pl.Float64,
                "sessionality_long_threshold": pl.Float64,
                "sessionality_short_threshold": pl.Float64,
                "dispersion_long_threshold": pl.Float64,
                "dispersion_short_threshold": pl.Float64,
                "funding_skew_long_threshold": pl.Float64,
                "funding_skew_short_threshold": pl.Float64,
                "vol_regime_long_threshold": pl.Float64,
                "vol_regime_short_threshold": pl.Float64,
                "signal_state": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.VOL_CONE_SERIES: DatasetSpec(
        name=DatasetName.VOL_CONE_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source", "tenor"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "tenor": pl.String,
                "implied_vol": pl.Float64,
                "realized_vol": pl.Float64,
                "iv_rv_spread": pl.Float64,
                "implied_percentile": pl.Float64,
                "realized_percentile": pl.Float64,
                "iv_rv_spread_percentile": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.TERM_STRUCTURE_MOMENTUM_SERIES: DatasetSpec(
        name=DatasetName.TERM_STRUCTURE_MOMENTUM_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "term_slope_7_30": pl.Float64,
                "term_slope_30_60": pl.Float64,
                "term_slope_change_24h": pl.Float64,
                "term_slope_change_72h": pl.Float64,
                "momentum_regime": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.GAMMA_EXPOSURE_SERIES: DatasetSpec(
        name=DatasetName.GAMMA_EXPOSURE_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source", "expiry_bucket", "strike_bucket"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "expiry_bucket": pl.String,
                "strike_bucket": pl.String,
                "gamma_exposure": pl.Float64,
                "gamma_share": pl.Float64,
                "pinning_concentration": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.FUNDING_CONTEXT_SERIES: DatasetSpec(
        name=DatasetName.FUNDING_CONTEXT_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "funding_rate": pl.Float64,
                "funding_rate_zscore": pl.Float64,
                "funding_change_24h": pl.Float64,
                "rvvix_14d": pl.Float64,
                "qvvix_30d": pl.Float64,
                "funding_regime": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.STRADDLE_BREAKEVEN_SERIES: DatasetSpec(
        name=DatasetName.STRADDLE_BREAKEVEN_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "atm_iv_7d": pl.Float64,
                "breakeven_move_pct": pl.Float64,
                "realized_move_7d_pct": pl.Float64,
                "breakeven_gap_pct": pl.Float64,
                "breakeven_regime": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.VOL_REGIME_TRANSITION_SERIES: DatasetSpec(
        name=DatasetName.VOL_REGIME_TRANSITION_SERIES,
        version=1,
        partition_columns=("underlying", "source", "date"),
        primary_key=("ts", "underlying", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "underlying": pl.String,
                "source": pl.String,
                "signal_state": pl.String,
                "prior_state": pl.String,
                "next_state_probability_long_vol_bias": pl.Float64,
                "next_state_probability_neutral": pl.Float64,
                "next_state_probability_short_vol_bias": pl.Float64,
                "median_state_duration_hours": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.PERPS_SIGNAL_SERIES: DatasetSpec(
        name=DatasetName.PERPS_SIGNAL_SERIES,
        version=1,
        partition_columns=("asset", "venue_scope", "source", "date"),
        primary_key=("ts", "asset", "venue_scope", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "asset": pl.String,
                "venue_scope": pl.String,
                "source": pl.String,
                "basis_bps": pl.Float64,
                "funding_rate_1h": pl.Float64,
                "carry_annualized_pct": pl.Float64,
                "carry_zscore_7d": pl.Float64,
                "carry_zscore_30d": pl.Float64,
                "carry_momentum_1h": pl.Float64,
                "carry_momentum_24h": pl.Float64,
                "cross_venue_funding_spread": pl.Float64,
                "long_liq_notional": pl.Float64,
                "short_liq_notional": pl.Float64,
                "liq_intensity_zscore": pl.Float64,
                "liq_asymmetry": pl.Float64,
                "oi_concentration": pl.Float64,
                "crowding_score": pl.Float64,
                "cascade_risk_score": pl.Float64,
                "cascade_regime": pl.String,
                "vpin_50": pl.Float64,
                "toxic_flow_zscore_4h": pl.Float64,
                "toxic_flow_zscore_24h": pl.Float64,
                "ofi_persistence": pl.Float64,
                "large_trade_ratio": pl.Float64,
                "toxicity_regime": pl.String,
                "oi_change_1h": pl.Float64,
                "oi_change_4h": pl.Float64,
                "oi_change_24h": pl.Float64,
                "price_change_1h": pl.Float64,
                "oi_price_divergence": pl.Float64,
                "oi_acceleration": pl.Float64,
                "leverage_regime": pl.String,
                "lead_lag_score": pl.Float64,
                "venue_dislocation_bps": pl.Float64,
                "convergence_half_life_secs": pl.Float64,
                "price_discovery_leader": pl.String,
                "depth_recovery_speed": pl.Float64,
                "spread_bps": pl.Float64,
                "spread_zscore_4h": pl.Float64,
                "spread_zscore_24h": pl.Float64,
                "depth_imbalance_momentum": pl.Float64,
                "spread_regime": pl.String,
                "rv_5s": pl.Float64,
                "rv_1m": pl.Float64,
                "rv_5m": pl.Float64,
                "vol_signature_ratio": pl.Float64,
                "vol_clustering": pl.Float64,
                "rv_regime": pl.String,
                "cum_delta_acceleration": pl.Float64,
                "delta_divergence": pl.Float64,
                "absorption_ratio": pl.Float64,
                "delta_regime": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.PERPS_COMPOSITE_SERIES: DatasetSpec(
        name=DatasetName.PERPS_COMPOSITE_SERIES,
        version=1,
        partition_columns=("asset", "venue_scope", "source", "date"),
        primary_key=("ts", "asset", "venue_scope", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "asset": pl.String,
                "venue_scope": pl.String,
                "source": pl.String,
                "carry_score": pl.Float64,
                "cascade_risk_score": pl.Float64,
                "toxicity_score": pl.Float64,
                "positioning_score": pl.Float64,
                "fragility_score": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.PERPS_TRANSITION_SERIES: DatasetSpec(
        name=DatasetName.PERPS_TRANSITION_SERIES,
        version=1,
        partition_columns=("asset", "venue_scope", "source", "signal_family", "date"),
        primary_key=("ts", "asset", "venue_scope", "source", "signal_family"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "asset": pl.String,
                "venue_scope": pl.String,
                "source": pl.String,
                "signal_family": pl.String,
                "current_state": pl.String,
                "prior_state": pl.String,
                "next_state_distribution": pl.String,
                "median_state_duration_minutes": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.COMPOSITE_SIGNAL_SERIES: DatasetSpec(
        name=DatasetName.COMPOSITE_SIGNAL_SERIES,
        version=1,
        partition_columns=("scope", "asset", "source", "date"),
        primary_key=("ts", "scope", "asset", "source"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "scope": pl.String,
                "asset": pl.String,
                "source": pl.String,
                "sentiment_index": pl.Float64,
                "directional_tilt": pl.Float64,
                "fragility_index": pl.Float64,
                "confidence_score": pl.Float64,
                "options_sentiment_subindex": pl.Float64,
                "perps_sentiment_subindex": pl.Float64,
                "micro_sentiment_subindex": pl.Float64,
                "micro_direction_subindex": pl.Float64,
                "perps_direction_subindex": pl.Float64,
                "options_direction_subindex": pl.Float64,
                "perps_fragility_subindex": pl.Float64,
                "options_fragility_subindex": pl.Float64,
                "micro_fragility_subindex": pl.Float64,
                "coverage_ratio": pl.Float64,
                "freshness_score": pl.Float64,
                "calibrator_mode": pl.String,
                "sentiment_regime": pl.String,
                "directional_regime": pl.String,
                "fragility_regime": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.COMPOSITE_COMPONENT_SERIES: DatasetSpec(
        name=DatasetName.COMPOSITE_COMPONENT_SERIES,
        version=1,
        partition_columns=("scope", "asset", "source", "date"),
        primary_key=("ts", "scope", "asset", "source", "pillar", "component"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "scope": pl.String,
                "asset": pl.String,
                "source": pl.String,
                "pillar": pl.String,
                "family": pl.String,
                "component": pl.String,
                "raw_value": pl.Float64,
                "normalized_value": pl.Float64,
                "component_weight": pl.Float64,
                "pillar_score": pl.Float64,
                "pillar_weight": pl.Float64,
                "signed_contribution": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.COMPOSITE_TRANSITION_SERIES: DatasetSpec(
        name=DatasetName.COMPOSITE_TRANSITION_SERIES,
        version=1,
        partition_columns=("scope", "asset", "source", "date"),
        primary_key=("ts", "scope", "asset", "source", "index_type"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "scope": pl.String,
                "asset": pl.String,
                "source": pl.String,
                "index_type": pl.String,
                "current_regime": pl.String,
                "prior_regime": pl.String,
                "stay_probability": pl.Float64,
                "upshift_probability": pl.Float64,
                "downshift_probability": pl.Float64,
                "median_regime_duration_hours": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.EXECUTION_ORDER_INTENT: DatasetSpec(
        name=DatasetName.EXECUTION_ORDER_INTENT,
        version=1,
        partition_columns=("venue", "account", "date"),
        primary_key=("request_id",),
        default_read_mode=ReadMode.ALL_RUNS,
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "request_id": pl.String,
                "idempotency_key": pl.String,
                "venue": pl.String,
                "account": pl.String,
                "instrument_type": pl.String,
                "asset": pl.String,
                "symbol": pl.String,
                "side": pl.String,
                "order_type": pl.String,
                "time_in_force": pl.String,
                "price": pl.Float64,
                "size": pl.Float64,
                "notional_usd": pl.Float64,
                "status": pl.String,
                "dry_run": pl.Boolean,
                "paper_mode": pl.Boolean,
                "payload": pl.String,
                "error_code": pl.String,
                "error_message": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.EXECUTION_ORDER_EVENT: DatasetSpec(
        name=DatasetName.EXECUTION_ORDER_EVENT,
        version=1,
        partition_columns=("venue", "account", "date"),
        primary_key=("ts", "request_id", "venue_order_id", "event_type"),
        default_read_mode=ReadMode.ALL_RUNS,
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "request_id": pl.String,
                "venue_order_id": pl.String,
                "client_order_id": pl.String,
                "venue": pl.String,
                "account": pl.String,
                "asset": pl.String,
                "symbol": pl.String,
                "event_type": pl.String,
                "status": pl.String,
                "side": pl.String,
                "order_type": pl.String,
                "time_in_force": pl.String,
                "price": pl.Float64,
                "size": pl.Float64,
                "remaining_size": pl.Float64,
                "filled_size": pl.Float64,
                "avg_fill_price": pl.Float64,
                "reason": pl.String,
                "payload": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.EXECUTION_FILL_EVENT: DatasetSpec(
        name=DatasetName.EXECUTION_FILL_EVENT,
        version=1,
        partition_columns=("venue", "account", "date"),
        primary_key=("fill_id",),
        default_read_mode=ReadMode.ALL_RUNS,
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "fill_id": pl.String,
                "request_id": pl.String,
                "venue_order_id": pl.String,
                "venue": pl.String,
                "account": pl.String,
                "asset": pl.String,
                "symbol": pl.String,
                "side": pl.String,
                "price": pl.Float64,
                "size": pl.Float64,
                "fee": pl.Float64,
                "fee_asset": pl.String,
                "liquidity": pl.String,
                "payload": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.EXECUTION_POSITION_SNAPSHOT: DatasetSpec(
        name=DatasetName.EXECUTION_POSITION_SNAPSHOT,
        version=1,
        partition_columns=("venue", "account", "date"),
        primary_key=("ts", "venue", "account", "symbol"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "venue": pl.String,
                "account": pl.String,
                "asset": pl.String,
                "symbol": pl.String,
                "instrument_type": pl.String,
                "position_side": pl.String,
                "quantity": pl.Float64,
                "entry_price": pl.Float64,
                "mark_price": pl.Float64,
                "unrealized_pnl": pl.Float64,
                "realized_pnl": pl.Float64,
                "notional_usd": pl.Float64,
                "leverage": pl.Float64,
                "margin_mode": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.EXECUTION_BALANCE_SNAPSHOT: DatasetSpec(
        name=DatasetName.EXECUTION_BALANCE_SNAPSHOT,
        version=1,
        partition_columns=("venue", "account", "date"),
        primary_key=("ts", "venue", "account", "asset", "balance_type"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "venue": pl.String,
                "account": pl.String,
                "asset": pl.String,
                "balance_type": pl.String,
                "total": pl.Float64,
                "available": pl.Float64,
                "locked": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.EXECUTION_QUOTE_SNAPSHOT: DatasetSpec(
        name=DatasetName.EXECUTION_QUOTE_SNAPSHOT,
        version=1,
        partition_columns=("venue", "account", "date"),
        primary_key=("request_id", "quote_id"),
        default_read_mode=ReadMode.ALL_RUNS,
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "request_id": pl.String,
                "quote_id": pl.String,
                "venue": pl.String,
                "account": pl.String,
                "asset": pl.String,
                "symbol": pl.String,
                "side": pl.String,
                "price": pl.Float64,
                "size": pl.Float64,
                "notional_usd": pl.Float64,
                "expires_at": UTC_DT,
                "route": pl.String,
                "status": pl.String,
                "payload": pl.String,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.EXECUTION_MARKET_SNAPSHOT: DatasetSpec(
        name=DatasetName.EXECUTION_MARKET_SNAPSHOT,
        version=1,
        partition_columns=("venue", "asset", "date"),
        primary_key=("ts", "venue", "symbol"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "venue": pl.String,
                "asset": pl.String,
                "symbol": pl.String,
                "instrument_type": pl.String,
                "status": pl.String,
                "tick_size": pl.Float64,
                "step_size": pl.Float64,
                "min_size": pl.Float64,
                "min_notional": pl.Float64,
                "best_bid": pl.Float64,
                "best_ask": pl.Float64,
                "mark_price": pl.Float64,
                "index_price": pl.Float64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.EXECUTION_HEALTH_SNAPSHOT: DatasetSpec(
        name=DatasetName.EXECUTION_HEALTH_SNAPSHOT,
        version=1,
        partition_columns=("venue", "account", "date"),
        primary_key=("ts", "venue", "account"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "venue": pl.String,
                "account": pl.String,
                "status": pl.String,
                "ws_connected": pl.Boolean,
                "private_ws_connected": pl.Boolean,
                "last_heartbeat_ms": pl.Int64,
                "latency_ms": pl.Int64,
                "last_error": pl.String,
                "reconnect_count": pl.Int64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.RAW_DERIBIT_OPTION_SNAPSHOT: DatasetSpec(
        name=DatasetName.RAW_DERIBIT_OPTION_SNAPSHOT,
        version=1,
        partition_columns=("venue", "underlying", "date"),
        primary_key=("logical_ts", "venue", "underlying"),
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "venue": pl.String,
                "underlying": pl.String,
                "payload": pl.String,
                "instrument_count": pl.Int64,
                "quality_flag": pl.String,
            }
        ),
    ),
    DatasetName.INGESTION_RUNS: DatasetSpec(
        name=DatasetName.INGESTION_RUNS,
        version=1,
        partition_columns=("dataset", "date"),
        primary_key=("run_id",),
        default_read_mode=ReadMode.ALL_RUNS,
        schema=_with_storage_metadata(
            {
                "ts": UTC_DT,
                "date": pl.Date,
                "dataset": pl.String,
                "venue": pl.String,
                "underlying": pl.String,
                "status": pl.String,
                "rows_written_raw": pl.Int64,
                "rows_written_normalized": pl.Int64,
                "error_message": pl.String,
                "duration_ms": pl.Int64,
            }
        ),
    ),
}


def empty_frame(dataset: DatasetName) -> pl.DataFrame:
    spec = DATASET_SPECS[dataset]
    return pl.DataFrame(schema=spec.schema)
