from __future__ import annotations

import asyncio
import math
from collections.abc import AsyncIterator
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from threading import Lock
from time import monotonic
from typing import Any, cast

import orjson
import polars as pl

from uforia.api.models import (
    CompositeComponentRecord,
    CompositeDecompositionResponse,
    CompositeSeriesPoint,
    CompositeSignalSeriesResponse,
    CompositeSignalSummary,
    CompositeTransitionRecord,
    CompositeTransitionResponse,
    DatasetDetail,
    DatasetRowsResponse,
    DatasetSummary,
    EngineSummary,
    ExecutionAccountSummary,
    ExecutionActionResponse,
    ExecutionBalanceRecord,
    ExecutionFillRecord,
    ExecutionHealthRecord,
    ExecutionMarketSummary,
    ExecutionOrderRecord,
    ExecutionPositionRecord,
    ExecutionVenueSummary,
    MicrostructureFeatureResponse,
    MicrostructureHealth,
    MicrostructureRegimeResponse,
    MicrostructureSeriesPoint,
    MicrostructureSignalSeriesResponse,
    MicrostructureSignalSummary,
    OptionsDashboardResponse,
    OptionsDashboardSummary,
    OptionsSeriesPoint,
    OptionsSignalSeriesResponse,
    OptionsSignalSummary,
    PerpsDashboardResponse,
    PerpsDashboardSummary,
    PerpsSeriesPoint,
    PerpsSignalSeriesResponse,
    PerpsSignalSummary,
    PriceCandlePoint,
    PriceSeriesResponse,
    PriceSummary,
    RunRecord,
    RunsSummary,
    SignalSeriesPoint,
    SignalSeriesResponse,
    SignalSummary,
    StreamEvent,
    SystemHealth,
)
from uforia.api.settings import ApiSettings
from uforia.config import get_config
from uforia.contracts import (
    COMPOSITE_SIGNAL_IDS,
    EXECUTION_VENUES,
    OPTIONS_DASHBOARD_IDS,
    OPTIONS_SIGNAL_IDS,
    PERPS_DASHBOARD_IDS,
    PERPS_SIGNAL_IDS,
    RAZOR_SOURCES,
    UI_EXPOSED_DATASETS,
    VVIX_SOURCES,
    EngineId,
    ProducerId,
    SignalId,
)
from uforia.execution import (
    ExecutionCancelAllRequest,
    ExecutionCancelRequest,
    ExecutionIntent,
    ExecutionQuoteRequest,
    ExecutionService,
)
from uforia.microstructure.constants import MODEL_VERSION as MICROSTRUCTURE_MODEL_VERSION
from uforia.storage import DATASET_SPECS, DatasetName, DatasetStore, ReadMode
from uforia.storage.datasets import empty_frame


@dataclass
class CachedFrame:
    expires_at: float
    last_accessed_at: float
    frame: pl.DataFrame


PREFERRED_PRICE_VENUES: dict[str, tuple[str, ...]] = {
    "binance": ("binance_spot", "binance_futures"),
    "bybit": ("bybit_spot", "bybit_perps"),
}


class ReadCache:
    def __init__(self, ttl_seconds: float, max_entries: int = 64) -> None:
        self.ttl_seconds = ttl_seconds
        self.max_entries = max_entries
        self.entries: dict[tuple[Any, ...], CachedFrame] = {}
        self._lock = Lock()

    def _prune(self, now: float) -> None:
        expired = [key for key, entry in self.entries.items() if entry.expires_at <= now]
        for key in expired:
            self.entries.pop(key, None)

        while len(self.entries) >= self.max_entries:
            oldest_key = min(
                self.entries,
                key=lambda key: self.entries[key].last_accessed_at,
            )
            del self.entries[oldest_key]

    def read(
        self,
        store: DatasetStore,
        dataset: DatasetName,
        *,
        start: datetime | None = None,
        end: datetime | None = None,
        filters: dict[str, Any] | None = None,
        read_mode: ReadMode | None = None,
    ) -> pl.DataFrame:
        key = (
            dataset.value,
            start.isoformat() if start else None,
            end.isoformat() if end else None,
            tuple(sorted((filters or {}).items())),
            read_mode.value if read_mode else None,
        )
        now = monotonic()
        with self._lock:
            self._prune(now)
            cached = self.entries.get(key)
            if cached and cached.expires_at > now:
                cached.last_accessed_at = now
                return cached.frame.clone()
        frame = store.read(dataset, start=start, end=end, filters=filters, read_mode=read_mode)
        with self._lock:
            self._prune(now)
            self.entries[key] = CachedFrame(
                expires_at=now + self.ttl_seconds,
                last_accessed_at=now,
                frame=frame.clone(),
            )
        return frame

    def read_latest(
        self,
        store: DatasetStore,
        dataset: DatasetName,
        *,
        filters: dict[str, Any] | None = None,
        read_mode: ReadMode | None = None,
    ) -> pl.DataFrame:
        key = (
            dataset.value,
            "latest_partition",
            tuple(sorted((filters or {}).items())),
            read_mode.value if read_mode else None,
        )
        now = monotonic()
        with self._lock:
            self._prune(now)
            cached = self.entries.get(key)
            if cached and cached.expires_at > now:
                cached.last_accessed_at = now
                return cached.frame.clone()
        frame = store.read_latest(dataset, filters=filters, read_mode=read_mode)
        with self._lock:
            self._prune(now)
            self.entries[key] = CachedFrame(
                expires_at=now + self.ttl_seconds,
                last_accessed_at=now,
                frame=frame.clone(),
            )
        return frame


@dataclass(frozen=True)
class SignalDefinition:
    id: str
    name: str
    symbol: str
    family: str
    description: str
    available_sources: tuple[str, ...]
    default_source: str


@dataclass(frozen=True)
class OptionsDashboardDefinition:
    id: str
    name: str
    description: str
    dataset: DatasetName
    default_underlying: str | None
    supported_underlyings: tuple[str, ...]
    default_source: str
    latest_keys: tuple[str, ...]
    group_columns: tuple[str, ...] = ()


@dataclass(frozen=True)
class OptionsSignalDefinition:
    id: str
    name: str
    description: str
    default_underlying: str
    supported_underlyings: tuple[str, ...]
    default_source: str
    latest_keys: tuple[str, ...]


@dataclass(frozen=True)
class PerpsDashboardDefinition:
    id: str
    name: str
    description: str
    dataset: DatasetName
    default_asset: str
    supported_assets: tuple[str, ...]
    default_source: str
    latest_keys: tuple[str, ...]


@dataclass(frozen=True)
class PerpsSignalDefinition:
    id: str
    name: str
    description: str
    default_asset: str
    supported_assets: tuple[str, ...]
    default_source: str
    latest_keys: tuple[str, ...]


@dataclass(frozen=True)
class CompositeSignalDefinition:
    id: str
    name: str
    description: str
    scope: str
    asset: str
    default_source: str
    latest_keys: tuple[str, ...]


@dataclass(frozen=True)
class MicrostructureSignalDefinition:
    id: str
    name: str
    asset: str
    horizon: str
    description: str
    source: str
    latest_keys: tuple[str, ...]


SIGNAL_DEFINITIONS: dict[str, SignalDefinition] = {
    SignalId.BTC_VVIX.value: SignalDefinition(
        id=SignalId.BTC_VVIX.value,
        name="BTC VVIX",
        symbol="BTC",
        family="vvix",
        description="Volatility-of-volatility view over BTC vol-index and surface state.",
        available_sources=VVIX_SOURCES,
        default_source="dvol",
    ),
    SignalId.ETH_VVIX.value: SignalDefinition(
        id=SignalId.ETH_VVIX.value,
        name="ETH VVIX",
        symbol="ETH",
        family="vvix",
        description="Volatility-of-volatility view over ETH vol-index and surface state.",
        available_sources=VVIX_SOURCES,
        default_source="dvol",
    ),
    SignalId.BTC_RAZOR.value: SignalDefinition(
        id=SignalId.BTC_RAZOR.value,
        name="BTC Razor",
        symbol="BTC",
        family="razor",
        description="Running edge between systematic short-vol and long-vol BTC straddle programs.",
        available_sources=RAZOR_SOURCES,
        default_source=RAZOR_SOURCES[0],
    ),
    SignalId.ETH_RAZOR.value: SignalDefinition(
        id=SignalId.ETH_RAZOR.value,
        name="ETH Razor",
        symbol="ETH",
        family="razor",
        description="Running edge between systematic short-vol and long-vol ETH straddle programs.",
        available_sources=RAZOR_SOURCES,
        default_source=RAZOR_SOURCES[0],
    ),
}

OPTIONS_DASHBOARD_DEFINITIONS: dict[str, OptionsDashboardDefinition] = {
    OPTIONS_DASHBOARD_IDS[0]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[0],
        name="Vol Surface",
        description="Smile, term structure, and surface change monitoring by asset.",
        dataset=DatasetName.OPTIONS_SURFACE_SNAPSHOT,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="surface_monitor",
        latest_keys=("mean_mark_iv", "iv_change_24h", "option_count"),
        group_columns=("tenor_bucket", "moneyness_bucket"),
    ),
    OPTIONS_DASHBOARD_IDS[1]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[1],
        name="Carry / VRP",
        description="Implied vs realized carry, roll-down, and volatility-risk-premium context.",
        dataset=DatasetName.OPTIONS_CARRY_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="carry_vrp",
        latest_keys=("carry_spread_7d", "carry_zscore", "roll_down_carry"),
    ),
    OPTIONS_DASHBOARD_IDS[2]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[2],
        name="Skew / Crash Premium",
        description="Short and long skew, crash-premium state, and skew stress changes.",
        dataset=DatasetName.OPTIONS_SKEW_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="skew_crash_premium",
        latest_keys=("rr_25_7d", "rr_25_30d", "skew_zscore"),
    ),
    OPTIONS_DASHBOARD_IDS[3]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[3],
        name="Surface Dislocation",
        description="Quote-versus-fit residuals and local smile dislocations by tenor.",
        dataset=DatasetName.OPTIONS_DISLOCATION_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="surface_dislocation",
        latest_keys=("max_residual", "dislocation_zscore", "wing_residual"),
        group_columns=("tenor_bucket",),
    ),
    OPTIONS_DASHBOARD_IDS[4]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[4],
        name="Cross-Asset Relative Value",
        description="BTC versus ETH spread monitoring across vol, skew, VVIX, and Razor context.",
        dataset=DatasetName.OPTIONS_RELATIVE_VALUE_SERIES,
        default_underlying=None,
        supported_underlyings=(),
        default_source="btc_eth",
        latest_keys=("atm_iv_spread_7d", "vvix_spread", "spread_zscore"),
    ),
    OPTIONS_DASHBOARD_IDS[5]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[5],
        name="Positioning / Flow",
        description="OI and volume concentration across expiry and moneyness buckets.",
        dataset=DatasetName.OPTIONS_POSITIONING_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="positioning_flow",
        latest_keys=("open_interest", "open_interest_share", "atm_pinning_concentration"),
        group_columns=("expiry_bucket", "moneyness_bucket"),
    ),
    OPTIONS_DASHBOARD_IDS[6]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[6],
        name="Vol Cone",
        description="Historical percentile context for implied and realized volatility by tenor.",
        dataset=DatasetName.VOL_CONE_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="vol_cone",
        latest_keys=("implied_vol", "realized_vol", "iv_rv_spread_percentile"),
        group_columns=("tenor",),
    ),
    OPTIONS_DASHBOARD_IDS[7]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[7],
        name="Term Structure Momentum",
        description="Front-to-back term slope changes over 24h and 72h windows.",
        dataset=DatasetName.TERM_STRUCTURE_MOMENTUM_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="term_structure_momentum",
        latest_keys=("term_slope_7_30", "term_slope_change_24h", "term_slope_change_72h"),
    ),
    OPTIONS_DASHBOARD_IDS[8]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[8],
        name="Gamma Exposure",
        description="True gamma concentration and pinning pressure by expiry and strike bucket.",
        dataset=DatasetName.GAMMA_EXPOSURE_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="gamma_exposure",
        latest_keys=("gamma_exposure", "gamma_share", "pinning_concentration"),
        group_columns=("expiry_bucket", "strike_bucket"),
    ),
    OPTIONS_DASHBOARD_IDS[9]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[9],
        name="Funding Context",
        description="Perpetual funding extremes and their interaction with the current vol regime.",
        dataset=DatasetName.FUNDING_CONTEXT_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="funding_context",
        latest_keys=("funding_rate", "funding_rate_zscore", "funding_change_24h"),
    ),
    OPTIONS_DASHBOARD_IDS[10]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[10],
        name="Straddle Breakeven",
        description=(
            "ATM 7D breakeven move versus realized move to explain gamma richness or cheapness."
        ),
        dataset=DatasetName.STRADDLE_BREAKEVEN_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="straddle_breakeven",
        latest_keys=("breakeven_move_pct", "realized_move_7d_pct", "breakeven_gap_pct"),
    ),
    OPTIONS_DASHBOARD_IDS[11]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[11],
        name="Vol Regime Transitions",
        description=(
            "Empirical transition probabilities and duration statistics for the current vol regime."
        ),
        dataset=DatasetName.VOL_REGIME_TRANSITION_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="vol_regime_transitions",
        latest_keys=(
            "next_state_probability_long_vol_bias",
            "next_state_probability_neutral",
            "median_state_duration_hours",
        ),
    ),
    OPTIONS_DASHBOARD_IDS[12]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[12],
        name="Surface Factor RV",
        description="Latent surface-factor momentum and node-level residual value signals.",
        dataset=DatasetName.OPTIONS_SURFACE_FACTOR_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="surface_factor_rv",
        latest_keys=("factor_momentum_24h", "node_residual_zscore", "residual_value_score"),
        group_columns=("tenor_bucket", "moneyness_bucket"),
    ),
    OPTIONS_DASHBOARD_IDS[13]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[13],
        name="ML Carry Toxicity",
        description=(
            "Cost-adjusted ATM IV minus forecast-RV carry with "
            "jump-risk and toxicity veto layers."
        ),
        dataset=DatasetName.OPTIONS_ML_CARRY_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="ml_carry_toxicity",
        latest_keys=("net_carry_score", "toxicity_score", "carry_confidence"),
    ),
    OPTIONS_DASHBOARD_IDS[14]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[14],
        name="Expiry Flow Map",
        description=(
            "Pain strike, strike concentration, gamma/vanna/charm, "
            "and pin-versus-breakout context."
        ),
        dataset=DatasetName.OPTIONS_EXPIRY_FLOW_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="expiry_flow_map",
        latest_keys=("pinning_score", "breakout_score", "distance_to_pain_pct"),
        group_columns=("expiry_bucket", "strike_bucket"),
    ),
    OPTIONS_DASHBOARD_IDS[15]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[15],
        name="Adaptive Hedging",
        description=(
            "Smile-aware delta, no-trade regions, and route "
            "preference across hedge instruments."
        ),
        dataset=DatasetName.OPTIONS_ADAPTIVE_HEDGE_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="adaptive_hedging",
        latest_keys=("smile_aware_delta", "hedge_band_width", "route_score"),
    ),
    OPTIONS_DASHBOARD_IDS[16]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[16],
        name="Sessionality Clocks",
        description="UTC session, funding-window, weekend, and expiry-window vol/carry context.",
        dataset=DatasetName.OPTIONS_SESSIONALITY_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="sessionality_clocks",
        latest_keys=("session_realized_vol", "session_carry", "sessionality_score"),
        group_columns=("session_bucket",),
    ),
    OPTIONS_DASHBOARD_IDS[17]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[17],
        name="BTC-ETH Dispersion",
        description=(
            "Surface-implied correlation proxy versus forecast "
            "realized correlation and cross-smile consistency."
        ),
        dataset=DatasetName.OPTIONS_DISPERSION_SERIES,
        default_underlying=None,
        supported_underlyings=(),
        default_source="btc_eth_dispersion",
        latest_keys=("implied_correlation", "corr_gap", "dispersion_score"),
        group_columns=("tenor",),
    ),
    OPTIONS_DASHBOARD_IDS[18]: OptionsDashboardDefinition(
        id=OPTIONS_DASHBOARD_IDS[18],
        name="Funding-Basis Skew",
        description="Funding and basis extremes mapped into skew richness and RR/BF stress.",
        dataset=DatasetName.OPTIONS_FUNDING_SKEW_SERIES,
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="funding_basis_skew",
        latest_keys=("funding_basis_skew_score", "basis_pressure", "skew_richness"),
    ),
}

OPTIONS_SIGNAL_DEFINITIONS: dict[str, OptionsSignalDefinition] = {
    OPTIONS_SIGNAL_IDS[0]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[0],
        name="Vol Carry Score",
        description="Short-tenor carry richness and short-vol carry attractiveness.",
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("vol_carry_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[1]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[1],
        name="Skew Stress Score",
        description="Crash-premium pressure and skew stress normalization.",
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("skew_stress_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[2]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[2],
        name="Surface Dislocation Score",
        description="Aggregated local smile residual and surface distortion pressure.",
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("surface_dislocation_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[3]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[3],
        name="Cross-Asset Vol Score",
        description="BTC versus ETH relative volatility spread score.",
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("cross_asset_vol_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[4]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[4],
        name="Vol Regime Score",
        description="Combined VVIX, Razor, carry, and skew regime state.",
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("vol_regime_score", "signal_state"),
    ),
    OPTIONS_SIGNAL_IDS[5]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[5],
        name="Funding / Vol Context Score",
        description="Funding extremity conditioned on the current vol regime.",
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("funding_vol_context_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[6]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[6],
        name="Breakeven Gap Score",
        description=(
            "Whether realized movement is outrunning or lagging short-dated gamma breakevens."
        ),
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("breakeven_gap_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[7]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[7],
        name="Term Structure Momentum Score",
        description="Acceleration or decay in front-end term structure pressure.",
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("term_structure_momentum_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[8]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[8],
        name="Surface Factor Score",
        description="Latent surface-factor momentum score across normalized surface cells.",
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("surface_factor_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[9]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[9],
        name="Residual Surface RV Score",
        description="Residual node value score after removing common surface-factor moves.",
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("residual_surface_rv_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[10]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[10],
        name="ML Carry Toxicity Score",
        description="Net carry score after toxicity, jump-risk, spread, and hedge-cost penalties.",
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("ml_carry_toxicity_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[11]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[11],
        name="Expiry Pin Score",
        description="Positive-gamma pinning strength around dominant strikes into expiry.",
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("expiry_pin_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[12]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[12],
        name="Gamma Flip Score",
        description=(
            "Breakout risk when negative gamma meets distance from "
            "pain and thin local flow."
        ),
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("gamma_flip_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[13]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[13],
        name="Adaptive Hedge Score",
        description="Research-only hedge quality score for smile-aware delta and route choice.",
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("adaptive_hedge_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[14]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[14],
        name="Sessionality Score",
        description="UTC session, weekend, funding-window, and expiry-window regime score.",
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("sessionality_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[15]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[15],
        name="Dispersion Score",
        description=(
            "BTC-ETH surface-implied correlation proxy gap and "
            "cross-smile consistency score."
        ),
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("dispersion_score", "vol_regime_score"),
    ),
    OPTIONS_SIGNAL_IDS[16]: OptionsSignalDefinition(
        id=OPTIONS_SIGNAL_IDS[16],
        name="Funding Skew Score",
        description="Funding/basis pressure mapped into skew richness and smile stress.",
        default_underlying="BTC",
        supported_underlyings=("BTC", "ETH"),
        default_source="options_composite",
        latest_keys=("funding_skew_score", "vol_regime_score"),
    ),
}

PERPS_DASHBOARD_DEFINITIONS: dict[str, PerpsDashboardDefinition] = {
    PERPS_DASHBOARD_IDS[0]: PerpsDashboardDefinition(
        id=PERPS_DASHBOARD_IDS[0],
        name="Funding-Basis Carry",
        description="Perps carry economics across basis, funding, and carry momentum.",
        dataset=DatasetName.PERPS_SIGNAL_SERIES,
        default_asset="BTC",
        supported_assets=("BTC", "ETH"),
        default_source="perps_family_v1",
        latest_keys=("carry_annualized_pct", "carry_zscore_7d", "cross_venue_funding_spread"),
    ),
    PERPS_DASHBOARD_IDS[1]: PerpsDashboardDefinition(
        id=PERPS_DASHBOARD_IDS[1],
        name="Liquidation Cascade Risk",
        description="Liquidation intensity, asymmetry, and OI/depth fragility.",
        dataset=DatasetName.PERPS_SIGNAL_SERIES,
        default_asset="BTC",
        supported_assets=("BTC", "ETH"),
        default_source="perps_family_v1",
        latest_keys=("cascade_risk_score", "liq_intensity_zscore", "oi_concentration"),
    ),
    PERPS_DASHBOARD_IDS[2]: PerpsDashboardDefinition(
        id=PERPS_DASHBOARD_IDS[2],
        name="Order Flow Toxicity",
        description="VPIN-style toxicity, flow persistence, and large-trade pressure.",
        dataset=DatasetName.PERPS_SIGNAL_SERIES,
        default_asset="BTC",
        supported_assets=("BTC", "ETH"),
        default_source="perps_family_v1",
        latest_keys=("vpin_50", "toxic_flow_zscore_4h", "large_trade_ratio"),
    ),
    PERPS_DASHBOARD_IDS[3]: PerpsDashboardDefinition(
        id=PERPS_DASHBOARD_IDS[3],
        name="OI-Price Divergence",
        description="Leverage build-up and unwind pressure independent of price direction.",
        dataset=DatasetName.PERPS_SIGNAL_SERIES,
        default_asset="BTC",
        supported_assets=("BTC", "ETH"),
        default_source="perps_family_v1",
        latest_keys=("oi_change_1h", "oi_price_divergence", "oi_acceleration"),
    ),
    PERPS_DASHBOARD_IDS[4]: PerpsDashboardDefinition(
        id=PERPS_DASHBOARD_IDS[4],
        name="Cross-Venue Price Discovery",
        description="Leader venue, dislocation bps, and convergence speed.",
        dataset=DatasetName.PERPS_SIGNAL_SERIES,
        default_asset="BTC",
        supported_assets=("BTC", "ETH"),
        default_source="perps_family_v1",
        latest_keys=("lead_lag_score", "venue_dislocation_bps", "convergence_half_life_secs"),
    ),
    PERPS_DASHBOARD_IDS[5]: PerpsDashboardDefinition(
        id=PERPS_DASHBOARD_IDS[5],
        name="Depth Resilience",
        description="Spread regime, refill speed, and book fragility.",
        dataset=DatasetName.PERPS_SIGNAL_SERIES,
        default_asset="BTC",
        supported_assets=("BTC", "ETH"),
        default_source="perps_family_v1",
        latest_keys=("depth_recovery_speed", "spread_zscore_4h", "depth_imbalance_momentum"),
    ),
    PERPS_DASHBOARD_IDS[6]: PerpsDashboardDefinition(
        id=PERPS_DASHBOARD_IDS[6],
        name="Volatility Microstructure",
        description="Multi-frequency realized vol, signature ratio, and clustering.",
        dataset=DatasetName.PERPS_SIGNAL_SERIES,
        default_asset="BTC",
        supported_assets=("BTC", "ETH"),
        default_source="perps_family_v1",
        latest_keys=("rv_1m", "rv_5m", "vol_signature_ratio"),
    ),
    PERPS_DASHBOARD_IDS[7]: PerpsDashboardDefinition(
        id=PERPS_DASHBOARD_IDS[7],
        name="Cumulative Delta Momentum",
        description="Delta acceleration, absorption, and exhaustion regimes.",
        dataset=DatasetName.PERPS_SIGNAL_SERIES,
        default_asset="BTC",
        supported_assets=("BTC", "ETH"),
        default_source="perps_family_v1",
        latest_keys=("cum_delta_acceleration", "delta_divergence", "absorption_ratio"),
    ),
}

PERPS_SIGNAL_DEFINITIONS: dict[str, PerpsSignalDefinition] = {
    PERPS_SIGNAL_IDS[0]: PerpsSignalDefinition(
        id=PERPS_SIGNAL_IDS[0],
        name="Carry Score",
        description="Composite crowding and carry richness score for perps.",
        default_asset="BTC",
        supported_assets=("BTC", "ETH"),
        default_source="perps_composite_v1",
        latest_keys=("carry_score",),
    ),
    PERPS_SIGNAL_IDS[1]: PerpsSignalDefinition(
        id=PERPS_SIGNAL_IDS[1],
        name="Cascade Risk Score",
        description="Composite liquidation-cascade and fragility score.",
        default_asset="BTC",
        supported_assets=("BTC", "ETH"),
        default_source="perps_composite_v1",
        latest_keys=("cascade_risk_score",),
    ),
    PERPS_SIGNAL_IDS[2]: PerpsSignalDefinition(
        id=PERPS_SIGNAL_IDS[2],
        name="Toxicity Score",
        description="Composite flow-toxicity and adverse-selection score.",
        default_asset="BTC",
        supported_assets=("BTC", "ETH"),
        default_source="perps_composite_v1",
        latest_keys=("toxicity_score",),
    ),
    PERPS_SIGNAL_IDS[3]: PerpsSignalDefinition(
        id=PERPS_SIGNAL_IDS[3],
        name="Positioning Score",
        description="Composite OI divergence and leverage pressure score.",
        default_asset="BTC",
        supported_assets=("BTC", "ETH"),
        default_source="perps_composite_v1",
        latest_keys=("positioning_score",),
    ),
    PERPS_SIGNAL_IDS[4]: PerpsSignalDefinition(
        id=PERPS_SIGNAL_IDS[4],
        name="Fragility Score",
        description="Composite spread, depth, and dislocation fragility score.",
        default_asset="BTC",
        supported_assets=("BTC", "ETH"),
        default_source="perps_composite_v1",
        latest_keys=("fragility_score",),
    ),
}

COMPOSITE_SIGNAL_DEFINITIONS: dict[str, CompositeSignalDefinition] = {
    COMPOSITE_SIGNAL_IDS[0]: CompositeSignalDefinition(
        id=COMPOSITE_SIGNAL_IDS[0],
        name="BTC Composite",
        description=(
            "Composite BTC market-state index separating sentiment, "
            "direction, and fragility."
        ),
        scope="asset",
        asset="BTC",
        default_source="composite_v1",
        latest_keys=("sentiment_index", "directional_tilt", "fragility_index", "confidence_score"),
    ),
    COMPOSITE_SIGNAL_IDS[1]: CompositeSignalDefinition(
        id=COMPOSITE_SIGNAL_IDS[1],
        name="ETH Composite",
        description=(
            "Composite ETH market-state index separating sentiment, "
            "direction, and fragility."
        ),
        scope="asset",
        asset="ETH",
        default_source="composite_v1",
        latest_keys=("sentiment_index", "directional_tilt", "fragility_index", "confidence_score"),
    ),
    COMPOSITE_SIGNAL_IDS[2]: CompositeSignalDefinition(
        id=COMPOSITE_SIGNAL_IDS[2],
        name="Market Composite",
        description=(
            "BTC/ETH weighted composite market-state index separating "
            "sentiment, direction, and fragility."
        ),
        scope="market",
        asset="MARKET",
        default_source="composite_v1",
        latest_keys=("sentiment_index", "directional_tilt", "fragility_index", "confidence_score"),
    ),
}

MICROSTRUCTURE_SIGNAL_DEFINITIONS: dict[str, MicrostructureSignalDefinition] = {
    SignalId.BTC_MICROSTRUCTURE_30S.value: MicrostructureSignalDefinition(
        id=SignalId.BTC_MICROSTRUCTURE_30S.value,
        name="BTC Microstructure 30s",
        asset="BTC",
        horizon="30s",
        description="Short-horizon BTC microstructure direction and tradability signal.",
        source="tcn_boost_fusion",
        latest_keys=("prob_up", "prob_down", "expected_move_bps", "tradability_score"),
    ),
    SignalId.BTC_MICROSTRUCTURE_1M.value: MicrostructureSignalDefinition(
        id=SignalId.BTC_MICROSTRUCTURE_1M.value,
        name="BTC Microstructure 1m",
        asset="BTC",
        horizon="1m",
        description="One-minute BTC microstructure direction and tradability signal.",
        source="tcn_boost_fusion",
        latest_keys=("prob_up", "prob_down", "expected_move_bps", "tradability_score"),
    ),
    SignalId.BTC_MICROSTRUCTURE_2M.value: MicrostructureSignalDefinition(
        id=SignalId.BTC_MICROSTRUCTURE_2M.value,
        name="BTC Microstructure 2m",
        asset="BTC",
        horizon="2m",
        description="Two-minute BTC microstructure direction and tradability signal.",
        source="tcn_boost_fusion",
        latest_keys=("prob_up", "prob_down", "expected_move_bps", "tradability_score"),
    ),
    SignalId.BTC_MICROSTRUCTURE_5M.value: MicrostructureSignalDefinition(
        id=SignalId.BTC_MICROSTRUCTURE_5M.value,
        name="BTC Microstructure 5m",
        asset="BTC",
        horizon="5m",
        description="Five-minute BTC microstructure direction and tradability signal.",
        source="tcn_boost_fusion",
        latest_keys=("prob_up", "prob_down", "expected_move_bps", "tradability_score"),
    ),
    SignalId.ETH_MICROSTRUCTURE_30S.value: MicrostructureSignalDefinition(
        id=SignalId.ETH_MICROSTRUCTURE_30S.value,
        name="ETH Microstructure 30s",
        asset="ETH",
        horizon="30s",
        description="Short-horizon ETH microstructure direction and tradability signal.",
        source="tcn_boost_fusion",
        latest_keys=("prob_up", "prob_down", "expected_move_bps", "tradability_score"),
    ),
    SignalId.ETH_MICROSTRUCTURE_1M.value: MicrostructureSignalDefinition(
        id=SignalId.ETH_MICROSTRUCTURE_1M.value,
        name="ETH Microstructure 1m",
        asset="ETH",
        horizon="1m",
        description="One-minute ETH microstructure direction and tradability signal.",
        source="tcn_boost_fusion",
        latest_keys=("prob_up", "prob_down", "expected_move_bps", "tradability_score"),
    ),
    SignalId.ETH_MICROSTRUCTURE_2M.value: MicrostructureSignalDefinition(
        id=SignalId.ETH_MICROSTRUCTURE_2M.value,
        name="ETH Microstructure 2m",
        asset="ETH",
        horizon="2m",
        description="Two-minute ETH microstructure direction and tradability signal.",
        source="tcn_boost_fusion",
        latest_keys=("prob_up", "prob_down", "expected_move_bps", "tradability_score"),
    ),
    SignalId.ETH_MICROSTRUCTURE_5M.value: MicrostructureSignalDefinition(
        id=SignalId.ETH_MICROSTRUCTURE_5M.value,
        name="ETH Microstructure 5m",
        asset="ETH",
        horizon="5m",
        description="Five-minute ETH microstructure direction and tradability signal.",
        source="tcn_boost_fusion",
        latest_keys=("prob_up", "prob_down", "expected_move_bps", "tradability_score"),
    ),
}


@dataclass(frozen=True)
class ProviderRegistry:
    signal_provider: DefaultSignalProvider
    microstructure_provider: DefaultMicrostructureProvider
    price_provider: DefaultPriceProvider
    options_dashboard_provider: DefaultOptionsDashboardProvider
    options_signal_provider: DefaultOptionsSignalProvider
    perps_dashboard_provider: DefaultPerpsDashboardProvider
    perps_signal_provider: DefaultPerpsSignalProvider
    composite_provider: DefaultCompositeProvider
    execution_provider: DefaultExecutionProvider
    dataset_provider: DefaultDatasetProvider
    run_provider: DefaultRunStatusProvider
    stream_provider: PollingStreamProvider


class DefaultSignalProvider:
    def __init__(self, store: DatasetStore, cache: ReadCache) -> None:
        self.store = store
        self.cache = cache

    def list_signals(self) -> list[SignalSummary]:
        # Read each dataset family once instead of N+1 per-signal reads
        vvix_frames: dict[str, pl.DataFrame] = {}
        dvol_frames: dict[str, pl.DataFrame] = {}
        razor_frames: dict[str, pl.DataFrame] = {}
        for definition in SIGNAL_DEFINITIONS.values():
            if definition.family == "vvix":
                if definition.symbol not in vvix_frames:
                    vvix_frames[definition.symbol] = self.cache.read_latest(
                        self.store,
                        DatasetName.VVIX_SERIES,
                        filters={
                            "underlying": definition.symbol,
                            "source": definition.default_source,
                        },
                    )
                    dvol_frames[definition.symbol] = self.cache.read_latest(
                        self.store,
                        DatasetName.DVOL_SNAPSHOT,
                        filters={"underlying": definition.symbol, "source": "dvol"},
                    )
            else:
                if definition.symbol not in razor_frames:
                    razor_frames[definition.symbol] = self.cache.read_latest(
                        self.store,
                        DatasetName.RAZOR_SERIES,
                        filters={
                            "underlying": definition.symbol,
                            "source": definition.default_source,
                        },
                    )

        results: list[SignalSummary] = []
        for definition in SIGNAL_DEFINITIONS.values():
            if definition.family == "vvix":
                latest_vvix = vvix_frames[definition.symbol].sort("ts").tail(1)
                latest_dvol = dvol_frames[definition.symbol].sort("ts").tail(1)
                latest_ts = (
                    None if latest_vvix.is_empty() else cast(datetime, latest_vvix.item(0, "ts"))
                )
                latest_quality = (
                    None if latest_vvix.is_empty() else str(latest_vvix.item(0, "quality_flag"))
                )
                latest_values = {
                    "dvol": (
                        None if latest_dvol.is_empty() else float(latest_dvol.item(0, "dvol_close"))
                    ),
                    "vol_index_value": (
                        None
                        if latest_vvix.is_empty()
                        else _float_or_none(latest_vvix.item(0, "vol_index_value"))
                    ),
                    "rvvix_14d": (
                        None
                        if latest_vvix.is_empty()
                        else _float_or_none(latest_vvix.item(0, "rvvix_14d"))
                    ),
                    "rvvix_30d": (
                        None
                        if latest_vvix.is_empty()
                        else _float_or_none(latest_vvix.item(0, "rvvix_30d"))
                    ),
                    "qvvix_30d": (
                        None
                        if latest_vvix.is_empty()
                        else _float_or_none(latest_vvix.item(0, "qvvix_30d"))
                    ),
                }
            else:
                latest_razor = razor_frames[definition.symbol].sort("ts").tail(1)
                latest_ts = (
                    None if latest_razor.is_empty() else cast(datetime, latest_razor.item(0, "ts"))
                )
                latest_quality = (
                    None if latest_razor.is_empty() else str(latest_razor.item(0, "quality_flag"))
                )
                latest_values = {
                    "razor_pnl": (
                        None
                        if latest_razor.is_empty()
                        else _float_or_none(latest_razor.item(0, "razor_pnl"))
                    ),
                    "razor_vrp": (
                        None
                        if latest_razor.is_empty()
                        else _float_or_none(latest_razor.item(0, "razor_vrp"))
                    ),
                    "razor_pnl_zscore": (
                        None
                        if latest_razor.is_empty()
                        else _float_or_none(latest_razor.item(0, "razor_pnl_zscore"))
                    ),
                    "razor_vrp_zscore": (
                        None
                        if latest_razor.is_empty()
                        else _float_or_none(latest_razor.item(0, "razor_vrp_zscore"))
                    ),
                }
            results.append(
                SignalSummary(
                    id=definition.id,
                    name=definition.name,
                    symbol=definition.symbol,
                    family=definition.family,
                    description=definition.description,
                    available_sources=list(definition.available_sources),
                    latest_source=definition.default_source,
                    latest_updated_at=latest_ts,
                    latest_quality=latest_quality,
                    latest_values=latest_values,
                )
            )
        return results

    def get_latest(self, signal_id: str, source: str | None = None) -> SignalSummary:
        definition = _signal_definition(signal_id)
        chosen_source = _source_or_default(definition, source)
        if definition.family == "vvix":
            latest_vvix = (
                self.cache.read_latest(
                    self.store,
                    DatasetName.VVIX_SERIES,
                    filters={"underlying": definition.symbol, "source": chosen_source},
                )
                .sort("ts")
                .tail(1)
            )
            latest_dvol = (
                self.cache.read_latest(
                    self.store,
                    DatasetName.DVOL_SNAPSHOT,
                    filters={"underlying": definition.symbol, "source": "dvol"},
                )
                .sort("ts")
                .tail(1)
            )
            latest_ts = (
                None if latest_vvix.is_empty() else cast(datetime, latest_vvix.item(0, "ts"))
            )
            latest_quality = (
                None if latest_vvix.is_empty() else str(latest_vvix.item(0, "quality_flag"))
            )
            latest_values = {
                "dvol": (
                    None if latest_dvol.is_empty() else float(latest_dvol.item(0, "dvol_close"))
                ),
                "vol_index_value": (
                    None
                    if latest_vvix.is_empty()
                    else _float_or_none(latest_vvix.item(0, "vol_index_value"))
                ),
                "rvvix_14d": (
                    None
                    if latest_vvix.is_empty()
                    else _float_or_none(latest_vvix.item(0, "rvvix_14d"))
                ),
                "rvvix_30d": (
                    None
                    if latest_vvix.is_empty()
                    else _float_or_none(latest_vvix.item(0, "rvvix_30d"))
                ),
                "qvvix_30d": (
                    None
                    if latest_vvix.is_empty()
                    else _float_or_none(latest_vvix.item(0, "qvvix_30d"))
                ),
            }
        else:
            latest_razor = (
                self.cache.read_latest(
                    self.store,
                    DatasetName.RAZOR_SERIES,
                    filters={"underlying": definition.symbol, "source": chosen_source},
                )
                .sort("ts")
                .tail(1)
            )
            latest_ts = (
                None if latest_razor.is_empty() else cast(datetime, latest_razor.item(0, "ts"))
            )
            latest_quality = (
                None if latest_razor.is_empty() else str(latest_razor.item(0, "quality_flag"))
            )
            latest_values = {
                "razor_pnl": (
                    None
                    if latest_razor.is_empty()
                    else _float_or_none(latest_razor.item(0, "razor_pnl"))
                ),
                "razor_vrp": (
                    None
                    if latest_razor.is_empty()
                    else _float_or_none(latest_razor.item(0, "razor_vrp"))
                ),
                "razor_pnl_zscore": (
                    None
                    if latest_razor.is_empty()
                    else _float_or_none(latest_razor.item(0, "razor_pnl_zscore"))
                ),
                "razor_vrp_zscore": (
                    None
                    if latest_razor.is_empty()
                    else _float_or_none(latest_razor.item(0, "razor_vrp_zscore"))
                ),
            }
        return SignalSummary(
            id=definition.id,
            name=definition.name,
            symbol=definition.symbol,
            family=definition.family,
            description=definition.description,
            available_sources=list(definition.available_sources),
            latest_source=chosen_source,
            latest_updated_at=latest_ts,
            latest_quality=latest_quality,
            latest_values=latest_values,
        )

    def get_series(
        self,
        signal_id: str,
        *,
        source: str,
        start: datetime | None,
        end: datetime | None,
        quality: str | None,
        max_points: int | None = None,
    ) -> SignalSeriesResponse:
        definition = _signal_definition(signal_id)
        chosen_source = _source_or_default(definition, source)
        runs = self.cache.read(
            self.store,
            DatasetName.INGESTION_RUNS,
            start=start,
            end=end,
            filters={
                "dataset": DatasetName.OPTION_CHAIN_SNAPSHOT.value,
                "underlying": definition.symbol,
            },
            read_mode=ReadMode.ALL_RUNS,
        )
        gap_ts = set(
            cast(list[datetime], runs.filter(pl.col("status") == "gap")["logical_ts"].to_list())
        )
        if definition.family == "vvix":
            vvix = self.cache.read(
                self.store,
                DatasetName.VVIX_SERIES,
                start=start,
                end=end,
                filters={"underlying": definition.symbol, "source": chosen_source},
            )
            dvol = self.cache.read(
                self.store,
                DatasetName.DVOL_SNAPSHOT,
                start=start,
                end=end,
                filters={"underlying": definition.symbol, "source": "dvol"},
            ).select(["ts", pl.col("dvol_close").alias("dvol")])
            surface = self.cache.read(
                self.store,
                DatasetName.SURFACE_FEATURES,
                start=start,
                end=end,
                filters={"underlying": definition.symbol, "source": "ssvi"},
            ).select(["ts", "fit_quality"])
            series = (
                vvix.join(dvol, on="ts", how="left").join(surface, on="ts", how="left").sort("ts")
            )
        else:
            series = self.cache.read(
                self.store,
                DatasetName.RAZOR_SERIES,
                start=start,
                end=end,
                filters={"underlying": definition.symbol, "source": chosen_source},
            ).sort("ts")
        if quality:
            series = series.filter(pl.col("quality_flag") == quality)
        if max_points and len(series) > max_points:
            stride = max(1, math.ceil(len(series) / max_points))
            series = series.gather_every(stride)
        points = [
            SignalSeriesPoint(
                ts=cast(datetime, row["ts"]),
                source=chosen_source,
                underlying=definition.symbol,
                dvol=_float_or_none(row.get("dvol")),
                vol_index_value=_float_or_none(row.get("vol_index_value")),
                rvvix_14d=_float_or_none(row.get("rvvix_14d")),
                rvvix_30d=_float_or_none(row.get("rvvix_30d")),
                qvvix_30d=_float_or_none(row.get("qvvix_30d")),
                razor_pnl=_float_or_none(row.get("razor_pnl")),
                razor_vrp=_float_or_none(row.get("razor_vrp")),
                razor_pnl_zscore=_float_or_none(row.get("razor_pnl_zscore")),
                razor_vrp_zscore=_float_or_none(row.get("razor_vrp_zscore")),
                trade_band=cast(str | None, row.get("trade_band")),
                long_vol_signal=cast(bool | None, row.get("long_vol_signal")),
                short_vol_signal=cast(bool | None, row.get("short_vol_signal")),
                fit_quality=_float_or_none(row.get("fit_quality")),
                gap=cast(datetime, row["ts"]) in gap_ts,
                quality_flag=cast(str | None, row.get("quality_flag")),
            )
            for row in series.iter_rows(named=True)
        ]
        summary = self.get_latest(signal_id, source=chosen_source)
        return SignalSeriesResponse(
            signal_id=signal_id,
            symbol=definition.symbol,
            source=chosen_source,
            points=points,
            latest_values=summary.latest_values,
            context={
                "family": definition.family,
                "gap_count": len(gap_ts),
                "latest_quality": summary.latest_quality,
                "points": len(points),
            },
        )


class DefaultMicrostructureProvider:
    def __init__(self, store: DatasetStore, cache: ReadCache) -> None:
        self.store = store
        self.cache = cache

    def list_signals(self) -> list[MicrostructureSignalSummary]:
        # Read model output once, then extract per-signal rows
        all_outputs = self.cache.read_latest(
            self.store,
            DatasetName.MICROSTRUCTURE_MODEL_OUTPUT,
            filters={"source": "tcn_boost_fusion", "model_version": MICROSTRUCTURE_MODEL_VERSION},
        )
        results: list[MicrostructureSignalSummary] = []
        for definition in MICROSTRUCTURE_SIGNAL_DEFINITIONS.values():
            subset = (
                all_outputs.filter(
                    (pl.col("asset") == definition.asset)
                    & (pl.col("horizon") == definition.horizon)
                )
                .sort("ts")
                .tail(1)
            )
            latest_ts = None if subset.is_empty() else cast(datetime, subset.item(0, "ts"))
            latest_quality = (
                None if subset.is_empty() else cast(str | None, subset.item(0, "quality_flag"))
            )
            latest_values = (
                {}
                if subset.is_empty()
                else {
                    key: _coerce_value(subset.item(0, key))
                    for key in definition.latest_keys
                    if key in subset.columns
                }
            )
            results.append(
                MicrostructureSignalSummary(
                    id=definition.id,
                    name=definition.name,
                    asset=definition.asset,
                    horizon=definition.horizon,
                    description=definition.description,
                    latest_updated_at=latest_ts,
                    latest_quality=latest_quality,
                    latest_values=latest_values,
                )
            )
        return results

    def get_latest(self, signal_id: str) -> MicrostructureSignalSummary:
        definition = _microstructure_signal_definition(signal_id)
        frame = (
            self.cache.read_latest(
                self.store,
                DatasetName.MICROSTRUCTURE_MODEL_OUTPUT,
                filters={
                    "asset": definition.asset,
                    "horizon": definition.horizon,
                    "source": definition.source,
                    "model_version": MICROSTRUCTURE_MODEL_VERSION,
                },
            )
            .sort("ts")
            .tail(1)
        )
        latest_ts = None if frame.is_empty() else cast(datetime, frame.item(0, "ts"))
        latest_quality = (
            None if frame.is_empty() else cast(str | None, frame.item(0, "quality_flag"))
        )
        latest_values = (
            {}
            if frame.is_empty()
            else {
                key: _coerce_value(frame.item(0, key))
                for key in definition.latest_keys
                if key in frame.columns
            }
        )
        return MicrostructureSignalSummary(
            id=definition.id,
            name=definition.name,
            asset=definition.asset,
            horizon=definition.horizon,
            description=definition.description,
            latest_updated_at=latest_ts,
            latest_quality=latest_quality,
            latest_values=latest_values,
        )

    def get_series(
        self,
        signal_id: str,
        *,
        start: datetime | None,
        end: datetime | None,
        quality: str | None,
        max_points: int | None = None,
    ) -> MicrostructureSignalSeriesResponse:
        definition = _microstructure_signal_definition(signal_id)
        outputs = self.cache.read(
            self.store,
            DatasetName.MICROSTRUCTURE_MODEL_OUTPUT,
            start=start,
            end=end,
            filters={
                "asset": definition.asset,
                "horizon": definition.horizon,
                "source": definition.source,
                "model_version": MICROSTRUCTURE_MODEL_VERSION,
            },
        ).sort("ts")
        signals = self.cache.read(
            self.store,
            DatasetName.MICROSTRUCTURE_SIGNAL_SERIES,
            start=start,
            end=end,
            filters={
                "asset": definition.asset,
                "horizon": definition.horizon,
                "source": "tcn_boost_paper_signal",
            },
        ).select(
            [
                "ts",
                "signal_state",
                "paper_position",
                "expected_edge_bps",
                pl.col("quality_flag").alias("signal_quality_flag"),
            ]
        )
        series = outputs.join(signals, on="ts", how="left").sort("ts")
        if quality:
            series = series.filter(pl.col("quality_flag") == quality)
        # Change-point compression: keep only rows where values change meaningfully
        if len(series) > 2:
            keep_mask = [True]  # always keep first row
            prev = series.row(0, named=True)
            for i in range(1, len(series) - 1):
                row = series.row(i, named=True)
                max_delta = max(
                    abs((row.get("prob_up") or 0) - (prev.get("prob_up") or 0)),
                    abs((row.get("prob_down") or 0) - (prev.get("prob_down") or 0)),
                    abs((row.get("expected_move_bps") or 0) - (prev.get("expected_move_bps") or 0)),
                    abs((row.get("tradability_score") or 0) - (prev.get("tradability_score") or 0)),
                )
                state_changed = row.get("signal_state") != prev.get("signal_state")
                if max_delta > 0.001 or state_changed:
                    keep_mask.append(True)
                    prev = row
                else:
                    keep_mask.append(False)
            keep_mask.append(True)  # always keep last row
            series = series.filter(pl.Series(keep_mask))
        if max_points and len(series) > max_points:
            stride = max(1, math.ceil(len(series) / max_points))
            series = series.gather_every(stride)
        points = [
            MicrostructureSeriesPoint(
                ts=cast(datetime, row["ts"]),
                asset=definition.asset,
                horizon=definition.horizon,
                signal_state=cast(str | None, row.get("signal_state")),
                paper_position=cast(str | None, row.get("paper_position")),
                regime_state=cast(str | None, row.get("regime_state")),
                prob_down=_float_or_none(row.get("prob_down")),
                prob_flat=_float_or_none(row.get("prob_flat")),
                prob_up=_float_or_none(row.get("prob_up")),
                expected_move_bps=_float_or_none(row.get("expected_move_bps")),
                expected_edge_bps=_float_or_none(row.get("expected_edge_bps")),
                tradability_score=_float_or_none(row.get("tradability_score")),
                confidence=_float_or_none(row.get("confidence")),
                confidence_bucket=cast(str | None, row.get("confidence_bucket")),
                blocked_reason=cast(str | None, row.get("blocked_reason")),
                model_family=cast(str | None, row.get("model_family")),
                quality_flag=cast(str | None, row.get("quality_flag")),
                values={
                    key: _coerce_value(value)
                    for key, value in row.items()
                    if key
                    not in {
                        "ts",
                        "date",
                        "asset",
                        "horizon",
                        "source",
                        "signal_state",
                        "paper_position",
                        "regime_state",
                        "prob_down",
                        "prob_flat",
                        "prob_up",
                        "expected_move_bps",
                        "expected_edge_bps",
                        "tradability_score",
                        "confidence",
                        "confidence_bucket",
                        "blocked_reason",
                        "model_family",
                        "quality_flag",
                        "signal_quality_flag",
                        "run_id",
                        "run_ts",
                        "logical_ts",
                        "write_mode",
                    }
                },
            )
            for row in series.iter_rows(named=True)
        ]
        latest = self.get_latest(signal_id)
        return MicrostructureSignalSeriesResponse(
            signal_id=signal_id,
            asset=definition.asset,
            horizon=definition.horizon,
            points=points,
            latest_values=latest.latest_values,
            context={"description": definition.description, "points": len(points)},
        )

    def get_features(
        self,
        asset: str,
        *,
        horizon: str,
        start: datetime | None,
        end: datetime | None,
        quality: str | None,
        max_points: int | None,
    ) -> MicrostructureFeatureResponse:
        frame = self.cache.read(
            self.store,
            DatasetName.MICROSTRUCTURE_FEATURE_SERIES,
            start=start,
            end=end,
            filters={"asset": asset, "horizon": horizon},
        ).sort("ts")
        if quality:
            frame = frame.filter(pl.col("quality_flag") == quality)
        if max_points and frame.height > max_points:
            frame = frame.gather_every(max(1, math.ceil(frame.height / max_points)))
        points = [
            MicrostructureSeriesPoint(
                ts=cast(datetime, row["ts"]),
                asset=asset,
                horizon=horizon,
                quality_flag=cast(str | None, row.get("quality_flag")),
                values={
                    key: _coerce_value(value)
                    for key, value in row.items()
                    if key
                    not in {
                        "ts",
                        "date",
                        "asset",
                        "horizon",
                        "source",
                        "quality_flag",
                        "run_id",
                        "run_ts",
                        "logical_ts",
                        "write_mode",
                    }
                },
            )
            for row in frame.iter_rows(named=True)
        ]
        latest_values = points[-1].values if points else {}
        return MicrostructureFeatureResponse(
            asset=asset,
            horizon=horizon,
            points=points,
            latest_values=latest_values,
            context={"points": len(points)},
        )

    def get_regime(
        self,
        asset: str,
        *,
        start: datetime | None,
        end: datetime | None,
        quality: str | None,
        max_points: int | None,
    ) -> MicrostructureRegimeResponse:
        frame = self.cache.read(
            self.store,
            DatasetName.MICROSTRUCTURE_REGIME_SERIES,
            start=start,
            end=end,
            filters={"asset": asset, "source": "tcn_boost_regime_v1"},
        ).sort("ts")
        if quality:
            frame = frame.filter(pl.col("quality_flag") == quality)
        if max_points and frame.height > max_points:
            frame = frame.gather_every(max(1, math.ceil(frame.height / max_points)))
        points = [
            MicrostructureSeriesPoint(
                ts=cast(datetime, row["ts"]),
                asset=asset,
                horizon="1m",
                regime_state=cast(str | None, row.get("regime_state")),
                quality_flag=cast(str | None, row.get("quality_flag")),
                values={
                    "vol_percentile": _coerce_value(row.get("vol_percentile")),
                    "flow_percentile": _coerce_value(row.get("flow_percentile")),
                    "basis_percentile": _coerce_value(row.get("basis_percentile")),
                    "persistence_hours": _coerce_value(row.get("persistence_hours")),
                    "hysteresis_state": _coerce_value(row.get("hysteresis_state")),
                },
            )
            for row in frame.iter_rows(named=True)
        ]
        latest_values = points[-1].values if points else {}
        return MicrostructureRegimeResponse(
            asset=asset,
            points=points,
            latest_values=latest_values,
            context={"points": len(points)},
        )

    def get_health(self) -> MicrostructureHealth:
        outputs = self.cache.read_latest(self.store, DatasetName.MICROSTRUCTURE_MODEL_OUTPUT)
        outputs = outputs.filter(pl.col("model_version") == MICROSTRUCTURE_MODEL_VERSION)
        features = self.cache.read_latest(self.store, DatasetName.MICROSTRUCTURE_FEATURE_SERIES)
        training = self.cache.read_latest(self.store, DatasetName.MICROSTRUCTURE_TRAINING_RUN)
        training = training.filter(pl.col("model_version") == MICROSTRUCTURE_MODEL_VERSION)
        backtests = self.cache.read_latest(self.store, DatasetName.MICROSTRUCTURE_BACKTEST_RUN)
        backtests = backtests.filter(pl.col("model_version") == MICROSTRUCTURE_MODEL_VERSION)
        last_prediction = (
            None if outputs.is_empty() else cast(datetime, outputs.sort("ts").tail(1).item(0, "ts"))
        )
        last_feature = (
            None
            if features.is_empty()
            else cast(datetime, features.sort("ts").tail(1).item(0, "ts"))
        )
        last_training = (
            None
            if training.is_empty()
            else cast(datetime, training.sort("ts").tail(1).item(0, "ts"))
        )
        last_backtest = (
            None
            if backtests.is_empty()
            else cast(datetime, backtests.sort("ts").tail(1).item(0, "ts"))
        )
        latest_book = self.store.latest_logical_ts(DatasetName.MICROSTRUCTURE_BOOK_STATE)
        latest_raw = self.store.latest_logical_ts(DatasetName.MICROSTRUCTURE_RAW_EVENT)
        freshness_anchor = max(
            [value for value in (last_feature, last_prediction, latest_book, latest_raw) if value is not None],
            default=None,
        )
        engine_status = "online" if freshness_anchor else "cold"
        status = "ok" if last_prediction else "degraded"
        if freshness_anchor is not None:
            lag = datetime.now(tz=timezone.utc) - freshness_anchor
            if lag > timedelta(minutes=10):
                engine_status = "stale"
                status = "degraded"
        return MicrostructureHealth(
            status=status,
            engine_status=engine_status,
            last_feature_ts=last_feature,
            last_prediction_ts=last_prediction,
            last_training_ts=last_training,
            last_backtest_ts=last_backtest,
            signals_available=len(
                [
                    definition
                    for definition in MICROSTRUCTURE_SIGNAL_DEFINITIONS.values()
                    if self.get_latest(definition.id).latest_updated_at is not None
                ]
            ),
        )

    def get_calibration(
        self,
        asset: str,
        *,
        horizon: str | None = None,
        limit: int = 50,
    ) -> DatasetRowsResponse:
        frame = self.cache.read(
            self.store,
            DatasetName.MICROSTRUCTURE_TRAINING_RUN,
            filters={
                "asset": asset,
                "horizon": horizon,
                "source": "tcn_boost_fusion",
                "model_version": MICROSTRUCTURE_MODEL_VERSION,
            }
            if horizon
            else {
                "asset": asset,
                "source": "tcn_boost_fusion",
                "model_version": MICROSTRUCTURE_MODEL_VERSION,
            },
        ).sort("ts", descending=True)
        rows = frame.head(limit).to_dicts()
        return DatasetRowsResponse(
            dataset_id=DatasetName.MICROSTRUCTURE_TRAINING_RUN.value,
            rows=rows,
            limit=limit,
        )

    def get_backtests(
        self,
        *,
        asset: str | None = None,
        horizon: str | None = None,
        limit: int = 50,
    ) -> DatasetRowsResponse:
        filters: dict[str, Any] = {"source": "tcn_boost_fusion"}
        filters["model_version"] = MICROSTRUCTURE_MODEL_VERSION
        if asset:
            filters["asset"] = asset
        if horizon:
            filters["horizon"] = horizon
        frame = self.cache.read(
            self.store,
            DatasetName.MICROSTRUCTURE_BACKTEST_RUN,
            filters=filters,
        ).sort("ts", descending=True)
        rows = frame.head(limit).to_dicts()
        return DatasetRowsResponse(
            dataset_id=DatasetName.MICROSTRUCTURE_BACKTEST_RUN.value,
            rows=rows,
            limit=limit,
        )


class DefaultPriceProvider:
    def __init__(self, store: DatasetStore, cache: ReadCache) -> None:
        self.store = store
        self.cache = cache

    def _recent_bar_frame(
        self,
        *,
        asset: str | None = None,
        venue_scope: str = "all",
        windows: tuple[timedelta, ...] = (
            timedelta(days=1),
            timedelta(days=7),
            timedelta(days=30),
            timedelta(days=90),
        ),
    ) -> tuple[pl.DataFrame, datetime | None, bool]:
        now = datetime.now(timezone.utc)
        filters: dict[str, Any] = {"clock": "event_time", "bar_size_secs": 60}
        if asset:
            filters["asset"] = asset.upper()
        for index, window in enumerate(windows):
            frame = self.cache.read(
                self.store,
                DatasetName.MICROSTRUCTURE_BAR,
                start=now - window,
                end=now,
                filters=filters,
            )
            if venue_scope != "all" and not frame.is_empty():
                frame = frame.filter(pl.col("venue_group") == venue_scope)
            if frame.is_empty():
                continue
            latest = cast(datetime | None, frame.sort("ts").tail(1).item(0, "ts"))
            return frame, latest, index > 0
        return empty_frame(DatasetName.MICROSTRUCTURE_BAR), None, False

    def get_latest_prices(self) -> list[PriceSummary]:
        frame = self._latest_book_snapshot()
        if frame.is_empty():
            frame = self._latest_bar_snapshot()
            if frame.is_empty():
                return []
            fallback_results: list[PriceSummary] = []
            for asset in frame["asset"].unique().to_list():
                asset_rows = frame.filter(pl.col("asset") == str(asset)).sort("ts").tail(1)
                if asset_rows.is_empty():
                    continue
                row = asset_rows.row(0, named=True)
                close = _float_or_none(row.get("close"))
                fallback_results.append(
                    PriceSummary(
                        asset=str(asset),
                        mid_price=close,
                        best_bid=None,
                        best_ask=None,
                        spread_bps=None,
                        venue=cast(str | None, row.get("venue")),
                        ts=cast(datetime | None, row.get("ts")),
                    )
                )
            return sorted(fallback_results, key=lambda p: p.asset)
        results: list[PriceSummary] = []
        for asset in frame["asset"].unique().to_list():
            asset_rows = self._preferred_book_rows(frame, str(asset))
            if asset_rows.is_empty():
                continue
            row = asset_rows.row(0, named=True)
            mid = _float_or_none(row.get("mid_price"))
            bid = _float_or_none(row.get("best_bid"))
            ask = _float_or_none(row.get("best_ask"))
            spread_bps = None
            if mid and mid > 0 and bid is not None and ask is not None:
                spread_bps = round((ask - bid) / mid * 10_000, 2)
            results.append(
                PriceSummary(
                    asset=str(asset),
                    mid_price=mid,
                    best_bid=bid,
                    best_ask=ask,
                    spread_bps=spread_bps,
                    venue=cast(str | None, row.get("venue")),
                    ts=cast(datetime | None, row.get("ts")),
                )
            )
        return sorted(results, key=lambda p: p.asset)

    def get_price_series(
        self,
        asset: str,
        *,
        venue_scope: str = "all",
        timeframe: str = "15m",
        start: datetime | None = None,
        end: datetime | None = None,
        max_points: int | None = None,
    ) -> PriceSeriesResponse:
        timeframe_seconds = _timeframe_to_seconds(timeframe)
        query_end = end or datetime.now(timezone.utc)
        query_start = start or (query_end - _default_price_lookback(timeframe_seconds))
        filters = {"asset": asset.upper(), "clock": "event_time", "bar_size_secs": 60}
        frame = self.cache.read(
            self.store,
            DatasetName.MICROSTRUCTURE_BAR,
            start=query_start,
            end=query_end,
            filters=filters,
        )
        latest_available_at: datetime | None = None
        stale = False
        if frame.is_empty():
            window_frame, latest_available_at, stale = self._recent_bar_frame(
                asset=asset,
                venue_scope=venue_scope,
            )
            if not window_frame.is_empty():
                fallback_end = latest_available_at or query_end
                fallback_start = fallback_end - _default_price_lookback(timeframe_seconds)
                frame = window_frame.filter(
                    (pl.col("ts") >= fallback_start) & (pl.col("ts") <= fallback_end)
                )
                query_start = fallback_start
                query_end = fallback_end
            if frame.is_empty():
                return PriceSeriesResponse(
                    asset=asset.upper(),
                    venue_scope=venue_scope,
                    timeframe=timeframe,
                    points=[],
                    latest_values={},
                    context={
                        "bar_size_secs": timeframe_seconds,
                        "latest_available_at": latest_available_at,
                        "stale": stale,
                    },
                )
        if venue_scope != "all":
            frame = frame.filter(pl.col("venue_group") == venue_scope)
        if frame.is_empty():
            return PriceSeriesResponse(
                asset=asset.upper(),
                venue_scope=venue_scope,
                timeframe=timeframe,
                points=[],
                latest_values={},
                context={
                    "bar_size_secs": timeframe_seconds,
                    "latest_available_at": latest_available_at,
                    "stale": stale,
                },
            )
        aggregation_mode = "median_across_venues"
        source_venue: str | None = None
        source_frame = frame
        if venue_scope == "all":
            base = (
                source_frame.group_by(["ts", "asset"])
                .agg(
                    [
                        # A robust market-wide candle should not let one stale venue
                        # dominate the wick range for every aggregate bar.
                        pl.col("open").median().alias("open"),
                        pl.col("high").median().alias("high"),
                        pl.col("low").median().alias("low"),
                        pl.col("close").median().alias("close"),
                        pl.col("volume").sum().alias("volume"),
                        pl.col("trade_count").sum().alias("trade_count"),
                    ]
                )
                .with_columns(
                    [
                        pl.max_horizontal("high", "open", "close").alias("high"),
                        pl.min_horizontal("low", "open", "close").alias("low"),
                    ]
                )
                .sort("ts")
            )
        else:
            available_venues = {
                str(venue)
                for venue in source_frame.get_column("venue").drop_nulls().unique().to_list()
            }
            preferred_candidates = PREFERRED_PRICE_VENUES.get(venue_scope, ())
            source_venue = next(
                (candidate for candidate in preferred_candidates if candidate in available_venues),
                None,
            )
            if source_venue is None and available_venues:
                source_venue = sorted(available_venues)[0]
            if source_venue is not None:
                source_frame = source_frame.filter(pl.col("venue") == source_venue)
            aggregation_mode = "preferred_venue"
            base = (
                source_frame.group_by(["ts", "asset"])
                .agg(
                    [
                        pl.col("open").first().alias("open"),
                        pl.col("high").max().alias("high"),
                        pl.col("low").min().alias("low"),
                        pl.col("close").last().alias("close"),
                        pl.col("volume").sum().alias("volume"),
                        pl.col("trade_count").sum().alias("trade_count"),
                    ]
                )
                .with_columns(
                    [
                        pl.max_horizontal("high", "open", "close").alias("high"),
                        pl.min_horizontal("low", "open", "close").alias("low"),
                    ]
                )
                .sort("ts")
            )
        latest_available_at = cast(datetime | None, source_frame.get_column("ts").max())
        aggregated = (
            base.group_by_dynamic("ts", every=f"{timeframe_seconds}s", closed="left")
            .agg(
                [
                    pl.col("asset").first().alias("asset"),
                    pl.col("open").first().alias("open"),
                    pl.col("high").max().alias("high"),
                    pl.col("low").min().alias("low"),
                    pl.col("close").last().alias("close"),
                    pl.col("volume").sum().alias("volume"),
                    pl.col("trade_count").sum().cast(pl.Int64).alias("trade_count"),
                ]
            )
            .with_columns(
                [
                    pl.max_horizontal("high", "open", "close").alias("high"),
                    pl.min_horizontal("low", "open", "close").alias("low"),
                ]
            )
            .sort("ts")
        )
        if max_points and aggregated.height > max_points:
            stride = max(1, math.ceil(aggregated.height / max_points))
            aggregated = aggregated.slice(0, aggregated.height).gather_every(stride)
        points = [
            PriceCandlePoint(
                ts=cast(datetime, row["ts"]),
                asset=str(row["asset"]),
                venue_scope=venue_scope,
                timeframe=timeframe,
                open=_float_or_none(row.get("open")),
                high=_float_or_none(row.get("high")),
                low=_float_or_none(row.get("low")),
                close=_float_or_none(row.get("close")),
                volume=_float_or_none(row.get("volume")),
                trade_count=_int_or_none(row.get("trade_count")),
            )
            for row in aggregated.to_dicts()
        ]
        latest = aggregated.tail(1)
        latest_values = {}
        latest_point_ts: datetime | None = None
        if not latest.is_empty():
            latest_point_ts = cast(datetime | None, latest.item(0, "ts"))
            latest_values = {
                "open": _float_or_none(latest.item(0, "open")),
                "high": _float_or_none(latest.item(0, "high")),
                "low": _float_or_none(latest.item(0, "low")),
                "close": _float_or_none(latest.item(0, "close")),
                "volume": _float_or_none(latest.item(0, "volume")),
                "trade_count": _int_or_none(latest.item(0, "trade_count")),
            }
        freshest_available_at = latest_available_at or latest_point_ts
        if freshest_available_at is not None:
            lag_seconds = (query_end - freshest_available_at).total_seconds()
            stale = stale or lag_seconds > max(float(timeframe_seconds * 2), 30 * 60)
        context = {
            "bar_size_secs": timeframe_seconds,
            "start": query_start,
            "end": query_end,
            "point_count": len(points),
            "aggregation_mode": aggregation_mode,
            "latest_available_at": freshest_available_at,
            "source_venue": source_venue,
            "stale": stale,
        }
        return PriceSeriesResponse(
            asset=asset.upper(),
            venue_scope=venue_scope,
            timeframe=timeframe,
            points=points,
            latest_values=latest_values,
            context=context,
        )

    def _latest_bar_snapshot(self) -> pl.DataFrame:
        frame, _, _ = self._recent_bar_frame()
        if frame.is_empty():
            return frame
        return frame.sort("ts").group_by("asset").tail(1)

    def _latest_book_snapshot(self) -> pl.DataFrame:
        latest = self.store.latest_logical_ts(DatasetName.MICROSTRUCTURE_BOOK_STATE)
        if latest is None:
            return empty_frame(DatasetName.MICROSTRUCTURE_BOOK_STATE)
        frame = self.cache.read(
            self.store,
            DatasetName.MICROSTRUCTURE_BOOK_STATE,
            start=latest - timedelta(minutes=1),
            end=latest,
        )
        if frame.is_empty():
            return frame
        return frame.filter(pl.col("logical_ts") == latest) if "logical_ts" in frame.columns else frame

    def _preferred_book_rows(self, frame: pl.DataFrame, asset: str) -> pl.DataFrame:
        asset_rows = frame.filter(pl.col("asset") == asset)
        if asset_rows.is_empty():
            return asset_rows
        latest_ts = cast(datetime | None, asset_rows.select(pl.max("ts")).item())
        if latest_ts is not None:
            asset_rows = asset_rows.filter(pl.col("ts") == latest_ts)
        ranked = asset_rows.with_columns(
            [
                (
                    pl.col("mid_price").is_not_null()
                    | (pl.col("best_bid").is_not_null() & pl.col("best_ask").is_not_null())
                )
                .cast(pl.Int8)
                .alias("_has_price"),
                pl.when(pl.col("venue").eq("binance_spot"))
                .then(pl.lit(0))
                .when(pl.col("venue").eq("binance_futures"))
                .then(pl.lit(1))
                .when(pl.col("venue").eq("bybit_spot"))
                .then(pl.lit(2))
                .when(pl.col("venue").eq("bybit_perps"))
                .then(pl.lit(3))
                .otherwise(pl.lit(9))
                .alias("_venue_rank"),
            ]
        )
        return ranked.sort(["_has_price", "_venue_rank"], descending=[True, False]).head(1)


class DefaultOptionsDashboardProvider:
    def __init__(self, store: DatasetStore, cache: ReadCache) -> None:
        self.store = store
        self.cache = cache

    def list_dashboards(self) -> list[OptionsDashboardSummary]:
        return [
            self.get_latest(dashboard_id, underlying=definition.default_underlying)
            for dashboard_id, definition in OPTIONS_DASHBOARD_DEFINITIONS.items()
        ]

    def get_latest(self, dashboard_id: str, *, underlying: str | None) -> OptionsDashboardSummary:
        definition = _options_dashboard_definition(dashboard_id)
        chosen_underlying = underlying or definition.default_underlying
        frame = (
            self._read_dashboard_frame(definition, chosen_underlying, latest_only=True)
            .sort("ts")
            .tail(1)
        )
        latest_ts = None if frame.is_empty() else cast(datetime, frame.item(0, "ts"))
        latest_quality = (
            None if frame.is_empty() else cast(str | None, frame.item(0, "quality_flag"))
        )
        latest_values = (
            {}
            if frame.is_empty()
            else {
                key: _coerce_value(frame.item(0, key))
                for key in definition.latest_keys
                if key in frame.columns
            }
        )
        return OptionsDashboardSummary(
            id=definition.id,
            name=definition.name,
            family="options",
            description=definition.description,
            supported_underlyings=list(definition.supported_underlyings),
            latest_updated_at=latest_ts,
            latest_quality=latest_quality,
            latest_values=latest_values,
        )

    def get_series(
        self,
        dashboard_id: str,
        *,
        underlying: str | None,
        start: datetime | None,
        end: datetime | None,
        quality: str | None,
        max_points: int | None,
    ) -> OptionsDashboardResponse:
        definition = _options_dashboard_definition(dashboard_id)
        chosen_underlying = underlying or definition.default_underlying
        frame = self._read_dashboard_frame(
            definition,
            chosen_underlying,
            start=start,
            end=end,
        ).sort("ts")
        if quality and "quality_flag" in frame.columns:
            frame = frame.filter(pl.col("quality_flag") == quality)
        if max_points and frame.height > max_points:
            frame = frame.gather_every(max(1, math.ceil(frame.height / max_points)))
        points = [
            OptionsSeriesPoint(
                ts=cast(datetime, row["ts"]),
                underlying=cast(str | None, row.get("underlying")),
                pair_id=cast(str | None, row.get("pair_id")),
                tenor=cast(str | None, row.get("tenor")),
                tenor_bucket=cast(str | None, row.get("tenor_bucket")),
                moneyness_bucket=cast(str | None, row.get("moneyness_bucket")),
                expiry_bucket=cast(str | None, row.get("expiry_bucket")),
                strike_bucket=cast(str | None, row.get("strike_bucket")),
                values={
                    key: _coerce_value(value)
                    for key, value in row.items()
                    if key
                    not in {
                        "ts",
                        "date",
                        "underlying",
                        "source",
                        "pair_id",
                        "tenor",
                        "tenor_bucket",
                        "moneyness_bucket",
                        "expiry_bucket",
                        "strike_bucket",
                        "quality_flag",
                        "run_id",
                        "run_ts",
                        "logical_ts",
                        "write_mode",
                    }
                },
                quality_flag=cast(str | None, row.get("quality_flag")),
            )
            for row in frame.iter_rows(named=True)
        ]
        latest = self.get_latest(dashboard_id, underlying=chosen_underlying)
        return OptionsDashboardResponse(
            dashboard_id=dashboard_id,
            underlying=chosen_underlying,
            points=points,
            latest_values=latest.latest_values,
            context={
                "description": definition.description,
                "group_columns": list(definition.group_columns),
                "points": len(points),
                "latest_quality": latest.latest_quality,
            },
        )

    def _read_dashboard_frame(
        self,
        definition: OptionsDashboardDefinition,
        underlying: str | None,
        *,
        start: datetime | None = None,
        end: datetime | None = None,
        latest_only: bool = False,
    ) -> pl.DataFrame:
        filters: dict[str, Any] = {"source": definition.default_source}
        if underlying and "underlying" in DATASET_SPECS[definition.dataset].schema:
            filters["underlying"] = underlying
        if latest_only:
            if definition.dataset == DatasetName.VOL_REGIME_TRANSITION_SERIES:
                return self.cache.read(self.store, definition.dataset, filters=filters)
            return self.cache.read_latest(self.store, definition.dataset, filters=filters)
        return self.cache.read(
            self.store,
            definition.dataset,
            start=start,
            end=end,
            filters=filters,
        )


class DefaultOptionsSignalProvider:
    def __init__(self, store: DatasetStore, cache: ReadCache) -> None:
        self.store = store
        self.cache = cache

    def list_signals(self) -> list[OptionsSignalSummary]:
        return [
            self.get_latest(signal_id, underlying=definition.default_underlying)
            for signal_id, definition in OPTIONS_SIGNAL_DEFINITIONS.items()
        ]

    def get_latest(self, signal_id: str, *, underlying: str) -> OptionsSignalSummary:
        definition = _options_signal_definition(signal_id)
        frame = (
            self.cache.read_latest(
                self.store,
                DatasetName.OPTIONS_SIGNAL_SERIES,
                filters={"underlying": underlying, "source": definition.default_source},
            )
            .sort("ts")
            .tail(1)
        )
        latest_ts = None if frame.is_empty() else cast(datetime, frame.item(0, "ts"))
        latest_quality = (
            None if frame.is_empty() else cast(str | None, frame.item(0, "quality_flag"))
        )
        latest_values = (
            {}
            if frame.is_empty()
            else {
                key: _coerce_value(frame.item(0, key))
                for key in definition.latest_keys
                if key in frame.columns
            }
        )
        return OptionsSignalSummary(
            id=signal_id,
            name=definition.name,
            description=definition.description,
            supported_underlyings=list(definition.supported_underlyings),
            latest_updated_at=latest_ts,
            latest_quality=latest_quality,
            latest_values=latest_values,
        )

    def get_series(
        self,
        signal_id: str,
        *,
        underlying: str,
        start: datetime | None,
        end: datetime | None,
        quality: str | None,
        max_points: int | None,
    ) -> OptionsSignalSeriesResponse:
        definition = _options_signal_definition(signal_id)
        frame = self.cache.read(
            self.store,
            DatasetName.OPTIONS_SIGNAL_SERIES,
            start=start,
            end=end,
            filters={"underlying": underlying, "source": definition.default_source},
        ).sort("ts")
        if quality:
            frame = frame.filter(pl.col("quality_flag") == quality)
        if max_points and frame.height > max_points:
            frame = frame.gather_every(max(1, math.ceil(frame.height / max_points)))
        points = [
            OptionsSeriesPoint(
                ts=cast(datetime, row["ts"]),
                underlying=underlying,
                tenor=cast(str | None, row.get("tenor")),
                tenor_bucket=cast(str | None, row.get("tenor_bucket")),
                moneyness_bucket=cast(str | None, row.get("moneyness_bucket")),
                expiry_bucket=cast(str | None, row.get("expiry_bucket")),
                strike_bucket=cast(str | None, row.get("strike_bucket")),
                values={
                    key: _coerce_value(value)
                    for key, value in row.items()
                    if key
                    not in {
                        "ts",
                        "date",
                        "underlying",
                        "source",
                        "tenor",
                        "tenor_bucket",
                        "moneyness_bucket",
                        "expiry_bucket",
                        "strike_bucket",
                        "quality_flag",
                        "run_id",
                        "run_ts",
                        "logical_ts",
                        "write_mode",
                    }
                },
                quality_flag=cast(str | None, row.get("quality_flag")),
            )
            for row in frame.iter_rows(named=True)
        ]
        latest = self.get_latest(signal_id, underlying=underlying)
        return OptionsSignalSeriesResponse(
            signal_id=signal_id,
            underlying=underlying,
            points=points,
            latest_values=latest.latest_values,
            context={"description": definition.description, "points": len(points)},
        )


class DefaultPerpsDashboardProvider:
    def __init__(self, store: DatasetStore, cache: ReadCache) -> None:
        self.store = store
        self.cache = cache

    def list_dashboards(self) -> list[PerpsDashboardSummary]:
        return [
            self.get_latest(dashboard_id, asset=definition.default_asset, venue_scope="all")
            for dashboard_id, definition in PERPS_DASHBOARD_DEFINITIONS.items()
        ]

    def get_latest(
        self,
        dashboard_id: str,
        *,
        asset: str,
        venue_scope: str,
    ) -> PerpsDashboardSummary:
        definition = _perps_dashboard_definition(dashboard_id)
        frame = (
            self.cache.read_latest(
                self.store,
                definition.dataset,
                filters={
                    "asset": asset,
                    "venue_scope": venue_scope,
                    "source": definition.default_source,
                },
            )
            .sort("ts")
            .tail(1)
        )
        latest_ts = None if frame.is_empty() else cast(datetime, frame.item(0, "ts"))
        latest_values = (
            {}
            if frame.is_empty()
            else {
                key: _coerce_value(frame.item(0, key))
                for key in definition.latest_keys
                if key in frame.columns
            }
        )
        latest_quality = self._dashboard_quality(definition, frame)
        return PerpsDashboardSummary(
            id=definition.id,
            name=definition.name,
            family="perps",
            description=definition.description,
            supported_assets=list(definition.supported_assets),
            supported_venue_scopes=["all", "binance", "bybit"],
            latest_updated_at=latest_ts,
            latest_quality=latest_quality,
            latest_values=latest_values,
        )

    def _dashboard_quality(
        self,
        definition: PerpsDashboardDefinition,
        frame: pl.DataFrame,
    ) -> str | None:
        if frame.is_empty():
            return None
        raw_quality = cast(str | None, frame.item(0, "quality_flag"))
        if raw_quality == "insufficient_history":
            return raw_quality
        present = 0
        checked = 0
        for key in definition.latest_keys:
            if key not in frame.columns:
                continue
            checked += 1
            value = frame.item(0, key)
            if value is not None:
                present += 1
        if checked == 0:
            return raw_quality
        if present == 0:
            return raw_quality or "unknown"
        if raw_quality == "partial_context" and present / checked >= (2 / 3):
            return "ok"
        return raw_quality or "ok"

    def get_series(
        self,
        dashboard_id: str,
        *,
        asset: str,
        venue_scope: str,
        start: datetime | None,
        end: datetime | None,
        quality: str | None,
        max_points: int | None,
    ) -> PerpsDashboardResponse:
        definition = _perps_dashboard_definition(dashboard_id)
        frame = self.cache.read(
            self.store,
            definition.dataset,
            start=start,
            end=end,
            filters={
                "asset": asset,
                "venue_scope": venue_scope,
                "source": definition.default_source,
            },
        ).sort("ts")
        if quality:
            frame = frame.filter(pl.col("quality_flag") == quality)
        if max_points and frame.height > max_points:
            frame = frame.gather_every(max(1, math.ceil(frame.height / max_points)))
        points = [
            PerpsSeriesPoint(
                ts=cast(datetime, row["ts"]),
                asset=cast(str | None, row.get("asset")),
                venue_scope=cast(str | None, row.get("venue_scope")),
                values={
                    key: _coerce_value(value)
                    for key, value in row.items()
                    if key
                    not in {
                        "ts",
                        "date",
                        "asset",
                        "venue_scope",
                        "source",
                        "quality_flag",
                        "run_id",
                        "run_ts",
                        "logical_ts",
                        "write_mode",
                    }
                },
                quality_flag=cast(str | None, row.get("quality_flag")),
            )
            for row in frame.iter_rows(named=True)
        ]
        latest = self.get_latest(dashboard_id, asset=asset, venue_scope=venue_scope)
        return PerpsDashboardResponse(
            dashboard_id=dashboard_id,
            asset=asset,
            venue_scope=venue_scope,
            points=points,
            latest_values=latest.latest_values,
            context={"description": definition.description, "points": len(points)},
        )


class DefaultPerpsSignalProvider:
    def __init__(self, store: DatasetStore, cache: ReadCache) -> None:
        self.store = store
        self.cache = cache

    def list_signals(self) -> list[PerpsSignalSummary]:
        return [
            self.get_latest(signal_id, asset=definition.default_asset, venue_scope="all")
            for signal_id, definition in PERPS_SIGNAL_DEFINITIONS.items()
        ]

    def get_latest(self, signal_id: str, *, asset: str, venue_scope: str) -> PerpsSignalSummary:
        definition = _perps_signal_definition(signal_id)
        frame = (
            self.cache.read_latest(
                self.store,
                DatasetName.PERPS_COMPOSITE_SERIES,
                filters={
                    "asset": asset,
                    "venue_scope": venue_scope,
                    "source": definition.default_source,
                },
            )
            .sort("ts")
            .tail(1)
        )
        latest_ts = None if frame.is_empty() else cast(datetime, frame.item(0, "ts"))
        latest_quality = (
            None if frame.is_empty() else cast(str | None, frame.item(0, "quality_flag"))
        )
        latest_values = (
            {}
            if frame.is_empty()
            else {
                key: _coerce_value(frame.item(0, key))
                for key in definition.latest_keys
                if key in frame.columns
            }
        )
        return PerpsSignalSummary(
            id=signal_id,
            name=definition.name,
            description=definition.description,
            supported_assets=list(definition.supported_assets),
            supported_venue_scopes=["all", "binance", "bybit"],
            latest_updated_at=latest_ts,
            latest_quality=latest_quality,
            latest_values=latest_values,
        )

    def get_series(
        self,
        signal_id: str,
        *,
        asset: str,
        venue_scope: str,
        start: datetime | None,
        end: datetime | None,
        quality: str | None,
        max_points: int | None,
    ) -> PerpsSignalSeriesResponse:
        definition = _perps_signal_definition(signal_id)
        frame = self.cache.read(
            self.store,
            DatasetName.PERPS_COMPOSITE_SERIES,
            start=start,
            end=end,
            filters={
                "asset": asset,
                "venue_scope": venue_scope,
                "source": definition.default_source,
            },
        ).sort("ts")
        if quality:
            frame = frame.filter(pl.col("quality_flag") == quality)
        if max_points and frame.height > max_points:
            frame = frame.gather_every(max(1, math.ceil(frame.height / max_points)))
        points = [
            PerpsSeriesPoint(
                ts=cast(datetime, row["ts"]),
                asset=cast(str | None, row.get("asset")),
                venue_scope=cast(str | None, row.get("venue_scope")),
                values={
                    key: _coerce_value(value)
                    for key, value in row.items()
                    if key
                    not in {
                        "ts",
                        "date",
                        "asset",
                        "venue_scope",
                        "source",
                        "quality_flag",
                        "run_id",
                        "run_ts",
                        "logical_ts",
                        "write_mode",
                    }
                },
                quality_flag=cast(str | None, row.get("quality_flag")),
            )
            for row in frame.iter_rows(named=True)
        ]
        latest = self.get_latest(signal_id, asset=asset, venue_scope=venue_scope)
        return PerpsSignalSeriesResponse(
            signal_id=signal_id,
            asset=asset,
            venue_scope=venue_scope,
            points=points,
            latest_values=latest.latest_values,
            context={"description": definition.description, "points": len(points)},
        )


class DefaultCompositeProvider:
    def __init__(self, store: DatasetStore, cache: ReadCache) -> None:
        self.store = store
        self.cache = cache

    def list_signals(self) -> list[CompositeSignalSummary]:
        return [self.get_latest(signal_id) for signal_id in COMPOSITE_SIGNAL_IDS]

    def get_latest(self, signal_id: str) -> CompositeSignalSummary:
        definition = _composite_signal_definition(signal_id)
        frame = (
            self.cache.read_latest(
                self.store,
                DatasetName.COMPOSITE_SIGNAL_SERIES,
                filters={
                    "asset": definition.asset,
                    "scope": definition.scope,
                    "source": definition.default_source,
                },
            )
            .sort("ts")
            .tail(1)
        )
        latest_ts = None if frame.is_empty() else cast(datetime, frame.item(0, "ts"))
        latest_quality = (
            None if frame.is_empty() else cast(str | None, frame.item(0, "quality_flag"))
        )
        latest_values = (
            {}
            if frame.is_empty()
            else {
                key: _coerce_value(frame.item(0, key))
                for key in definition.latest_keys
                if key in frame.columns
            }
        )
        return CompositeSignalSummary(
            id=definition.id,
            name=definition.name,
            description=definition.description,
            scope=definition.scope,
            asset=definition.asset,
            latest_updated_at=latest_ts,
            latest_quality=latest_quality,
            latest_values=latest_values,
        )

    def get_series(
        self,
        signal_id: str,
        *,
        start: datetime | None,
        end: datetime | None,
        quality: str | None,
        max_points: int | None,
    ) -> CompositeSignalSeriesResponse:
        definition = _composite_signal_definition(signal_id)
        frame = self.cache.read(
            self.store,
            DatasetName.COMPOSITE_SIGNAL_SERIES,
            start=start,
            end=end,
            filters={
                "asset": definition.asset,
                "scope": definition.scope,
                "source": definition.default_source,
            },
        ).sort("ts")
        if quality:
            frame = frame.filter(pl.col("quality_flag") == quality)
        if max_points and frame.height > max_points:
            frame = frame.gather_every(max(1, math.ceil(frame.height / max_points)))
        points = [
            CompositeSeriesPoint(
                ts=cast(datetime, row["ts"]),
                scope=cast(str, row["scope"]),
                asset=cast(str, row["asset"]),
                values={
                    key: _coerce_value(value)
                    for key, value in row.items()
                    if key
                    not in {
                        "ts",
                        "date",
                        "scope",
                        "asset",
                        "source",
                        "quality_flag",
                        "run_id",
                        "run_ts",
                        "logical_ts",
                        "write_mode",
                    }
                },
                quality_flag=cast(str | None, row.get("quality_flag")),
            )
            for row in frame.iter_rows(named=True)
        ]
        latest = self.get_latest(signal_id)
        return CompositeSignalSeriesResponse(
            signal_id=signal_id,
            scope=definition.scope,
            asset=definition.asset,
            points=points,
            latest_values=latest.latest_values,
            context={"description": definition.description, "points": len(points)},
        )

    def get_decomposition(
        self,
        signal_id: str,
        *,
        start: datetime | None,
        end: datetime | None,
        quality: str | None,
        max_points: int | None,
    ) -> CompositeDecompositionResponse:
        definition = _composite_signal_definition(signal_id)
        frame = self.cache.read(
            self.store,
            DatasetName.COMPOSITE_COMPONENT_SERIES,
            start=start,
            end=end,
            filters={
                "asset": definition.asset,
                "scope": definition.scope,
                "source": definition.default_source,
            },
        ).sort("ts")
        if quality:
            frame = frame.filter(pl.col("quality_flag") == quality)
        if max_points and frame.height > max_points:
            frame = frame.gather_every(max(1, math.ceil(frame.height / max_points)))
        records = [_composite_component_record(row) for row in frame.iter_rows(named=True)]
        latest_ts = None if frame.is_empty() else cast(datetime, frame.item(-1, "ts"))
        latest_records = (
            []
            if latest_ts is None
            else sorted(
                [
                    _composite_component_record(row)
                    for row in frame.filter(pl.col("ts") == pl.lit(latest_ts)).iter_rows(named=True)
                ],
                key=lambda record: abs(record.signed_contribution or 0.0),
                reverse=True,
            )[:10]
        )
        return CompositeDecompositionResponse(
            signal_id=signal_id,
            scope=definition.scope,
            asset=definition.asset,
            records=records,
            latest_contributors=latest_records,
            context={"description": definition.description, "points": len(records)},
        )

    def get_transitions(
        self,
        signal_id: str,
        *,
        start: datetime | None,
        end: datetime | None,
        quality: str | None,
        max_points: int | None,
    ) -> CompositeTransitionResponse:
        definition = _composite_signal_definition(signal_id)
        frame = self.cache.read(
            self.store,
            DatasetName.COMPOSITE_TRANSITION_SERIES,
            start=start,
            end=end,
            filters={
                "asset": definition.asset,
                "scope": definition.scope,
                "source": definition.default_source,
            },
        ).sort("ts")
        if quality:
            frame = frame.filter(pl.col("quality_flag") == quality)
        if max_points and frame.height > max_points:
            frame = frame.gather_every(max(1, math.ceil(frame.height / max_points)))
        records = [
            CompositeTransitionRecord(
                ts=cast(datetime, row["ts"]),
                scope=cast(str, row["scope"]),
                asset=cast(str, row["asset"]),
                index_type=cast(str, row["index_type"]),
                current_regime=cast(str | None, row.get("current_regime")),
                prior_regime=cast(str | None, row.get("prior_regime")),
                stay_probability=_float_or_none(row.get("stay_probability")),
                upshift_probability=_float_or_none(row.get("upshift_probability")),
                downshift_probability=_float_or_none(row.get("downshift_probability")),
                median_regime_duration_hours=_float_or_none(
                    row.get("median_regime_duration_hours")
                ),
                quality_flag=cast(str | None, row.get("quality_flag")),
            )
            for row in frame.iter_rows(named=True)
        ]
        return CompositeTransitionResponse(
            signal_id=signal_id,
            scope=definition.scope,
            asset=definition.asset,
            records=records,
            context={"description": definition.description, "points": len(records)},
        )


class DefaultDatasetProvider:
    def __init__(self, store: DatasetStore, cache: ReadCache) -> None:
        self.store = store
        self.cache = cache

    def list_datasets(self) -> list[DatasetSummary]:
        return [self._build_summary(dataset) for dataset in UI_EXPOSED_DATASETS]

    def get_dataset(self, dataset_id: str) -> DatasetDetail:
        dataset = DatasetName(dataset_id)
        frame = self.cache.read(self.store, dataset, read_mode=ReadMode.ALL_RUNS)
        spec = DATASET_SPECS[dataset]
        sample_partitions = []
        if not frame.is_empty():
            select_cols = [column for column in spec.partition_columns if column in frame.columns]
            sample_partitions = frame.select(select_cols).unique().head(20).to_dicts()
        summary = self._build_summary(dataset)
        return DatasetDetail(
            **summary.model_dump(),
            sample_partitions=sample_partitions,
        )

    def get_rows(
        self,
        dataset_id: str,
        *,
        source: str | None,
        symbol: str | None,
        start: datetime | None,
        end: datetime | None,
        quality: str | None,
        limit: int,
    ) -> DatasetRowsResponse:
        dataset = DatasetName(dataset_id)
        filters: dict[str, Any] = {}
        if source and "source" in DATASET_SPECS[dataset].schema:
            filters["source"] = source
        if symbol and "underlying" in DATASET_SPECS[dataset].schema:
            filters["underlying"] = symbol
        elif symbol and "asset" in DATASET_SPECS[dataset].schema:
            filters["asset"] = symbol
        frame = self.cache.read(self.store, dataset, start=start, end=end, filters=filters)
        if quality and "quality_flag" in frame.columns:
            frame = frame.filter(pl.col("quality_flag") == quality)
        rows = frame.sort("ts", descending=True).head(limit).to_dicts()
        return DatasetRowsResponse(dataset_id=dataset_id, rows=rows, limit=limit)

    def _build_summary(self, dataset: DatasetName) -> DatasetSummary:
        frame = self.cache.read(self.store, dataset, read_mode=ReadMode.ALL_RUNS)
        spec = DATASET_SPECS[dataset]
        latest_ts = (
            None if frame.is_empty() else cast(datetime, frame.sort("ts").tail(1).item(0, "ts"))
        )
        latest_logical_ts = (
            None
            if frame.is_empty()
            else cast(datetime, frame.sort("logical_ts").tail(1).item(0, "logical_ts"))
        )
        return DatasetSummary(
            id=dataset.value,
            name=dataset.value.replace("_", " ").title(),
            rows=frame.height,
            latest_ts=latest_ts,
            latest_logical_ts=latest_logical_ts,
            partition_columns=list(spec.partition_columns),
            primary_key=list(spec.primary_key),
            schema_columns=list(spec.schema.keys()),
        )


class DefaultRunStatusProvider:
    def __init__(
        self,
        store: DatasetStore,
        cache: ReadCache,
        signal_provider: DefaultSignalProvider,
        microstructure_provider: DefaultMicrostructureProvider,
        perps_signal_provider: DefaultPerpsSignalProvider,
        composite_provider: DefaultCompositeProvider,
        settings: ApiSettings,
    ) -> None:
        self.store = store
        self.cache = cache
        self.signal_provider = signal_provider
        self.microstructure_provider = microstructure_provider
        self.perps_signal_provider = perps_signal_provider
        self.composite_provider = composite_provider
        self.settings = settings

    def list_runs(
        self,
        *,
        start: datetime | None,
        end: datetime | None,
        limit: int,
        quality: str | None,
    ) -> list[RunRecord]:
        frame = self.cache.read(
            self.store,
            DatasetName.INGESTION_RUNS,
            start=start,
            end=end,
            read_mode=ReadMode.ALL_RUNS,
        ).sort("run_ts", descending=True)
        if quality:
            frame = frame.filter(pl.col("status") == quality)
        return [
            RunRecord(
                run_id=str(row["run_id"]),
                logical_ts=cast(datetime, row["logical_ts"]),
                run_ts=cast(datetime, row["run_ts"]),
                dataset=str(row["dataset"]),
                venue=str(row["venue"]),
                underlying=str(row["underlying"]),
                status=str(row["status"]),
                rows_written_raw=int(row["rows_written_raw"]),
                rows_written_normalized=int(row["rows_written_normalized"]),
                error_message=cast(str | None, row["error_message"]),
                duration_ms=int(row["duration_ms"]),
            )
            for row in frame.head(limit).iter_rows(named=True)
        ]

    def get_summary(self) -> RunsSummary:
        frame = self.cache.read(self.store, DatasetName.INGESTION_RUNS, read_mode=ReadMode.ALL_RUNS)
        latest_logical_ts = (
            None
            if frame.is_empty()
            else cast(datetime, frame.sort("logical_ts").tail(1).item(0, "logical_ts"))
        )
        latest_run_lag = None
        if latest_logical_ts is not None:
            latest_run_lag = (
                datetime.now(tz=timezone.utc) - latest_logical_ts.astimezone(timezone.utc)
            ).total_seconds() / 3600.0
        return RunsSummary(
            total_runs=frame.height,
            success_count=frame.filter(pl.col("status") == "success").height,
            failed_count=frame.filter(pl.col("status") == "failed").height,
            gap_count=frame.filter(pl.col("status") == "gap").height,
            skipped_count=frame.filter(pl.col("status") == "skipped_existing").height,
            latest_logical_ts=latest_logical_ts,
            latest_run_lag_hours=latest_run_lag,
        )

    def list_engines(self) -> list[EngineSummary]:
        return [
            EngineSummary(
                id=EngineId.CORE_ANALYTICS.value,
                name="Core Analytics",
                status="online",
                detail="Python-backed research API. Rust engine adapters can register later.",
            ),
            EngineSummary(
                id=EngineId.MICROSTRUCTURE_LIVE_ENGINE.value,
                name="Microstructure Live Engine",
                status=self.microstructure_provider.get_health().engine_status,
                detail="Rust-backed live feed, inference, and paper-signal engine scaffold.",
            ),
            EngineSummary(
                id=EngineId.EXECUTION_ENGINE.value,
                name="Execution Engine",
                status="online",
                detail="Rust-backed venue adapter and operator execution scaffold.",
            ),
        ]

    def get_health(self) -> SystemHealth:
        signals = self.signal_provider.list_signals()
        microstructure_signals = self.microstructure_provider.list_signals()
        perps_signals = self.perps_signal_provider.list_signals()
        composite_signals = self.composite_provider.list_signals()
        latest_signal_update = None
        timestamps = (
            [signal.latest_updated_at for signal in signals if signal.latest_updated_at]
            + [
                signal.latest_updated_at
                for signal in microstructure_signals
                if signal.latest_updated_at
            ]
            + [signal.latest_updated_at for signal in perps_signals if signal.latest_updated_at]
            + [signal.latest_updated_at for signal in composite_signals if signal.latest_updated_at]
        )
        if timestamps:
            latest_signal_update = max(timestamps)
        latest_run_update = self.store.latest_logical_ts(DatasetName.INGESTION_RUNS)
        db_health = self.store.db_health()
        return SystemHealth(
            status="ok",
            api_version="v1",
            data_root=str(self.settings.data_root),
            websocket_enabled=True,
            dataset_count=len(UI_EXPOSED_DATASETS),
            signal_count=len(signals)
            + len(microstructure_signals)
            + len(perps_signals)
            + len(composite_signals),
            latest_signal_update=latest_signal_update,
            latest_run_update=latest_run_update,
            db_enabled=bool(db_health.get("enabled")),
            db_status=str(db_health.get("status")),
        )


class PollingStreamProvider:
    def __init__(
        self,
        store: DatasetStore,
        settings: ApiSettings,
    ) -> None:
        self.store = store
        self.settings = settings

    async def stream(self) -> AsyncIterator[StreamEvent]:
        previous: dict[str, str] = {}
        while True:
            signal_snapshot: dict[str, Any] = {
                "latest_vvix_ts": self.store.latest_logical_ts(DatasetName.VVIX_SERIES),
                "latest_razor_ts": self.store.latest_logical_ts(DatasetName.RAZOR_SERIES),
                "latest_options_signal_ts": self.store.latest_logical_ts(
                    DatasetName.OPTIONS_SIGNAL_SERIES
                ),
                "latest_microstructure_signal_ts": self.store.latest_logical_ts(
                    DatasetName.MICROSTRUCTURE_MODEL_OUTPUT
                ),
                "latest_perps_signal_ts": self.store.latest_logical_ts(
                    DatasetName.PERPS_COMPOSITE_SERIES
                ),
            }
            run_snapshot: dict[str, Any] = {
                "latest_run_ts": self.store.latest_logical_ts(DatasetName.INGESTION_RUNS),
            }
            engine_snapshot: dict[str, Any] = {
                "latest_feature_ts": self.store.latest_logical_ts(
                    DatasetName.MICROSTRUCTURE_FEATURE_SERIES
                ),
                "latest_prediction_ts": self.store.latest_logical_ts(
                    DatasetName.MICROSTRUCTURE_MODEL_OUTPUT
                ),
            }
            snapshots: dict[str, dict[str, Any]] = {
                "signal.updated": signal_snapshot,
                "run.updated": run_snapshot,
                "engine.updated": engine_snapshot,
            }

            for event_name, payload in snapshots.items():
                encoded = repr(payload)
                if previous.get(event_name) != encoded:
                    previous[event_name] = encoded
                    yield StreamEvent(
                        event=event_name,
                        producer=(
                            ProducerId.DERIBIT_OPTIONS_ARCHIVER.value
                            if event_name == "run.updated"
                            else ProducerId.UFORIA_PIPELINE.value
                        ),
                        emitted_at=datetime.now(tz=timezone.utc),
                        data=payload,
                    )
            await asyncio.sleep(self.settings.stream_interval_seconds)


class DefaultExecutionProvider:
    def __init__(self, store: DatasetStore, cache: ReadCache) -> None:
        self.store = store
        self.cache = cache
        self.service = ExecutionService(store, get_config())

    def list_venues(self) -> list[ExecutionVenueSummary]:
        return [
            ExecutionVenueSummary(**item)
            for item in self.service.list_venues()
            if item["venue"] in EXECUTION_VENUES
        ]

    def list_markets(
        self,
        *,
        venue: str | None,
        asset: str | None,
    ) -> list[ExecutionMarketSummary]:
        return [
            ExecutionMarketSummary(**item)
            for item in self.service.list_markets(venue=venue, asset=asset)
        ]

    def list_accounts(self) -> list[ExecutionAccountSummary]:
        return [
            ExecutionAccountSummary(**item.model_dump(mode="json"))
            for item in self.service.list_accounts()
        ]

    def list_orders(
        self,
        *,
        venue: str | None,
        account: str | None,
        asset: str | None,
        status: str | None,
        limit: int,
    ) -> list[ExecutionOrderRecord]:
        return [
            ExecutionOrderRecord(**item)
            for item in self.service.list_orders(
                venue=venue,
                account=account,
                asset=asset,
                status=status,
                limit=limit,
            )
        ]

    def list_fills(
        self,
        *,
        venue: str | None,
        account: str | None,
        asset: str | None,
        limit: int,
    ) -> list[ExecutionFillRecord]:
        return [
            ExecutionFillRecord(**item)
            for item in self.service.list_fills(
                venue=venue,
                account=account,
                asset=asset,
                limit=limit,
            )
        ]

    def list_positions(
        self,
        *,
        venue: str | None,
        account: str | None,
        asset: str | None,
    ) -> list[ExecutionPositionRecord]:
        return [
            ExecutionPositionRecord(**item)
            for item in self.service.list_positions(venue=venue, account=account, asset=asset)
        ]

    def list_balances(
        self,
        *,
        venue: str | None,
        account: str | None,
    ) -> list[ExecutionBalanceRecord]:
        return [
            ExecutionBalanceRecord(**item)
            for item in self.service.list_balances(venue=venue, account=account)
        ]

    def get_health(self) -> list[ExecutionHealthRecord]:
        return [ExecutionHealthRecord(**item) for item in self.service.get_health()]

    def request_quote(self, request: ExecutionQuoteRequest) -> ExecutionActionResponse:
        result = self.service.request_quote(request)
        return ExecutionActionResponse(**result.model_dump(mode="json"))

    def place_order(self, request: ExecutionIntent) -> ExecutionActionResponse:
        result = self.service.submit_order(request)
        return ExecutionActionResponse(**result.model_dump(mode="json"))

    def cancel(self, request: ExecutionCancelRequest) -> ExecutionActionResponse:
        result = self.service.cancel(request)
        return ExecutionActionResponse(**result.model_dump(mode="json"))

    def cancel_all(self, request: ExecutionCancelAllRequest) -> ExecutionActionResponse:
        result = self.service.cancel_all(request)
        return ExecutionActionResponse(**result.model_dump(mode="json"))


def build_registry(store: DatasetStore, settings: ApiSettings) -> ProviderRegistry:
    cache = ReadCache(settings.read_cache_ttl_seconds)
    signal_provider = DefaultSignalProvider(store, cache)
    microstructure_provider = DefaultMicrostructureProvider(store, cache)
    price_provider = DefaultPriceProvider(store, cache)
    options_dashboard_provider = DefaultOptionsDashboardProvider(store, cache)
    options_signal_provider = DefaultOptionsSignalProvider(store, cache)
    perps_dashboard_provider = DefaultPerpsDashboardProvider(store, cache)
    perps_signal_provider = DefaultPerpsSignalProvider(store, cache)
    composite_provider = DefaultCompositeProvider(store, cache)
    execution_provider = DefaultExecutionProvider(store, cache)
    dataset_provider = DefaultDatasetProvider(store, cache)
    run_provider = DefaultRunStatusProvider(
        store,
        cache,
        signal_provider,
        microstructure_provider,
        perps_signal_provider,
        composite_provider,
        settings,
    )
    stream_provider = PollingStreamProvider(
        store,
        settings,
    )
    return ProviderRegistry(
        signal_provider=signal_provider,
        microstructure_provider=microstructure_provider,
        price_provider=price_provider,
        options_dashboard_provider=options_dashboard_provider,
        options_signal_provider=options_signal_provider,
        perps_dashboard_provider=perps_dashboard_provider,
        perps_signal_provider=perps_signal_provider,
        composite_provider=composite_provider,
        execution_provider=execution_provider,
        dataset_provider=dataset_provider,
        run_provider=run_provider,
        stream_provider=stream_provider,
    )


def _float_or_none(value: Any) -> float | None:
    if value is None:
        return None
    return float(value)


def _int_or_none(value: Any) -> int | None:
    if value is None:
        return None
    return int(value)


def _timeframe_to_seconds(timeframe: str) -> int:
    mapping = {
        "1m": 60,
        "5m": 300,
        "15m": 900,
        "1h": 3600,
        "4h": 14_400,
        "1d": 86_400,
    }
    return mapping.get(timeframe, 900)


def _default_price_lookback(timeframe_seconds: int) -> timedelta:
    if timeframe_seconds <= 60:
        return timedelta(hours=24)
    if timeframe_seconds <= 900:
        return timedelta(days=7)
    if timeframe_seconds <= 3600:
        return timedelta(days=30)
    return timedelta(days=120)


def _signal_definition(signal_id: str) -> SignalDefinition:
    definition = SIGNAL_DEFINITIONS.get(signal_id)
    if definition is None:
        raise KeyError(signal_id)
    return definition


def _microstructure_signal_definition(signal_id: str) -> MicrostructureSignalDefinition:
    definition = MICROSTRUCTURE_SIGNAL_DEFINITIONS.get(signal_id)
    if definition is None:
        raise KeyError(signal_id)
    return definition


def _source_or_default(definition: SignalDefinition, source: str | None) -> str:
    if source and source in definition.available_sources:
        return source
    return definition.default_source


def _options_dashboard_definition(dashboard_id: str) -> OptionsDashboardDefinition:
    definition = OPTIONS_DASHBOARD_DEFINITIONS.get(dashboard_id)
    if definition is None:
        raise KeyError(dashboard_id)
    return definition


def _options_signal_definition(signal_id: str) -> OptionsSignalDefinition:
    definition = OPTIONS_SIGNAL_DEFINITIONS.get(signal_id)
    if definition is None:
        raise KeyError(signal_id)
    return definition


def _perps_dashboard_definition(dashboard_id: str) -> PerpsDashboardDefinition:
    definition = PERPS_DASHBOARD_DEFINITIONS.get(dashboard_id)
    if definition is None:
        raise KeyError(dashboard_id)
    return definition


def _perps_signal_definition(signal_id: str) -> PerpsSignalDefinition:
    definition = PERPS_SIGNAL_DEFINITIONS.get(signal_id)
    if definition is None:
        raise KeyError(signal_id)
    return definition


def _composite_signal_definition(signal_id: str) -> CompositeSignalDefinition:
    definition = COMPOSITE_SIGNAL_DEFINITIONS.get(signal_id)
    if definition is None:
        raise KeyError(signal_id)
    return definition


def _composite_component_record(row: dict[str, Any]) -> CompositeComponentRecord:
    return CompositeComponentRecord(
        ts=cast(datetime, row["ts"]),
        scope=cast(str, row["scope"]),
        asset=cast(str, row["asset"]),
        pillar=cast(str, row["pillar"]),
        family=cast(str, row["family"]),
        component=cast(str, row["component"]),
        raw_value=_float_or_none(row.get("raw_value")),
        normalized_value=_float_or_none(row.get("normalized_value")),
        component_weight=_float_or_none(row.get("component_weight")),
        pillar_score=_float_or_none(row.get("pillar_score")),
        pillar_weight=_float_or_none(row.get("pillar_weight")),
        signed_contribution=_float_or_none(row.get("signed_contribution")),
        quality_flag=cast(str | None, row.get("quality_flag")),
    )


def _coerce_value(value: Any) -> Any:
    if value is None:
        return None
    if isinstance(value, (datetime, str, bool, int, float)):
        return value
    try:
        return float(value)
    except (TypeError, ValueError):
        return str(value)
