from __future__ import annotations

from dataclasses import dataclass
from datetime import datetime, timedelta
from typing import Any

import numpy as np
import polars as pl
from sklearn.isotonic import IsotonicRegression
from sklearn.linear_model import Ridge

from uforia.storage import DATASET_SPECS, DatasetName, DatasetStore
from uforia.storage.datasets import empty_frame

SOURCE = "composite_v1"
MARKET_ASSET = "MARKET"
MARKET_SCOPE = "market"
ASSET_SCOPE = "asset"
LOOKBACK_WINDOW_HOURS = 45 * 24
NORMALIZATION_WINDOW_HOURS = 14 * 24
NORMALIZATION_MIN_SAMPLES = 24
NORMALIZATION_FALLBACK_MIN_SAMPLES = 6
CALIBRATION_MIN_SAMPLES = 48
CALIBRATION_WINDOW_HOURS = 30 * 24
MARKET_WEIGHTS = {"BTC": 0.65, "ETH": 0.35}
HORIZON_WEIGHTS = {"30s": 0.15, "1m": 0.30, "2m": 0.25, "5m": 0.30}
PILLAR_WEIGHTS = {
    "sentiment": {"options": 0.40, "perps": 0.40, "micro": 0.20},
    "directional": {"micro": 0.60, "perps": 0.30, "options": 0.10},
    "fragility": {"perps": 0.40, "options": 0.40, "micro": 0.20},
}


@dataclass(frozen=True)
class ComponentSpec:
    pillar: str
    family: str
    component: str
    source_column: str
    sign: float = 1.0
    weight: float = 1.0


COMPONENT_SPECS: tuple[ComponentSpec, ...] = (
    ComponentSpec("sentiment", "options", "rvvix_14d", "vvix_rvvix_14d", sign=-1.0),
    ComponentSpec("sentiment", "options", "qvvix_30d", "vvix_qvvix_30d", sign=-1.0),
    ComponentSpec("sentiment", "options", "razor_pnl_zscore", "razor_razor_pnl_zscore"),
    ComponentSpec("sentiment", "options", "razor_vrp_zscore", "razor_razor_vrp_zscore"),
    ComponentSpec("sentiment", "options", "vol_carry_score", "options_vol_carry_score"),
    ComponentSpec(
        "sentiment",
        "options",
        "skew_stress_score",
        "options_skew_stress_score",
        sign=-1.0,
    ),
    ComponentSpec(
        "sentiment",
        "options",
        "ml_carry_toxicity_score",
        "options_ml_carry_toxicity_score",
    ),
    ComponentSpec(
        "sentiment", "options", "funding_skew_score", "options_funding_skew_score"
    ),
    ComponentSpec("sentiment", "options", "sessionality_score", "options_sessionality_score"),
    ComponentSpec("sentiment", "options", "dispersion_score", "options_dispersion_score"),
    ComponentSpec("sentiment", "options", "expiry_pin_score", "options_expiry_pin_score"),
    ComponentSpec(
        "sentiment", "options", "gamma_flip_score", "options_gamma_flip_score", sign=-1.0
    ),
    ComponentSpec("sentiment", "perps", "carry_score", "perps_composite_carry_score"),
    ComponentSpec(
        "sentiment", "perps", "positioning_score", "perps_composite_positioning_score"
    ),
    ComponentSpec(
        "sentiment",
        "perps",
        "cascade_risk_score",
        "perps_composite_cascade_risk_score",
        sign=-1.0,
    ),
    ComponentSpec(
        "sentiment", "perps", "fragility_score", "perps_composite_fragility_score", sign=-1.0
    ),
    ComponentSpec(
        "sentiment", "perps", "toxicity_score", "perps_composite_toxicity_score", sign=-1.0
    ),
    ComponentSpec("sentiment", "micro", "micro_sentiment", "micro_sentiment_raw"),
    ComponentSpec("directional", "micro", "micro_direction", "micro_direction_raw"),
    ComponentSpec(
        "directional", "perps", "carry_momentum_1h", "perps_signal_carry_momentum_1h"
    ),
    ComponentSpec(
        "directional", "perps", "carry_momentum_24h", "perps_signal_carry_momentum_24h"
    ),
    ComponentSpec(
        "directional",
        "perps",
        "cum_delta_acceleration",
        "perps_signal_cum_delta_acceleration",
    ),
    ComponentSpec(
        "directional",
        "perps",
        "depth_imbalance_momentum",
        "perps_signal_depth_imbalance_momentum",
    ),
    ComponentSpec("directional", "perps", "lead_lag_score", "perps_signal_lead_lag_score"),
    ComponentSpec(
        "directional", "perps", "depth_recovery_speed", "perps_signal_depth_recovery_speed"
    ),
    ComponentSpec(
        "directional",
        "perps",
        "oi_price_divergence",
        "perps_signal_oi_price_divergence",
        sign=-1.0,
        weight=0.5,
    ),
    ComponentSpec(
        "directional",
        "perps",
        "delta_divergence",
        "perps_signal_delta_divergence",
        sign=-1.0,
        weight=0.5,
    ),
    ComponentSpec(
        "directional", "options", "funding_skew_score", "options_funding_skew_score"
    ),
    ComponentSpec("directional", "options", "gamma_flip_score", "options_gamma_flip_score"),
    ComponentSpec(
        "directional", "options", "sessionality_score", "options_sessionality_score", weight=0.5
    ),
    ComponentSpec("fragility", "options", "rvvix_14d", "vvix_rvvix_14d"),
    ComponentSpec("fragility", "options", "qvvix_30d", "vvix_qvvix_30d"),
    ComponentSpec(
        "fragility", "options", "skew_stress_score", "options_skew_stress_score"
    ),
    ComponentSpec("fragility", "options", "gamma_flip_score", "options_gamma_flip_score"),
    ComponentSpec(
        "fragility",
        "perps",
        "cascade_risk_score",
        "perps_composite_cascade_risk_score",
    ),
    ComponentSpec("fragility", "perps", "toxicity_score", "perps_composite_toxicity_score"),
    ComponentSpec("fragility", "perps", "fragility_score", "perps_composite_fragility_score"),
    ComponentSpec("fragility", "perps", "spread_zscore_4h", "perps_signal_spread_zscore_4h"),
    ComponentSpec("fragility", "perps", "oi_concentration", "perps_signal_oi_concentration"),
    ComponentSpec("fragility", "micro", "micro_fragility", "micro_fragility_raw"),
)

SENTIMENT_REGIMES = ((20.0, "panic"), (40.0, "fear"), (60.0, "neutral"), (80.0, "greed"))
DIRECTIONAL_REGIMES = (
    (-40.0, "bearish"),
    (-15.0, "mild_bearish"),
    (15.0, "neutral"),
    (40.0, "mild_bullish"),
)
FRAGILITY_REGIMES = ((25.0, "stable"), (50.0, "watch"), (75.0, "fragile"))
REGIME_ORDER = {
    "sentiment": ["panic", "fear", "neutral", "greed", "euphoria"],
    "directional": ["bearish", "mild_bearish", "neutral", "mild_bullish", "bullish"],
    "fragility": ["stable", "watch", "fragile", "squeeze_risk"],
}


def _frame_from_rows(dataset: DatasetName, rows: list[dict[str, Any]]) -> pl.DataFrame:
    if not rows:
        return empty_frame(dataset)
    return pl.DataFrame(rows, schema=DATASET_SPECS[dataset].schema, strict=False)


def build_composite_signal_series(
    store: DatasetStore,
    start: datetime,
    end: datetime,
    *,
    asset: str,
    source: str = SOURCE,
) -> pl.DataFrame:
    lookback_start = start - timedelta(hours=LOOKBACK_WINDOW_HOURS)
    raw_frame = _build_asset_raw_frame(store, lookback_start, end, asset.upper())
    if raw_frame.is_empty():
        return empty_frame(DatasetName.COMPOSITE_SIGNAL_SERIES)

    normalized = _normalize_components(raw_frame)
    component_frame = _build_component_frame(
        normalized, asset=asset.upper(), scope=ASSET_SCOPE, source=source
    )
    signal_frame = _build_signal_frame(
        normalized, component_frame, asset=asset.upper(), scope=ASSET_SCOPE, source=source
    )
    calibrated = _apply_calibration(signal_frame, raw_frame)
    return calibrated.filter((pl.col("ts") >= pl.lit(start)) & (pl.col("ts") <= pl.lit(end)))


def build_market_composite_signal_series(
    store: DatasetStore,
    start: datetime,
    end: datetime,
    *,
    source: str = SOURCE,
) -> pl.DataFrame:
    assets = {
        asset: store.read(
            DatasetName.COMPOSITE_SIGNAL_SERIES,
            start=start,
            end=end,
            filters={"asset": asset, "scope": ASSET_SCOPE, "source": source},
        )
        for asset in MARKET_WEIGHTS
    }
    if any(frame.is_empty() for frame in assets.values()):
        return empty_frame(DatasetName.COMPOSITE_SIGNAL_SERIES)
    rows: list[dict[str, Any]] = []
    joined = _join_market_frames(assets)
    for item in joined.iter_rows(named=True):
        sentiment = _weighted_market_value(item, "sentiment_index")
        directional = _weighted_market_value(item, "directional_tilt")
        fragility = _weighted_market_value(item, "fragility_index")
        confidence = _weighted_market_value(item, "confidence_score")
        rows.append(
            {
                "ts": item["ts"],
                "date": item["ts"].date(),
                "scope": MARKET_SCOPE,
                "asset": MARKET_ASSET,
                "source": source,
                "sentiment_index": sentiment,
                "directional_tilt": directional,
                "fragility_index": fragility,
                "confidence_score": confidence,
                "options_sentiment_subindex": _weighted_market_value(
                    item, "options_sentiment_subindex"
                ),
                "perps_sentiment_subindex": _weighted_market_value(
                    item, "perps_sentiment_subindex"
                ),
                "micro_sentiment_subindex": _weighted_market_value(
                    item, "micro_sentiment_subindex"
                ),
                "micro_direction_subindex": _weighted_market_value(
                    item, "micro_direction_subindex"
                ),
                "perps_direction_subindex": _weighted_market_value(
                    item, "perps_direction_subindex"
                ),
                "options_direction_subindex": _weighted_market_value(
                    item, "options_direction_subindex"
                ),
                "perps_fragility_subindex": _weighted_market_value(
                    item, "perps_fragility_subindex"
                ),
                "options_fragility_subindex": _weighted_market_value(
                    item, "options_fragility_subindex"
                ),
                "micro_fragility_subindex": _weighted_market_value(
                    item, "micro_fragility_subindex"
                ),
                "coverage_ratio": _weighted_market_value(item, "coverage_ratio"),
                "freshness_score": _weighted_market_value(item, "freshness_score"),
                "calibrator_mode": "market_weighted",
                "sentiment_regime": _sentiment_regime(sentiment),
                "directional_regime": _directional_regime(directional),
                "fragility_regime": _fragility_regime(fragility),
                "quality_flag": _quality_flag(
                    _weighted_market_value(item, "coverage_ratio"),
                    _weighted_market_value(item, "confidence_score"),
                ),
            }
        )
    if not rows:
        return empty_frame(DatasetName.COMPOSITE_SIGNAL_SERIES)
    return _frame_from_rows(DatasetName.COMPOSITE_SIGNAL_SERIES, rows)


def build_composite_component_series(
    store: DatasetStore,
    start: datetime,
    end: datetime,
    *,
    asset: str,
    source: str = SOURCE,
) -> pl.DataFrame:
    lookback_start = start - timedelta(hours=LOOKBACK_WINDOW_HOURS)
    raw_frame = _build_asset_raw_frame(store, lookback_start, end, asset.upper())
    if raw_frame.is_empty():
        return empty_frame(DatasetName.COMPOSITE_COMPONENT_SERIES)
    normalized = _normalize_components(raw_frame)
    return _build_component_frame(
        normalized,
        asset=asset.upper(),
        scope=ASSET_SCOPE,
        source=source,
    ).filter((pl.col("ts") >= pl.lit(start)) & (pl.col("ts") <= pl.lit(end)))


def build_market_composite_component_series(
    store: DatasetStore,
    start: datetime,
    end: datetime,
    *,
    source: str = SOURCE,
) -> pl.DataFrame:
    frames = {
        asset: store.read(
            DatasetName.COMPOSITE_COMPONENT_SERIES,
            start=start,
            end=end,
            filters={"asset": asset, "scope": ASSET_SCOPE, "source": source},
        )
        for asset in MARKET_WEIGHTS
    }
    if any(frame.is_empty() for frame in frames.values()):
        return empty_frame(DatasetName.COMPOSITE_COMPONENT_SERIES)
    rows: list[dict[str, Any]] = []
    btc = frames["BTC"]
    eth = frames["ETH"]
    joined = btc.join(
        eth,
        on=["ts", "pillar", "family", "component"],
        how="inner",
        suffix="_eth",
    )
    for item in joined.iter_rows(named=True):
        rows.append(
            {
                "ts": item["ts"],
                "date": item["ts"].date(),
                "scope": MARKET_SCOPE,
                "asset": MARKET_ASSET,
                "source": source,
                "pillar": item["pillar"],
                "family": item["family"],
                "component": item["component"],
                "raw_value": _weighted_pair(item["raw_value"], item["raw_value_eth"]),
                "normalized_value": _weighted_pair(
                    item["normalized_value"], item["normalized_value_eth"]
                ),
                "component_weight": _weighted_pair(
                    item["component_weight"], item["component_weight_eth"]
                ),
                "pillar_score": _weighted_pair(item["pillar_score"], item["pillar_score_eth"]),
                "pillar_weight": _weighted_pair(item["pillar_weight"], item["pillar_weight_eth"]),
                "signed_contribution": _weighted_pair(
                    item["signed_contribution"], item["signed_contribution_eth"]
                ),
                "quality_flag": _merge_quality_flags(
                    str(item["quality_flag"]), str(item["quality_flag_eth"])
                ),
            }
        )
    return _frame_from_rows(DatasetName.COMPOSITE_COMPONENT_SERIES, rows)


def build_composite_transition_series(
    store: DatasetStore,
    start: datetime,
    end: datetime,
    *,
    asset: str,
    source: str = SOURCE,
) -> pl.DataFrame:
    frame = store.read(
        DatasetName.COMPOSITE_SIGNAL_SERIES,
        start=start - timedelta(hours=CALIBRATION_WINDOW_HOURS),
        end=end,
        filters={"asset": asset.upper(), "scope": ASSET_SCOPE, "source": source},
    )
    return _build_transition_frame(frame, scope=ASSET_SCOPE, asset=asset.upper(), source=source)


def build_market_composite_transition_series(
    store: DatasetStore,
    start: datetime,
    end: datetime,
    *,
    source: str = SOURCE,
) -> pl.DataFrame:
    frame = store.read(
        DatasetName.COMPOSITE_SIGNAL_SERIES,
        start=start - timedelta(hours=CALIBRATION_WINDOW_HOURS),
        end=end,
        filters={"asset": MARKET_ASSET, "scope": MARKET_SCOPE, "source": source},
    )
    return _build_transition_frame(frame, scope=MARKET_SCOPE, asset=MARKET_ASSET, source=source)


def _build_asset_raw_frame(
    store: DatasetStore,
    start: datetime,
    end: datetime,
    asset: str,
) -> pl.DataFrame:
    options = _load_asof_frame(
        store.read(
            DatasetName.OPTIONS_SIGNAL_SERIES,
            start=start,
            end=end,
            filters={"underlying": asset, "source": "options_composite"},
        ),
        "options",
        timedelta(hours=12),
        [
            "vol_carry_score",
            "skew_stress_score",
            "ml_carry_toxicity_score",
            "funding_skew_score",
            "sessionality_score",
            "dispersion_score",
            "expiry_pin_score",
            "gamma_flip_score",
            "signal_state",
            "quality_flag",
        ],
    )
    vvix = _load_asof_frame(
        store.read(
            DatasetName.VVIX_SERIES,
            start=start,
            end=end,
            filters={"underlying": asset, "source": "dvol"},
        ),
        "vvix",
        timedelta(hours=12),
        ["rvvix_14d", "qvvix_30d", "quality_flag"],
    )
    razor = _load_asof_frame(
        store.read(
            DatasetName.RAZOR_SERIES,
            start=start,
            end=end,
            filters={"underlying": asset, "source": "synthetic_7d_atm"},
        ),
        "razor",
        timedelta(hours=12),
        ["razor_pnl_zscore", "razor_vrp_zscore", "quality_flag"],
    )
    perps_signal = _load_asof_frame(
        store.read(
            DatasetName.PERPS_SIGNAL_SERIES,
            start=start,
            end=end,
            filters={"asset": asset, "venue_scope": "all", "source": "perps_family_v1"},
        ),
        "perps_signal",
        timedelta(hours=6),
        [
            "carry_momentum_1h",
            "carry_momentum_24h",
            "oi_price_divergence",
            "cum_delta_acceleration",
            "delta_divergence",
            "lead_lag_score",
            "depth_recovery_speed",
            "depth_imbalance_momentum",
            "spread_zscore_4h",
            "oi_concentration",
            "quality_flag",
        ],
    )
    perps_composite = _load_asof_frame(
        store.read(
            DatasetName.PERPS_COMPOSITE_SERIES,
            start=start,
            end=end,
            filters={"asset": asset, "venue_scope": "all", "source": "perps_composite_v1"},
        ),
        "perps_composite",
        timedelta(hours=6),
        [
            "carry_score",
            "cascade_risk_score",
            "toxicity_score",
            "positioning_score",
            "fragility_score",
            "quality_flag",
        ],
    )
    micro = _build_micro_hourly_frame(store, start, end, asset)
    price = _build_hourly_price_frame(store, start, end, asset)

    timelines = [
        frame.select(pl.col(frame.columns[0]).alias("ts"))
        for frame in (options, vvix, razor, perps_signal, perps_composite, micro, price)
        if not frame.is_empty()
    ]
    if not timelines:
        return pl.DataFrame()
    timeline = pl.concat(timelines, how="vertical").unique().sort("ts")
    frame = timeline
    frame = _join_source(frame, options, "options_source_ts")
    frame = _join_source(frame, vvix, "vvix_source_ts")
    frame = _join_source(frame, razor, "razor_source_ts")
    frame = _join_source(frame, perps_signal, "perps_signal_source_ts")
    frame = _join_source(frame, perps_composite, "perps_composite_source_ts")
    frame = _join_source(frame, micro, "micro_source_ts")
    frame = _join_source(frame, price, "price_source_ts")
    return frame.sort("ts")


def _build_micro_hourly_frame(
    store: DatasetStore,
    start: datetime,
    end: datetime,
    asset: str,
) -> pl.DataFrame:
    frame = store.read(
        DatasetName.MICROSTRUCTURE_MODEL_OUTPUT,
        start=start,
        end=end,
        filters={"asset": asset, "source": "tcn_boost_fusion"},
    )
    if frame.is_empty():
        return pl.DataFrame()
    horizon_weight_expr = (
        pl.when(pl.col("horizon") == "30s")
        .then(pl.lit(HORIZON_WEIGHTS["30s"]))
        .when(pl.col("horizon") == "1m")
        .then(pl.lit(HORIZON_WEIGHTS["1m"]))
        .when(pl.col("horizon") == "2m")
        .then(pl.lit(HORIZON_WEIGHTS["2m"]))
        .otherwise(pl.lit(HORIZON_WEIGHTS["5m"]))
    )
    weighted = frame.with_columns(
        [
            pl.col("ts").dt.truncate("1h").alias("micro_source_ts"),
            horizon_weight_expr.alias("horizon_weight"),
            ((pl.col("prob_up") - pl.col("prob_down")) * pl.col("confidence")).alias(
                "signed_score"
            ),
            pl.when(pl.col("blocked_reason").is_null())
            .then(0.0)
            .otherwise(1.0)
            .alias("blocked_flag"),
        ]
    )
    grouped = weighted.group_by("micro_source_ts").agg(
        [
            (
                (pl.col("signed_score") * pl.col("horizon_weight")).sum()
                / pl.col("horizon_weight").sum()
            ).alias("micro_direction_raw"),
            (pl.mean("signed_score") * pl.mean("tradability_score")).alias("micro_sentiment_raw"),
            pl.mean("confidence").alias("micro_confidence"),
            pl.mean("blocked_flag").alias("micro_blocked_rate"),
            pl.std("signed_score").alias("micro_horizon_dispersion"),
            pl.n_unique("horizon").alias("micro_active_horizons"),
            pl.max("quality_flag").alias("micro_quality_flag"),
        ]
    )
    return grouped.with_columns(
        [
            (
                (1.0 - pl.col("micro_confidence").fill_null(0.0))
                + pl.col("micro_blocked_rate").fill_null(0.0)
                + pl.col("micro_horizon_dispersion").fill_null(0.0).clip(0.0, 1.0)
            ).alias("micro_fragility_raw"),
            pl.lit(timedelta(hours=2)).alias("_tolerance"),
        ]
    )


def _build_hourly_price_frame(
    store: DatasetStore,
    start: datetime,
    end: datetime,
    asset: str,
) -> pl.DataFrame:
    bars = store.read(
        DatasetName.MICROSTRUCTURE_BAR,
        start=start,
        end=end + timedelta(hours=2),
        filters={"asset": asset, "bar_size_secs": 60},
    )
    if bars.is_empty():
        return pl.DataFrame()
    preferred = bars.filter(pl.col("market_type") == "spot")
    if preferred.is_empty():
        preferred = bars
    hourly = (
        preferred.with_columns(pl.col("ts").dt.truncate("1h").alias("price_source_ts"))
        .sort(["price_source_ts", "ts"])
        .group_by("price_source_ts")
        .agg(pl.col("close").last().alias("hourly_close"))
        .sort("price_source_ts")
        .with_columns(
            (((pl.col("hourly_close").shift(-1) / pl.col("hourly_close")) - 1.0) * 10_000.0).alias(
                "next_return_bps"
            )
        )
        .with_columns(pl.lit(timedelta(hours=2)).alias("_tolerance"))
    )
    return hourly


def _load_asof_frame(
    frame: pl.DataFrame,
    prefix: str,
    tolerance: timedelta,
    columns: list[str],
) -> pl.DataFrame:
    if frame.is_empty():
        return pl.DataFrame()
    selected = ["ts", *[column for column in columns if column in frame.columns]]
    source_ts_column = f"{prefix}_source_ts"
    renamed = frame.select(selected).rename({"ts": source_ts_column})
    rename_map = {
        column: f"{prefix}_{column}" for column in renamed.columns if column != source_ts_column
    }
    return renamed.rename(rename_map).with_columns(pl.lit(tolerance).alias("_tolerance"))


def _normalize_components(frame: pl.DataFrame) -> pl.DataFrame:
    result = frame.sort("ts").with_row_index("_history_points", offset=1)
    for spec in COMPONENT_SPECS:
        if spec.source_column not in result.columns:
            result = result.with_columns(pl.lit(None, dtype=pl.Float64).alias(spec.source_column))
        result = result.with_columns(
            _normalized_expr(spec.source_column, spec.sign).alias(f"norm__{spec.component}")
        )
    return result


def _normalized_expr(column: str, sign: float) -> pl.Expr:
    primary = (
        (
            pl.col(column)
            - pl.col(column).rolling_mean(
                NORMALIZATION_WINDOW_HOURS, min_samples=NORMALIZATION_MIN_SAMPLES
            )
        )
        / (
            pl.col(column).rolling_std(
                NORMALIZATION_WINDOW_HOURS, min_samples=NORMALIZATION_MIN_SAMPLES
            )
            + 1e-6
        )
    )
    fallback = (
        (
            pl.col(column)
            - pl.col(column).rolling_mean(
                NORMALIZATION_WINDOW_HOURS, min_samples=NORMALIZATION_FALLBACK_MIN_SAMPLES
            )
        )
        / (
            pl.col(column).rolling_std(
                NORMALIZATION_WINDOW_HOURS,
                min_samples=NORMALIZATION_FALLBACK_MIN_SAMPLES,
            )
            + 1e-6
        )
    )
    base = pl.when(primary.is_not_null()).then(primary).otherwise(fallback * 0.6)
    base = base.clip(-3.0, 3.0) / 3.0
    if sign < 0:
        base = -base
    return base


def _build_component_frame(
    frame: pl.DataFrame,
    *,
    asset: str,
    scope: str,
    source: str,
) -> pl.DataFrame:
    rows: list[dict[str, Any]] = []
    for item in frame.iter_rows(named=True):
        family_scores = _compute_family_scores(item)
        pillar_scores = _compute_pillar_scores(family_scores)
        for spec in COMPONENT_SPECS:
            normalized_value = item.get(f"norm__{spec.component}")
            if normalized_value is None:
                continue
            family_total = _family_total_weight(spec.pillar, spec.family)
            family_weight = spec.weight / family_total if family_total else 0.0
            pillar_weight = PILLAR_WEIGHTS[spec.pillar][spec.family]
            signed_contribution = float(normalized_value) * family_weight * pillar_weight
            rows.append(
                {
                    "ts": item["ts"],
                    "date": item["ts"].date(),
                    "scope": scope,
                    "asset": asset,
                    "source": source,
                    "pillar": spec.pillar,
                    "family": spec.family,
                    "component": spec.component,
                    "raw_value": item.get(spec.source_column),
                    "normalized_value": float(normalized_value),
                    "component_weight": family_weight,
                    "pillar_score": pillar_scores.get(spec.pillar),
                    "pillar_weight": pillar_weight,
                    "signed_contribution": signed_contribution,
                    "quality_flag": _component_quality(item, spec),
                }
            )
    if not rows:
        return empty_frame(DatasetName.COMPOSITE_COMPONENT_SERIES)
    return _frame_from_rows(DatasetName.COMPOSITE_COMPONENT_SERIES, rows)


def _build_signal_frame(
    frame: pl.DataFrame,
    component_frame: pl.DataFrame,
    *,
    asset: str,
    scope: str,
    source: str,
) -> pl.DataFrame:
    rows: list[dict[str, Any]] = []
    for item in frame.iter_rows(named=True):
        family_scores = _compute_family_scores(item)
        coverage_ratio = _coverage_ratio(item)
        freshness_score = _freshness_score(item)
        confidence = _confidence_score(item, coverage_ratio, freshness_score)
        sentiment_raw = _weighted_family_value(
            family_scores,
            "sentiment",
            {
                "options": "options_sentiment_subindex",
                "perps": "perps_sentiment_subindex",
                "micro": "micro_sentiment_subindex",
            },
        )
        directional_raw = _weighted_family_value(
            family_scores,
            "directional",
            {
                "micro": "micro_direction_subindex",
                "perps": "perps_direction_subindex",
                "options": "options_direction_subindex",
            },
        )
        fragility_raw = _weighted_family_value(
            family_scores,
            "fragility",
            {
                "perps": "perps_fragility_subindex",
                "options": "options_fragility_subindex",
                "micro": "micro_fragility_subindex",
            },
        )
        rows.append(
            {
                "ts": item["ts"],
                "date": item["ts"].date(),
                "scope": scope,
                "asset": asset,
                "source": source,
                "sentiment_index": _sentiment_fallback(sentiment_raw),
                "directional_tilt": _directional_fallback(directional_raw),
                "fragility_index": _fragility_fallback(fragility_raw),
                "confidence_score": confidence,
                "options_sentiment_subindex": family_scores.get("options_sentiment_subindex"),
                "perps_sentiment_subindex": family_scores.get("perps_sentiment_subindex"),
                "micro_sentiment_subindex": family_scores.get("micro_sentiment_subindex"),
                "micro_direction_subindex": family_scores.get("micro_direction_subindex"),
                "perps_direction_subindex": family_scores.get("perps_direction_subindex"),
                "options_direction_subindex": family_scores.get("options_direction_subindex"),
                "perps_fragility_subindex": family_scores.get("perps_fragility_subindex"),
                "options_fragility_subindex": family_scores.get("options_fragility_subindex"),
                "micro_fragility_subindex": family_scores.get("micro_fragility_subindex"),
                "coverage_ratio": coverage_ratio,
                "freshness_score": freshness_score,
                "calibrator_mode": "fixed_fallback",
                "sentiment_regime": _sentiment_regime(_sentiment_fallback(sentiment_raw)),
                "directional_regime": _directional_regime(_directional_fallback(directional_raw)),
                "fragility_regime": _fragility_regime(_fragility_fallback(fragility_raw)),
                "quality_flag": _quality_flag(coverage_ratio, confidence),
            }
        )
    if not rows:
        return empty_frame(DatasetName.COMPOSITE_SIGNAL_SERIES)
    return _frame_from_rows(DatasetName.COMPOSITE_SIGNAL_SERIES, rows)


def _compute_family_scores(item: dict[str, Any]) -> dict[str, float | None]:
    scores: dict[str, float | None] = {}
    for pillar in PILLAR_WEIGHTS:
        for family in PILLAR_WEIGHTS[pillar]:
            subset = [
                spec for spec in COMPONENT_SPECS if spec.pillar == pillar and spec.family == family
            ]
            total_weight = 0.0
            weighted_sum = 0.0
            for spec in subset:
                value = item.get(f"norm__{spec.component}")
                if value is None:
                    continue
                weighted_sum += float(value) * spec.weight
                total_weight += spec.weight
            key = (
                f"{family}_{pillar}_subindex"
                if pillar != "directional" or family != "micro"
                else "micro_direction_subindex"
            )
            if total_weight:
                scores[key] = weighted_sum / total_weight
            else:
                scores[key] = None
    # normalize names to the dataset schema we expose
    return {
        "options_sentiment_subindex": scores.get("options_sentiment_subindex"),
        "perps_sentiment_subindex": scores.get("perps_sentiment_subindex"),
        "micro_sentiment_subindex": scores.get("micro_sentiment_subindex"),
        "micro_direction_subindex": scores.get("micro_direction_subindex"),
        "perps_direction_subindex": scores.get("perps_direction_subindex"),
        "options_direction_subindex": scores.get("options_direction_subindex"),
        "perps_fragility_subindex": scores.get("perps_fragility_subindex"),
        "options_fragility_subindex": scores.get("options_fragility_subindex"),
        "micro_fragility_subindex": scores.get("micro_fragility_subindex"),
    }


def _compute_pillar_scores(family_scores: dict[str, float | None]) -> dict[str, float | None]:
    return {
        "sentiment": _weighted_mean(
            [
                (
                    family_scores.get("options_sentiment_subindex"),
                    PILLAR_WEIGHTS["sentiment"]["options"],
                ),
                (
                    family_scores.get("perps_sentiment_subindex"),
                    PILLAR_WEIGHTS["sentiment"]["perps"],
                ),
                (
                    family_scores.get("micro_sentiment_subindex"),
                    PILLAR_WEIGHTS["sentiment"]["micro"],
                ),
            ]
        ),
        "directional": _weighted_mean(
            [
                (
                    family_scores.get("micro_direction_subindex"),
                    PILLAR_WEIGHTS["directional"]["micro"],
                ),
                (
                    family_scores.get("perps_direction_subindex"),
                    PILLAR_WEIGHTS["directional"]["perps"],
                ),
                (
                    family_scores.get("options_direction_subindex"),
                    PILLAR_WEIGHTS["directional"]["options"],
                ),
            ]
        ),
        "fragility": _weighted_mean(
            [
                (
                    family_scores.get("perps_fragility_subindex"),
                    PILLAR_WEIGHTS["fragility"]["perps"],
                ),
                (
                    family_scores.get("options_fragility_subindex"),
                    PILLAR_WEIGHTS["fragility"]["options"],
                ),
                (
                    family_scores.get("micro_fragility_subindex"),
                    PILLAR_WEIGHTS["fragility"]["micro"],
                ),
            ]
        ),
    }


def _apply_calibration(signal_frame: pl.DataFrame, raw_frame: pl.DataFrame) -> pl.DataFrame:
    if signal_frame.is_empty():
        return signal_frame
    calibrated = signal_frame.sort("ts")
    calibrated = _apply_percentile_calibration(
        calibrated,
        "sentiment_index",
        "options_sentiment_subindex",
        "perps_sentiment_subindex",
        "micro_sentiment_subindex",
    )
    calibrated = _apply_percentile_calibration(
        calibrated,
        "fragility_index",
        "perps_fragility_subindex",
        "options_fragility_subindex",
        "micro_fragility_subindex",
    )
    calibrated = _apply_directional_calibration(calibrated, raw_frame)
    return calibrated.with_columns(
        [
            pl.col("sentiment_index")
            .map_elements(_sentiment_regime, return_dtype=pl.String)
            .alias("sentiment_regime"),
            pl.col("directional_tilt")
            .map_elements(_directional_regime, return_dtype=pl.String)
            .alias("directional_regime"),
            pl.col("fragility_index")
            .map_elements(_fragility_regime, return_dtype=pl.String)
            .alias("fragility_regime"),
            pl.struct(["coverage_ratio", "confidence_score"])
            .map_elements(
                lambda item: _quality_flag(item["coverage_ratio"], item["confidence_score"]),
                return_dtype=pl.String,
            )
            .alias("quality_flag"),
        ]
    )


def _apply_percentile_calibration(
    frame: pl.DataFrame,
    target_column: str,
    *support_columns: str,
) -> pl.DataFrame:
    values = frame.get_column(target_column).to_list()
    valid = [float(value) for value in values if value is not None]
    if len(valid) < CALIBRATION_MIN_SAMPLES:
        return frame
    ranked = (
        pl.DataFrame({"value": valid})
        .with_row_index("row_nr")
        .with_columns(
            (pl.col("value").rank(method="average").truediv(len(valid)).mul(100.0)).alias(
                "percentile"
            )
        )
        .sort("row_nr")
    )
    x = ranked.get_column("value").to_numpy()
    y = ranked.get_column("percentile").to_numpy()
    model = IsotonicRegression(y_min=0.0, y_max=100.0, out_of_bounds="clip")
    model.fit(x, y)
    calibrated_values: list[float | None] = []
    valid_iter = iter(model.predict(np.array(valid, dtype=float)).tolist())
    for value in values:
        calibrated_values.append(None if value is None else float(next(valid_iter)))
    mode = "isotonic_percentile"
    return frame.with_columns(
        [
            pl.Series(name=target_column, values=calibrated_values, dtype=pl.Float64),
            pl.when(pl.all_horizontal([pl.col(column).is_null() for column in support_columns]))
            .then(pl.col("calibrator_mode"))
            .otherwise(pl.lit(mode))
            .alias("calibrator_mode"),
        ]
    )


def _apply_directional_calibration(
    signal_frame: pl.DataFrame, raw_frame: pl.DataFrame
) -> pl.DataFrame:
    if "next_return_bps" not in raw_frame.columns:
        return signal_frame
    calibration_columns: list[pl.Expr] = [
        pl.col("ts"),
        pl.col("next_return_bps"),
        (
            pl.col("micro_confidence")
            if "micro_confidence" in raw_frame.columns
            else pl.lit(None, dtype=pl.Float64).alias("micro_confidence")
        ),
        (
            pl.col("micro_active_horizons")
            if "micro_active_horizons" in raw_frame.columns
            else pl.lit(None, dtype=pl.Float64).alias("micro_active_horizons")
        ),
    ]
    joined = signal_frame.join(
        raw_frame.select(calibration_columns),
        on="ts",
        how="left",
    )
    valid = joined.filter(
        pl.col("directional_tilt").is_not_null()
        & pl.col("next_return_bps").is_not_null()
        & pl.col("coverage_ratio").is_not_null()
    )
    if valid.height < CALIBRATION_MIN_SAMPLES:
        return signal_frame
    x = valid.select(
        [
            ((pl.col("directional_tilt") / 100.0).alias("raw_direction")),
            pl.col("coverage_ratio"),
            pl.col("micro_confidence").fill_null(0.0),
            (pl.col("micro_active_horizons").fill_null(0.0) / len(HORIZON_WEIGHTS)).alias(
                "horizon_coverage"
            ),
        ]
    ).to_numpy()
    y = np.clip(valid.get_column("next_return_bps").to_numpy(), -50.0, 50.0)
    model = Ridge(alpha=1.0)
    model.fit(x, y)
    full_x = joined.select(
        [
            ((pl.col("directional_tilt") / 100.0).fill_null(0.0).alias("raw_direction")),
            pl.col("coverage_ratio").fill_null(0.0),
            pl.col("micro_confidence").fill_null(0.0),
            (pl.col("micro_active_horizons").fill_null(0.0) / len(HORIZON_WEIGHTS)).alias(
                "horizon_coverage"
            ),
        ]
    ).to_numpy()
    predictions = model.predict(full_x)
    tilt = [float(np.clip(np.tanh(pred / 12.0) * 100.0, -100.0, 100.0)) for pred in predictions]
    return signal_frame.with_columns(
        [
            pl.Series(name="directional_tilt", values=tilt, dtype=pl.Float64),
            pl.lit("ridge_directional").alias("calibrator_mode"),
        ]
    )


def _build_transition_frame(
    frame: pl.DataFrame,
    *,
    scope: str,
    asset: str,
    source: str,
) -> pl.DataFrame:
    if frame.is_empty():
        return empty_frame(DatasetName.COMPOSITE_TRANSITION_SERIES)
    rows: list[dict[str, Any]] = []
    ordered = frame.sort("ts")
    for index_type, column in (
        ("sentiment", "sentiment_regime"),
        ("directional", "directional_regime"),
        ("fragility", "fragility_regime"),
    ):
        regimes = ordered.select(["ts", column]).drop_nulls()
        if regimes.is_empty():
            continue
        order = REGIME_ORDER[index_type]
        values = regimes.get_column(column).to_list()
        timestamps = regimes.get_column("ts").to_list()
        durations = _regime_durations(values, timestamps)
        for idx, (ts, current) in enumerate(zip(timestamps, values, strict=False)):
            prior = values[idx - 1] if idx else None
            window_values = values[max(0, idx - CALIBRATION_WINDOW_HOURS) : idx + 1]
            transitions = list(zip(window_values[:-1], window_values[1:], strict=False))
            outgoing = [nxt for prev, nxt in transitions if prev == current]
            stay_probability = _probability(outgoing, current)
            current_rank = order.index(current) if current in order else None
            upshift_probability = None
            downshift_probability = None
            if current_rank is not None and outgoing:
                upshift_probability = float(
                    sum(1 for nxt in outgoing if nxt in order and order.index(nxt) > current_rank)
                    / len(outgoing)
                )
                downshift_probability = float(
                    sum(1 for nxt in outgoing if nxt in order and order.index(nxt) < current_rank)
                    / len(outgoing)
                )
            rows.append(
                {
                    "ts": ts,
                    "date": ts.date(),
                    "scope": scope,
                    "asset": asset,
                    "source": source,
                    "index_type": index_type,
                    "current_regime": current,
                    "prior_regime": prior,
                    "stay_probability": stay_probability,
                    "upshift_probability": upshift_probability,
                    "downshift_probability": downshift_probability,
                    "median_regime_duration_hours": durations.get(current),
                    "quality_flag": "ok"
                    if len(window_values) >= NORMALIZATION_MIN_SAMPLES
                    else "partial_context",
                }
            )
    return _frame_from_rows(DatasetName.COMPOSITE_TRANSITION_SERIES, rows)


def _family_total_weight(pillar: str, family: str) -> float:
    return sum(
        spec.weight for spec in COMPONENT_SPECS if spec.pillar == pillar and spec.family == family
    )


def _coverage_ratio(item: dict[str, Any]) -> float:
    total = len(COMPONENT_SPECS)
    available = sum(
        1 for spec in COMPONENT_SPECS if item.get(f"norm__{spec.component}") is not None
    )
    return float(available / total) if total else 0.0


def _freshness_score(item: dict[str, Any]) -> float:
    ages = []
    ts = item["ts"]
    for prefix in ("options", "vvix", "razor", "perps_signal", "perps_composite", "micro", "price"):
        source_ts = item.get(f"{prefix}_source_ts")
        if source_ts is None:
            continue
        age_hours = max((ts - source_ts).total_seconds() / 3600.0, 0.0)
        ages.append(max(0.0, 1.0 - min(age_hours / 6.0, 1.0)))
    return float(np.mean(ages)) if ages else 0.0


def _confidence_score(item: dict[str, Any], coverage_ratio: float, freshness_score: float) -> float:
    quality_penalty = 0.0
    for column in (
        "options_quality_flag",
        "vvix_quality_flag",
        "razor_quality_flag",
        "perps_signal_quality_flag",
        "perps_composite_quality_flag",
        "micro_quality_flag",
    ):
        value = item.get(column)
        if value in {"partial_context", "gated"}:
            quality_penalty += 0.05
        elif value in {"insufficient_data", "blocked"}:
            quality_penalty += 0.15
    micro_confidence = float(item.get("micro_confidence") or 0.0)
    history_points = float(item.get("_history_points") or 0.0)
    history_score = min(history_points / NORMALIZATION_MIN_SAMPLES, 1.0)
    score = (
        0.30 * coverage_ratio
        + 0.20 * freshness_score
        + 0.25 * micro_confidence
        + 0.25 * history_score
        - quality_penalty
    )
    return float(np.clip(score, 0.0, 1.0))


def _weighted_family_value(
    family_scores: dict[str, float | None],
    pillar: str,
    mapping: dict[str, str],
) -> float | None:
    pairs = [
        (family_scores.get(key), PILLAR_WEIGHTS[pillar][family]) for family, key in mapping.items()
    ]
    return _weighted_mean(pairs)


def _weighted_mean(values: list[tuple[float | None, float]]) -> float | None:
    numerator = 0.0
    denominator = 0.0
    for value, weight in values:
        if value is None:
            continue
        numerator += float(value) * weight
        denominator += weight
    if denominator == 0.0:
        return None
    return numerator / denominator


def _sentiment_fallback(value: float | None) -> float | None:
    if value is None:
        return None
    return float(np.clip(50.0 + 25.0 * np.tanh(value * 1.5), 0.0, 100.0))


def _directional_fallback(value: float | None) -> float | None:
    if value is None:
        return None
    return float(np.clip(np.tanh(value * 1.5) * 100.0, -100.0, 100.0))


def _fragility_fallback(value: float | None) -> float | None:
    if value is None:
        return None
    return float(np.clip(50.0 + 25.0 * np.tanh(value * 1.5), 0.0, 100.0))


def _sentiment_regime(value: float | None) -> str | None:
    if value is None:
        return None
    for threshold, label in SENTIMENT_REGIMES:
        if value < threshold:
            return label
    return "euphoria"


def _directional_regime(value: float | None) -> str | None:
    if value is None:
        return None
    if value <= -40.0:
        return "bearish"
    if value <= -15.0:
        return "mild_bearish"
    if value < 15.0:
        return "neutral"
    if value < 40.0:
        return "mild_bullish"
    return "bullish"


def _fragility_regime(value: float | None) -> str | None:
    if value is None:
        return None
    for threshold, label in FRAGILITY_REGIMES:
        if value < threshold:
            return label
    return "squeeze_risk"


def _quality_flag(coverage_ratio: float | None, confidence: float | None) -> str:
    if coverage_ratio is None or coverage_ratio < 0.2:
        return "insufficient_data"
    if confidence is None or confidence < 0.4 or coverage_ratio < 0.75:
        return "partial_context"
    return "ok"


def _component_quality(item: dict[str, Any], spec: ComponentSpec) -> str:
    value = item.get(f"norm__{spec.component}")
    if value is None:
        return "insufficient_data"
    return "ok"


def _join_market_frames(frames: dict[str, pl.DataFrame]) -> pl.DataFrame:
    joined = frames["BTC"].rename(
        {column: f"{column}_btc" for column in frames["BTC"].columns if column != "ts"}
    )
    joined = joined.join(
        frames["ETH"].rename(
            {column: f"{column}_eth" for column in frames["ETH"].columns if column != "ts"}
        ),
        on="ts",
        how="inner",
    )
    return joined


def _weighted_market_value(item: dict[str, Any], field: str) -> float | None:
    return _weighted_pair(item.get(f"{field}_btc"), item.get(f"{field}_eth"))


def _weighted_pair(first: Any, second: Any) -> float | None:
    pairs = []
    if first is not None:
        pairs.append((float(first), MARKET_WEIGHTS["BTC"]))
    if second is not None:
        pairs.append((float(second), MARKET_WEIGHTS["ETH"]))
    if not pairs:
        return None
    return float(
        sum(value * weight for value, weight in pairs) / sum(weight for _, weight in pairs)
    )


def _merge_quality_flags(first: str | None, second: str | None) -> str:
    values = {first, second}
    if "insufficient_data" in values:
        return "insufficient_data"
    if "partial_context" in values:
        return "partial_context"
    return "ok"


def _probability(values: list[str], target: str) -> float | None:
    if not values:
        return None
    return float(sum(1 for value in values if value == target) / len(values))


def _regime_durations(regimes: list[str], timestamps: list[datetime]) -> dict[str, float]:
    durations: dict[str, list[float]] = {}
    if not regimes:
        return {}
    current = regimes[0]
    start = timestamps[0]
    for regime, ts in zip(regimes[1:], timestamps[1:], strict=False):
        if regime != current:
            durations.setdefault(current, []).append((ts - start).total_seconds() / 3600.0)
            current = regime
            start = ts
    durations.setdefault(current, []).append((timestamps[-1] - start).total_seconds() / 3600.0)
    return {regime: float(np.median(values)) for regime, values in durations.items()}


def _join_source(base: pl.DataFrame, source_frame: pl.DataFrame, ts_column: str) -> pl.DataFrame:
    if source_frame.is_empty():
        return base
    tolerance = source_frame.item(0, "_tolerance")
    return base.join_asof(
        source_frame.sort(ts_column),
        left_on="ts",
        right_on=ts_column,
        strategy="backward",
        tolerance=tolerance,
    ).drop("_tolerance")
