#!/usr/bin/env python3
# FILE-SIZE-OK: Seeder daemon (Issue #270, #278) — Vision download + TickStorage write
"""Proactive Parquet tick cache seeder (Issue #270).

Downloads Binance Vision daily aggTrade ZIPs for all registry-enabled symbols
and writes raw trades to the local Parquet tick cache (TickStorage). This
ensures kintsugi's 64-worker Parquet path (_shard_is_parquet_cached) is
always active — zero Vision downloads needed during kintsugi repairs.

Issue #277: Simplified to write one daily Parquet file per Vision ZIP.
No more monthly merge/append logic (MEM-016/MEM-017/MEM-018 no longer needed).

Issue #278: Added UM futures market support. Configure via
OPENDEVIATIONBAR_SEEDER_MARKETS="spot,um_futures". Futures data is cached
under separate dirs (e.g., BINANCE_UM_BTCUSDT/) with identical Parquet schema.

Key design:
  - Writes with plain symbol key ("BTCUSDT") so kintsugi can find the data.
    (checkpoint.py uses "BINANCE_SPOT_BTCUSDT" — different key, different path.)
  - Does NOT construct bars; pure tick download → Parquet write.
  - Runs idempotently: skips days already cached (daily file existence check).
  - Newest-first ordering matches kintsugi shard priority.
  - Exits 0 and writes state file on success; monit checks state file freshness.

Usage:
  # One-off run (runs all symbols, seeds missing days, exits):
  python scripts/tick_cache_seeder.py

  # Daemon mode via systemd timer (every 5 min):
  systemctl --user start opendeviationbar-seeder.timer
"""

from __future__ import annotations

import concurrent.futures
import io
import logging
import os
import sys
import time
import zipfile
from datetime import UTC, date, datetime, timedelta
from pathlib import Path

import polars as pl

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Vision constants
# ---------------------------------------------------------------------------

_VISION_SPOT_BASE = "https://data.binance.vision/data/spot/daily/aggTrades"
_VISION_UM_BASE = "https://data.binance.vision/data/futures/um/daily/aggTrades"

# Binance Vision spot aggTrades CSV — no header, 7 named columns
# (8th column `was_best_price` silently dropped by Polars new_columns truncation)
_SPOT_COLUMNS = [
    "agg_trade_id",   # int64
    "price",          # float64
    "quantity",       # float64 (Vision calls it "qty"; we rename to "quantity"
                      #          so _select_trade_columns() finds it)
    "first_trade_id", # int64
    "last_trade_id",  # int64
    "timestamp",      # int64 ms epoch (Vision calls it "transact_time")
    "is_buyer_maker", # bool
]

# Binance Vision UM futures aggTrades CSV — 7 columns exactly (no was_best_price).
# Has header row from 2022-08-11 onward. Timestamps always ms (no µs switch).
# Booleans are lowercase (true/false) vs spot's Title case (True/False).
_UM_COLUMNS = [
    "agg_trade_id",   # int64
    "price",          # float64
    "quantity",       # float64
    "first_trade_id", # int64
    "last_trade_id",  # int64
    "timestamp",      # int64 ms epoch (always ms, never µs)
    "is_buyer_maker", # bool
]

# Date from which UM futures CSVs include a header row
_UM_HEADER_CUTOFF = "2022-08-11"

# Backward compat alias used internally
_VISION_BASE = _VISION_SPOT_BASE
_VISION_COLUMNS = _SPOT_COLUMNS

# Valid market identifiers
VALID_MARKETS = ("spot", "um_futures")


# ---------------------------------------------------------------------------
# Environment loading
# ---------------------------------------------------------------------------

def _load_env_file(env_path: Path) -> None:
    """Load KEY=VALUE lines from env file into os.environ (setdefault)."""
    if not env_path.exists():
        return
    with env_path.open() as f:
        for raw in f:
            stripped = raw.strip()
            if stripped and not stripped.startswith("#") and "=" in stripped:
                k, _, v = stripped.partition("=")
                os.environ.setdefault(k.strip(), v.strip())


# ---------------------------------------------------------------------------
# Vision download helpers
# ---------------------------------------------------------------------------

def _vision_url(symbol: str, date_str: str, *, market: str = "spot") -> str:
    """Build Vision ZIP URL for the given market."""
    if market == "um_futures":
        return f"{_VISION_UM_BASE}/{symbol}/{symbol}-aggTrades-{date_str}.zip"
    return f"{_VISION_SPOT_BASE}/{symbol}/{symbol}-aggTrades-{date_str}.zip"


def _download_vision_day(
    symbol: str,
    date_str: str,
    *,
    timeout_s: int,
    market: str = "spot",
) -> pl.DataFrame | None:
    """Download and parse one Vision ZIP. Returns None if unavailable or error.

    Parameters
    ----------
    symbol : str
        Trading symbol (e.g., "BTCUSDT").
    date_str : str
        Date string "YYYY-MM-DD".
    timeout_s : int
        HTTP timeout in seconds.
    market : str
        Market type: "spot" (default) or "um_futures".
    """
    import requests

    url = _vision_url(symbol, date_str, market=market)
    try:
        resp = requests.get(url, timeout=timeout_s, allow_redirects=True)
    except requests.Timeout:
        logger.warning("%s %s [%s]: Vision download timed out", symbol, date_str, market)
        return None
    except Exception as exc:
        logger.warning("%s %s [%s]: Vision download error: %s", symbol, date_str, market, exc)
        return None

    if resp.status_code == 404:
        logger.debug("%s %s [%s]: Vision 404 — not yet published", symbol, date_str, market)
        return None
    if resp.status_code != 200:
        logger.warning("%s %s [%s]: Vision HTTP %d", symbol, date_str, market, resp.status_code)
        return None

    # Select columns and header handling based on market
    if market == "um_futures":
        columns = _UM_COLUMNS
        # UM futures CSVs have a header row from 2022-08-11 onward
        has_header = date_str >= _UM_HEADER_CUTOFF
    else:
        columns = _SPOT_COLUMNS
        has_header = False

    schema_overrides = {
        "agg_trade_id": pl.Int64,
        "price": pl.Float64,
        "quantity": pl.Float64,
        "first_trade_id": pl.Int64,
        "last_trade_id": pl.Int64,
        "timestamp": pl.Int64,
        "is_buyer_maker": pl.Boolean,
    }

    try:
        with zipfile.ZipFile(io.BytesIO(resp.content)) as zf:
            csv_name = f"{symbol}-aggTrades-{date_str}.csv"
            with zf.open(csv_name) as csv_f:
                if has_header:
                    # UM futures with header: read with header, then rename
                    df = pl.read_csv(
                        csv_f,
                        has_header=True,
                        schema_overrides={
                            # Use the Vision header names for schema overrides
                            "agg_trade_id": pl.Int64,
                            "price": pl.Float64,
                            "quantity": pl.Float64,
                            "first_trade_id": pl.Int64,
                            "last_trade_id": pl.Int64,
                            "transact_time": pl.Int64,
                            "is_buyer_maker": pl.Boolean,
                        },
                    )
                    # Rename transact_time → timestamp to match our schema
                    if "transact_time" in df.columns:
                        df = df.rename({"transact_time": "timestamp"})
                    # Select only the columns we need (in case of extra columns)
                    df = df.select(columns)
                else:
                    # Spot or early UM futures: no header, positional columns
                    df = pl.read_csv(
                        csv_f,
                        has_header=False,
                        new_columns=columns,
                        schema_overrides=schema_overrides,
                    )
    except (zipfile.BadZipFile, KeyError) as exc:
        logger.warning("%s %s [%s]: ZIP parse error: %s", symbol, date_str, market, exc)
        return None
    except Exception as exc:
        logger.warning("%s %s [%s]: CSV parse error: %s", symbol, date_str, market, exc)
        return None

    # Normalize µs → ms for spot data (spot switched to µs on 2025-01-01).
    # UM futures timestamps are ALWAYS ms — skip normalization for futures.
    if market != "um_futures" and not df.is_empty():
        max_ts = df["timestamp"].max()
        if max_ts is not None and max_ts > 1_500_000_000_000_000:
            df = df.with_columns((pl.col("timestamp") // 1000).alias("timestamp"))

    if df.is_empty():
        logger.warning("%s %s [%s]: Vision CSV was empty", symbol, date_str, market)
        return None

    # Sort by agg_trade_id for deterministic ordering within each daily file
    df = df.sort("agg_trade_id")

    return df


# ---------------------------------------------------------------------------
# Day gap discovery
# ---------------------------------------------------------------------------

def _find_missing_days(
    symbol: str,
    start_date: date,
    end_date: date,
    storage: TickStorage,  # noqa: F821
) -> list[str]:
    """Return date strings needing seeding, newest-first.

    Issue #277: With daily Parquet files, checking is simple:
    for each day in [start_date, end_date], check if the daily file exists.
    """
    missing: list[str] = []

    current = start_date
    while current <= end_date:
        date_str = current.isoformat()
        day_path = storage._get_parquet_path(symbol, date_str)
        if not day_path.exists():
            missing.append(date_str)
        current += timedelta(days=1)

    # Newest-first (matches kintsugi's priority ordering)
    missing.sort(reverse=True)
    return missing


# ---------------------------------------------------------------------------
# Per-day download worker
# ---------------------------------------------------------------------------

def _download_day(
    symbol: str,
    date_str: str,
    timeout_s: int,
    market: str = "spot",
) -> tuple[str, pl.DataFrame | None]:
    """Download one Vision day. Returns (date_str, df_or_None)."""
    return date_str, _download_vision_day(symbol, date_str, timeout_s=timeout_s, market=market)


# ---------------------------------------------------------------------------
# Per-symbol seeder
# ---------------------------------------------------------------------------

def _cache_key(symbol: str, market: str) -> str:
    """Build the TickStorage cache key for a (symbol, market) pair.

    Spot uses the plain symbol ("BTCUSDT") for backward compatibility.
    Futures use a prefixed key ("BINANCE_UM_BTCUSDT") to avoid collision.
    """
    if market == "um_futures":
        return f"BINANCE_UM_{symbol}"
    return symbol


def _seed_symbol(
    symbol: str,
    effective_start: date,
    storage: TickStorage,  # noqa: F821
    max_workers: int,
    vision_timeout_s: int,
    state_path: Path | None = None,
    run_start_iso: str | None = None,
    market: str = "spot",
) -> int:
    """Seed all missing days for one symbol. Returns count of newly seeded days.

    Issue #277: Each Vision daily ZIP → one daily Parquet file.
    No more monthly merge/append logic. Each day is one atomic write.

    Issue #278: market parameter selects spot vs um_futures Vision source.
    """
    from opendeviationbar.storage.parquet_io import _atomic_write_parquet

    today = datetime.now(UTC).date()
    yesterday = today - timedelta(days=1)  # Vision T-1 cutoff

    cache_sym = _cache_key(symbol, market)

    if effective_start > yesterday:
        logger.debug("%s [%s]: effective_start %s is in the future — skip", symbol, market, effective_start)
        return 0

    missing = _find_missing_days(cache_sym, effective_start, yesterday, storage)
    if not missing:
        logger.debug("%s [%s]: fully cached up to %s", symbol, market, yesterday)
        return 0

    logger.info("%s [%s]: %d days to seed (newest: %s)", symbol, market, len(missing), missing[0])

    # Ensure symbol directory exists
    symbol_dir = storage._get_symbol_dir(cache_sym)
    symbol_dir.mkdir(parents=True, exist_ok=True)

    newly_seeded = 0
    completed_days = 0

    # MEM-018: Bounded future submission — only max_workers futures in flight
    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as ex:
        pending: dict[concurrent.futures.Future[tuple[str, object]], str] = {}
        days_iter = iter(missing)

        # Seed the pool up to max_workers in-flight futures.
        for _ in range(max_workers):
            d = next(days_iter, None)
            if d is None:
                break
            pending[ex.submit(_download_day, symbol, d, vision_timeout_s, market)] = d

        while pending:
            done, _ = concurrent.futures.wait(
                pending,
                return_when=concurrent.futures.FIRST_COMPLETED,
            )
            for fut in done:
                pending.pop(fut)  # Drop strong reference
                try:
                    date_str, df = fut.result()
                    if df is not None:
                        # Write daily Parquet file atomically
                        day_path = storage._get_parquet_path(cache_sym, date_str)
                        _atomic_write_parquet(df, day_path)
                        newly_seeded += 1
                        completed_days += 1
                        del df  # Only reference gone → immediate GC

                        if completed_days % 50 == 0:
                            logger.info(
                                "%s [%s]: seeded %d/%d days so far",
                                symbol, market, completed_days, len(missing),
                            )
                except Exception as exc:
                    logger.warning("%s [%s]: download/write error: %s", symbol, market, exc)

                # Refill pool: submit next day to keep workers busy.
                d = next(days_iter, None)
                if d is not None:
                    pending[
                        ex.submit(_download_day, symbol, d, vision_timeout_s, market)
                    ] = d

    if newly_seeded > 0:
        logger.info("Seeded %s [%s]: %d day(s)", symbol, market, newly_seeded)

    # Write progress state if applicable
    if state_path is not None and newly_seeded > 0:
        try:
            state_path.write_text(
                f"running\n"
                f"timestamp={run_start_iso or datetime.now(UTC).isoformat()}\n"
                f"current_symbol={symbol}\n"
                f"market={market}\n"
                f"days_done={newly_seeded}\n"
            )
        except OSError:
            logger.warning("Failed to write progress state file %s", state_path, exc_info=True)

    return newly_seeded


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def _parse_markets(raw: str) -> list[str]:
    """Parse comma-separated markets string into validated list."""
    markets = [m.strip() for m in raw.split(",") if m.strip()]
    for m in markets:
        if m not in VALID_MARKETS:
            msg = f"Invalid market '{m}'. Valid: {', '.join(VALID_MARKETS)}"
            raise ValueError(msg)
    return markets if markets else ["spot"]


def main() -> int:
    """Run seeder over all enabled symbols. Returns exit code."""
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s %(levelname)s %(name)s %(message)s",
        stream=sys.stdout,
    )

    # Load deploy env (non-secrets: OPENDEVIATIONBAR_SEEDER_* vars live here)
    deploy_env = Path.home() / "opendeviationbar-py/deploy/opendeviationbar.env"
    _load_env_file(deploy_env)

    from opendeviationbar.config.seeder import SeederConfig
    from opendeviationbar.storage.parquet import TickStorage
    from opendeviationbar.symbol_registry import (
        get_effective_start_date,
        get_registered_symbols,
    )

    cfg = SeederConfig()
    storage = TickStorage()
    symbols = get_registered_symbols(enabled_only=True)
    markets = _parse_markets(cfg.markets)

    t0 = time.monotonic()
    total_seeded = 0
    symbols_processed = 0
    symbol_counts: dict[str, int] = {}
    symbols_failed: list[str] = []

    logger.info(
        "Seeder start: %d symbols, markets=%s, historical_start=%s, workers=%d",
        len(symbols),
        markets,
        cfg.historical_start,
        cfg.max_concurrent_downloads,
    )

    # Write initial state file immediately so monit doesn't see "Does not exist"
    # during long-running seeder passes (thousands of days can take hours).
    run_start_iso = datetime.now(UTC).isoformat()
    state_path = Path(cfg.state_file)
    try:
        state_path.write_text(
            f"running\ntimestamp={run_start_iso}\nsymbols={len(symbols)}\nmarkets={','.join(markets)}\n"
        )
    except OSError:
        logger.warning("Failed to write initial state file %s", state_path, exc_info=True)

    for market in markets:
        for symbol in symbols:
            # Use symbol's effective_start (or listing_date) when later than config default
            registry_start = get_effective_start_date(symbol)
            config_start = date.fromisoformat(cfg.historical_start)
            if registry_start:
                symbol_start = max(config_start, date.fromisoformat(registry_start))
            else:
                symbol_start = config_start

            task_key = f"{symbol}@{market}"
            try:
                count = _seed_symbol(
                    symbol,
                    symbol_start,
                    storage,
                    cfg.max_concurrent_downloads,
                    cfg.vision_timeout_s,
                    state_path=state_path,
                    run_start_iso=run_start_iso,
                    market=market,
                )
                total_seeded += count
                symbols_processed += 1
                symbol_counts[task_key] = count
            except Exception:
                logger.exception("Seeder failed for %s", task_key)
                symbols_failed.append(task_key)

    elapsed = time.monotonic() - t0
    logger.info(
        "Seeder complete: %d days seeded across %d tasks (%d symbols x %d markets) in %.1fs",
        total_seeded,
        symbols_processed,
        len(symbols),
        len(markets),
        elapsed,
    )

    # Overwrite state file with final completion status.
    # Per-symbol counts enable heartbeat and monit to detect ETHUSDT-specific failures.
    per_symbol = "".join(f"{s}_seeded={c}\n" for s, c in symbol_counts.items())
    try:
        state_path.write_text(
            f"ok\n"
            f"timestamp={datetime.now(UTC).isoformat()}\n"
            f"symbols={symbols_processed}\n"
            f"markets={','.join(markets)}\n"
            f"seeded={total_seeded}\n"
            f"elapsed_s={elapsed:.1f}\n"
            f"failed={len(symbols_failed)}\n"
            + per_symbol
        )
    except OSError:
        logger.warning("Failed to write final state file %s", state_path, exc_info=True)

    if symbols_failed:
        logger.error("Seeder failed for %d task(s): %s", len(symbols_failed), symbols_failed)
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main())
