#!/usr/bin/env python3
# FILE-SIZE-OK: Seeder daemon (Issue #270) — Vision download + TickStorage write
"""Proactive Parquet tick cache seeder (Issue #270).

Downloads Binance Vision daily aggTrade ZIPs for all registry-enabled symbols
and writes raw trades to the local Parquet tick cache (TickStorage). This
ensures kintsugi's 64-worker Parquet path (_shard_is_parquet_cached) is
always active — zero Vision downloads needed during kintsugi repairs.

Key design:
  - Writes with plain symbol key ("BTCUSDT") so kintsugi can find the data.
    (checkpoint.py uses "BINANCE_SPOT_BTCUSDT" — different key, different path.)
  - Does NOT construct bars; pure tick download → Parquet write.
  - Runs idempotently: skips months already cached (list_months file-stat check),
    re-checks current+previous months per-day for partial coverage.
  - Newest-first ordering matches kintsugi shard priority.
  - Exits 0 and writes state file on success; monit checks state file freshness.

Usage:
  # One-off run (runs all symbols, seeds missing days, exits):
  python scripts/tick_cache_seeder.py

  # Daemon mode via systemd timer (every 5 min):
  systemctl --user start opendeviationbar-seeder.timer
"""

from __future__ import annotations

import concurrent.futures
import contextlib
import ctypes
import io
import logging
import os
import sys
import time
import zipfile
from calendar import monthrange
from collections.abc import Iterator
from datetime import UTC, date, datetime, timedelta
from pathlib import Path

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Vision constants
# ---------------------------------------------------------------------------

_VISION_BASE = "https://data.binance.vision/data/spot/daily/aggTrades"

# Binance Vision aggTrades CSV — no header, 7 columns in this order:
_VISION_COLUMNS = [
    "agg_trade_id",   # int64
    "price",          # float64
    "quantity",       # float64 (Vision calls it "qty"; we rename to "quantity"
                      #          so _select_trade_columns() finds it)
    "first_trade_id", # int64
    "last_trade_id",  # int64
    "timestamp",      # int64 ms epoch (Vision calls it "transact_time")
    "is_buyer_maker", # bool
]


# ---------------------------------------------------------------------------
# Environment loading
# ---------------------------------------------------------------------------

def _load_env_file(env_path: Path) -> None:
    """Load KEY=VALUE lines from env file into os.environ (setdefault)."""
    if not env_path.exists():
        return
    with env_path.open() as f:
        for raw in f:
            stripped = raw.strip()
            if stripped and not stripped.startswith("#") and "=" in stripped:
                k, _, v = stripped.partition("=")
                os.environ.setdefault(k.strip(), v.strip())


# ---------------------------------------------------------------------------
# Vision download helpers
# ---------------------------------------------------------------------------

def _vision_url(symbol: str, date_str: str) -> str:
    return f"{_VISION_BASE}/{symbol}/{symbol}-aggTrades-{date_str}.zip"


def _download_vision_day(
    symbol: str,
    date_str: str,
    *,
    timeout_s: int,
) -> pl.DataFrame | None:  # noqa: F821
    """Download and parse one Vision ZIP. Returns None if unavailable or error."""
    import polars as pl
    import requests

    url = _vision_url(symbol, date_str)
    try:
        resp = requests.get(url, timeout=timeout_s, allow_redirects=True)
    except requests.Timeout:
        logger.warning("%s %s: Vision download timed out", symbol, date_str)
        return None
    except Exception as exc:
        logger.warning("%s %s: Vision download error: %s", symbol, date_str, exc)
        return None

    if resp.status_code == 404:
        logger.debug("%s %s: Vision 404 — not yet published", symbol, date_str)
        return None
    if resp.status_code != 200:
        logger.warning("%s %s: Vision HTTP %d", symbol, date_str, resp.status_code)
        return None

    try:
        with zipfile.ZipFile(io.BytesIO(resp.content)) as zf:
            csv_name = f"{symbol}-aggTrades-{date_str}.csv"
            with zf.open(csv_name) as csv_f:
                df = pl.read_csv(
                    csv_f,
                    has_header=False,
                    new_columns=_VISION_COLUMNS,
                    schema_overrides={
                        "agg_trade_id": pl.Int64,
                        "price": pl.Float64,
                        "quantity": pl.Float64,
                        "first_trade_id": pl.Int64,
                        "last_trade_id": pl.Int64,
                        "timestamp": pl.Int64,
                        "is_buyer_maker": pl.Boolean,
                    },
                )
    except (zipfile.BadZipFile, KeyError) as exc:
        logger.warning("%s %s: ZIP parse error: %s", symbol, date_str, exc)
        return None
    except Exception as exc:
        logger.warning("%s %s: CSV parse error: %s", symbol, date_str, exc)
        return None

    if df.is_empty():
        logger.warning("%s %s: Vision CSV was empty", symbol, date_str)
        return None

    return df


# ---------------------------------------------------------------------------
# Month/day gap discovery
# ---------------------------------------------------------------------------

def _month_str(year: int, month: int) -> str:
    return f"{year:04d}-{month:02d}"


def _iter_months(start: date, end: date) -> Iterator[tuple[int, int]]:
    """Yield (year, month) tuples from start's month through end's month."""
    cur = date(start.year, start.month, 1)
    stop = date(end.year, end.month, 1)
    while cur <= stop:
        yield cur.year, cur.month
        # Advance to next month
        if cur.month == 12:
            cur = date(cur.year + 1, 1, 1)
        else:
            cur = date(cur.year, cur.month + 1, 1)


def _days_in_month(year: int, month: int) -> list[str]:
    """Return all YYYY-MM-DD strings in the given month."""
    _, n = monthrange(year, month)
    return [f"{year:04d}-{month:02d}-{day:02d}" for day in range(1, n + 1)]


def _day_ms_range(date_str: str) -> tuple[int, int]:
    """Return (start_ms, end_ms) for a UTC day (exclusive end)."""
    day_dt = datetime.strptime(date_str, "%Y-%m-%d").replace(tzinfo=UTC)
    start_ms = int(day_dt.timestamp() * 1000)
    end_ms = int((day_dt + timedelta(days=1)).timestamp() * 1000) - 1
    return start_ms, end_ms


def _find_missing_days(
    symbol: str,
    start_date: date,
    end_date: date,
    storage: TickStorage,  # noqa: F821
) -> list[str]:
    """Return date strings needing seeding, newest-first.

    Algorithm:
      1. list_months() → set of cached month strings (fast file-stat glob).
      2. For months fully before last 2 months: if month file exists, skip all
         days (assume complete); if missing, queue all days.
      3. For last 2 months (may be partial): per-day has_ticks() check.
    """
    cached_months = set(storage.list_months(symbol))
    today = datetime.now(UTC).date()
    # Treat current and previous month as potentially partial
    partial_threshold = date(today.year, today.month, 1)
    if today.month > 1:
        partial_threshold = date(today.year, today.month - 1, 1)
    else:
        partial_threshold = date(today.year - 1, 12, 1)

    missing: list[str] = []

    for year, month in _iter_months(start_date, end_date):
        month_key = _month_str(year, month)
        days = _days_in_month(year, month)

        # Clamp days to [start_date, end_date]
        days = [
            d for d in days
            if start_date <= date.fromisoformat(d) <= end_date
        ]
        if not days:
            continue

        is_partial_month = date(year, month, 1) >= partial_threshold

        if not is_partial_month:
            # Historical month: file exists → fully seeded; no file → seed all
            if month_key not in cached_months:
                missing.extend(days)
        else:
            # Recent month: per-day check for partial coverage
            for date_str in days:
                start_ms, end_ms = _day_ms_range(date_str)
                if not storage.has_ticks(symbol, start_ms, end_ms):
                    missing.append(date_str)

    # Newest-first (matches kintsugi's priority ordering)
    missing.sort(reverse=True)
    return missing


# ---------------------------------------------------------------------------
# Per-day download worker
# ---------------------------------------------------------------------------

def _download_day(
    symbol: str,
    date_str: str,
    timeout_s: int,
) -> tuple[str, pl.DataFrame | None]:  # noqa: F821
    """Download one Vision day. Returns (date_str, df_or_None)."""
    return date_str, _download_vision_day(symbol, date_str, timeout_s=timeout_s)


# ---------------------------------------------------------------------------
# Per-symbol seeder
# ---------------------------------------------------------------------------

def _seed_symbol(
    symbol: str,
    effective_start: date,
    storage: TickStorage,  # noqa: F821
    max_workers: int,
    vision_timeout_s: int,
    state_path: Path | None = None,
) -> int:
    """Seed all missing days for one symbol. Returns count of newly seeded days.

    MEM-016v2 + MEM-017 + MEM-018: Per-day staging + sequential appends +
    bounded future submission.

    Phase 1 (staging): each downloaded day is written to a temp Parquet file
    and the in-memory DataFrame is deleted immediately.  MEM-018 ensures
    concurrent.futures.Future objects are freed as soon as their result is
    processed (bounded sliding-window submission), so only max_workers
    DataFrames are alive at any time — not the entire month's worth.

    Phase 2 (MEM-017): staged files are appended to the month file one at a
    time via sequential two-source streaming merges.  Peak memory per append
    = buffers for two files (existing_month + one_day), never more.  This
    avoids MEM-016v2's 30-source concat which caused ~200 MB/month Polars/
    Python arena accumulation (4.5 GB → 7.3 GB over 14 months in production).
    """
    import polars as pl
    from opendeviationbar.storage.parquet_io import (
        _atomic_sink_parquet,
        _atomic_write_parquet,
    )

    today = datetime.now(UTC).date()
    yesterday = today - timedelta(days=1)  # Vision T-1 cutoff

    if effective_start > yesterday:
        logger.debug("%s: effective_start %s is in the future — skip", symbol, effective_start)
        return 0

    missing = _find_missing_days(symbol, effective_start, yesterday, storage)
    if not missing:
        logger.debug("%s: fully cached up to %s", symbol, yesterday)
        return 0

    logger.info("%s: %d days to seed (newest: %s)", symbol, len(missing), missing[0])

    # Group missing days by month (YYYY-MM), newest month first.
    month_groups: dict[str, list[str]] = {}
    for d in missing:
        month_groups.setdefault(d[:7], []).append(d)

    newly_seeded = 0
    completed_months = 0

    for month, days in sorted(month_groups.items(), reverse=True):
        logger.info("%s %s: downloading %d missing day(s)", symbol, month, len(days))

        symbol_dir = storage._get_symbol_dir(symbol)
        symbol_dir.mkdir(parents=True, exist_ok=True)
        month_path = storage._get_parquet_path(symbol, month)

        staged: list[Path] = []
        staged_rows = 0

        try:
            # MEM-018: Bounded future submission — only max_workers futures in flight
            # at any time.  concurrent.futures.Future caches its result in _result
            # even after result() is called, so a naive [submit() for d in days]
            # accumulates ALL day DataFrames in memory (31 x ~60 MB = ~1.9 GB for a
            # typical month) until the 'with' block exits.  Using a dict + pop()
            # keeps each future alive only until its result is processed, allowing
            # the GC to reclaim the DataFrame immediately after del df.
            with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as ex:
                pending: dict[concurrent.futures.Future[tuple[str, object]], str] = {}
                days_iter = iter(days)

                # Seed the pool up to max_workers in-flight futures.
                for _ in range(max_workers):
                    d = next(days_iter, None)
                    if d is None:
                        break
                    pending[ex.submit(_download_day, symbol, d, vision_timeout_s)] = d

                while pending:
                    done, _ = concurrent.futures.wait(
                        pending,
                        return_when=concurrent.futures.FIRST_COMPLETED,
                    )
                    for fut in done:
                        pending.pop(fut)  # Drop strong reference; fut GC'd after loop
                        try:
                            date_str, df = fut.result()
                            if df is not None:
                                stage_path = symbol_dir / f".seed_{date_str}.parquet.tmp"
                                _atomic_write_parquet(df, stage_path)
                                staged_rows += len(df)
                                del df  # Only reference gone → immediate GC
                                staged.append(stage_path)
                        except Exception as exc:
                            logger.warning("%s %s: stage error: %s", symbol, month, exc)
                        # Refill pool: submit next day to keep workers busy.
                        d = next(days_iter, None)
                        if d is not None:
                            pending[
                                ex.submit(_download_day, symbol, d, vision_timeout_s)
                            ] = d

            if not staged:
                logger.warning("%s %s: all downloads failed — skipping month", symbol, month)
                continue

            # MEM-017: Sequential per-day streaming appends.
            # One 2-source concat per staged file (existing_month + one_day).
            # Peak memory = read buffers for two files (~15-450 MB total), vs
            # MEM-016v2's 30-source concat that caused ~200 MB/month arena growth
            # in Polars/Python, accumulating to 7.3 GB over 14 months.
            known_cols = list(_VISION_COLUMNS)
            for stage_path in staged:
                try:
                    stage_schema = pl.scan_parquet(stage_path).collect_schema()
                    select_staged = [c for c in known_cols if c in stage_schema.names()]
                    sources: list[pl.LazyFrame] = []
                    if month_path.exists():
                        existing_schema = pl.scan_parquet(month_path).collect_schema()
                        select_existing = [c for c in known_cols if c in existing_schema.names()]
                        sources.append(pl.scan_parquet(month_path).select(select_existing))
                    sources.append(pl.scan_parquet(stage_path).select(select_staged))
                    _atomic_sink_parquet(
                        pl.concat(sources, how="vertical", rechunk=False),
                        month_path,
                    )
                except Exception as exc:
                    # Skip corrupt staging files rather than aborting the whole month.
                    # The missing day will be re-downloaded on the next seeder pass
                    # (for partial months) or left as a minor gap (for historical months).
                    logger.warning(
                        "%s %s: staging merge failed for %s — skipping day: %s",
                        symbol, month, stage_path.name, exc,
                    )

            newly_seeded += len(days)
            logger.info(
                "Seeded %s %s: %d day(s), %d trades",
                symbol, month, len(days), staged_rows,
            )
        finally:
            for f in staged:
                f.unlink(missing_ok=True)
            # MEM-024: Force glibc to return MADV_FREE'd arena pages to the OS
            # after each month. Without this, glibc holds the high-water-mark
            # RSS from the largest month processed (e.g. 4.47 GB from 2021-05)
            # for the entire process lifetime. malloc_trim(0) trims the top of
            # the heap and releases unmapped arenas back to the kernel.
            # Complements MEM-021 (mimalloc) — belt-and-suspenders approach.
            with contextlib.suppress(OSError):  # Non-Linux (dev environment)
                ctypes.CDLL("libc.so.6").malloc_trim(0)

        completed_months += 1
        if state_path is not None and completed_months % 2 == 0:
            state_path.touch()

    return newly_seeded


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def main() -> int:
    """Run seeder over all enabled symbols. Returns exit code."""
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s %(levelname)s %(name)s %(message)s",
        stream=sys.stdout,
    )

    # Load deploy env (non-secrets: OPENDEVIATIONBAR_SEEDER_* vars live here)
    deploy_env = Path.home() / "opendeviationbar-py/deploy/opendeviationbar.env"
    _load_env_file(deploy_env)

    from opendeviationbar.config.seeder import SeederConfig
    from opendeviationbar.storage.parquet import TickStorage
    from opendeviationbar.symbol_registry import (
        get_effective_start_date,
        get_registered_symbols,
    )

    cfg = SeederConfig()
    storage = TickStorage()
    symbols = get_registered_symbols(enabled_only=True)

    t0 = time.monotonic()
    total_seeded = 0
    symbols_processed = 0

    logger.info(
        "Seeder start: %d symbols, historical_start=%s, workers=%d",
        len(symbols),
        cfg.historical_start,
        cfg.max_concurrent_downloads,
    )

    # Write initial state file immediately so monit doesn't see "Does not exist"
    # during long-running seeder passes (thousands of days can take hours).
    state_path = Path(cfg.state_file)
    state_path.write_text(
        f"running\ntimestamp={datetime.now(UTC).isoformat()}\nsymbols={len(symbols)}\n"
    )

    for symbol in symbols:
        # Use symbol's effective_start (or listing_date) when later than config default
        registry_start = get_effective_start_date(symbol)
        config_start = date.fromisoformat(cfg.historical_start)
        if registry_start:
            symbol_start = max(config_start, date.fromisoformat(registry_start))
        else:
            symbol_start = config_start

        try:
            count = _seed_symbol(
                symbol,
                symbol_start,
                storage,
                cfg.max_concurrent_downloads,
                cfg.vision_timeout_s,
                state_path=state_path,
            )
            total_seeded += count
            symbols_processed += 1
        except Exception:
            logger.exception("Seeder failed for %s", symbol)

    elapsed = time.monotonic() - t0
    logger.info(
        "Seeder complete: %d days seeded across %d/%d symbols in %.1fs",
        total_seeded,
        symbols_processed,
        len(symbols),
        elapsed,
    )

    # Overwrite state file with final completion status
    state_path.write_text(
        f"ok\n"
        f"timestamp={datetime.now(UTC).isoformat()}\n"
        f"symbols={symbols_processed}\n"
        f"seeded={total_seeded}\n"
        f"elapsed_s={elapsed:.1f}\n"
    )

    return 0


if __name__ == "__main__":
    sys.exit(main())
