#!/usr/bin/env python3
# FILE-SIZE-OK: Seeder daemon (Issue #270, #278) — Vision download + TickStorage write
"""Proactive Parquet tick cache seeder (Issue #270).

Downloads Binance Vision daily aggTrade ZIPs for all registry-enabled symbols
and writes raw trades to the local Parquet tick cache (TickStorage). This
ensures kintsugi's 64-worker Parquet path (_shard_is_parquet_cached) is
always active — zero Vision downloads needed during kintsugi repairs.

Issue #277: Simplified to write one daily Parquet file per Vision ZIP.
No more merge/append logic (MEM-016/MEM-017/MEM-018 no longer needed).

Issue #278: Added UM futures market support. Configure via
OPENDEVIATIONBAR_SEEDER_MARKETS="spot,um_futures". Futures data is cached
under separate dirs (e.g., BINANCE_UM_BTCUSDT/) with identical Parquet schema.

Key design:
  - Writes with plain symbol key ("BTCUSDT") so kintsugi can find the data.
    (checkpoint.py uses "BINANCE_SPOT_BTCUSDT" — different key, different path.)
  - Does NOT construct bars; pure tick download → Parquet write.
  - Runs idempotently: skips days already cached (daily file existence check).
  - Newest-first ordering matches kintsugi shard priority.
  - Exits 0 and writes state file on success; monit checks state file freshness.

Usage:
  # One-off run (runs all symbols, seeds missing days, exits):
  python scripts/tick_cache_seeder.py

  # Daemon mode via systemd timer (every 5 min):
  systemctl --user start opendeviationbar-seeder.timer
"""

from __future__ import annotations

import concurrent.futures
import io
import json
import logging
import os
import sys
import threading
import time
import zipfile
from datetime import UTC, date, datetime, timedelta
from pathlib import Path
from typing import TYPE_CHECKING

import polars as pl
import requests
from opendeviationbar._core import parse_checksum_file_content, verify_checksum_hex
from opendeviationbar.checksum_telemetry import log_checksum_event
from opendeviationbar.notify.telegram import send_telegram

if TYPE_CHECKING:
    from opendeviationbar.checksum_registry import ChecksumRegistry
    from opendeviationbar.config.seeder import SeederConfig
    from opendeviationbar.storage.parquet import TickStorage

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Vision constants
# ---------------------------------------------------------------------------

_VISION_SPOT_BASE = "https://data.binance.vision/data/spot/daily/aggTrades"
_VISION_UM_BASE = "https://data.binance.vision/data/futures/um/daily/aggTrades"

# Binance Vision spot aggTrades CSV — no header, 7 named columns
# (8th column `was_best_price` silently dropped by Polars new_columns truncation)
_SPOT_COLUMNS = [
    "agg_trade_id",   # int64
    "price",          # float64
    "quantity",       # float64 (Vision calls it "qty"; we rename to "quantity"
                      #          so _select_trade_columns() finds it)
    "first_trade_id", # int64
    "last_trade_id",  # int64
    "timestamp",      # int64 — ms before 2025, µs from 2025 (normalized to µs before write)
    "is_buyer_maker", # bool
]

# Binance Vision UM futures aggTrades CSV — 7 columns exactly (no was_best_price).
# Has header row from 2022-08-11 onward. Timestamps always ms (no µs switch).
# Booleans are lowercase (true/false) vs spot's Title case (True/False).
_UM_COLUMNS = [
    "agg_trade_id",   # int64
    "price",          # float64
    "quantity",       # float64
    "first_trade_id", # int64
    "last_trade_id",  # int64
    "timestamp",      # int64 ms epoch (always ms, never µs)
    "is_buyer_maker", # bool
]

# Date from which UM futures CSVs include a header row
_UM_HEADER_CUTOFF = "2022-08-11"

# Backward compat alias used internally
_VISION_BASE = _VISION_SPOT_BASE
_VISION_COLUMNS = _SPOT_COLUMNS

# Valid market identifiers
VALID_MARKETS = ("spot", "um_futures")


# ---------------------------------------------------------------------------
# Environment loading
# ---------------------------------------------------------------------------

def _load_env_file(env_path: Path) -> None:
    """Load KEY=VALUE lines from env file into os.environ (setdefault)."""
    if not env_path.exists():
        return
    with env_path.open() as f:
        for raw in f:
            stripped = raw.strip()
            if stripped and not stripped.startswith("#") and "=" in stripped:
                k, _, v = stripped.partition("=")
                os.environ.setdefault(k.strip(), v.strip())


# ---------------------------------------------------------------------------
# Vision download helpers
# ---------------------------------------------------------------------------

def _vision_url(symbol: str, date_str: str, *, market: str = "spot") -> str:
    """Build Vision ZIP URL for the given market."""
    if market == "um_futures":
        return f"{_VISION_UM_BASE}/{symbol}/{symbol}-aggTrades-{date_str}.zip"
    return f"{_VISION_SPOT_BASE}/{symbol}/{symbol}-aggTrades-{date_str}.zip"


def _download_vision_day(
    symbol: str,
    date_str: str,
    *,
    timeout_s: int,
    market: str = "spot",
) -> pl.DataFrame | None:
    """Download and parse one Vision ZIP. Returns None if unavailable or error.

    Parameters
    ----------
    symbol : str
        Trading symbol (e.g., "BTCUSDT").
    date_str : str
        Date string "YYYY-MM-DD".
    timeout_s : int
        HTTP timeout in seconds.
    market : str
        Market type: "spot" (default) or "um_futures".
    """
    import requests

    url = _vision_url(symbol, date_str, market=market)
    try:
        resp = requests.get(url, timeout=timeout_s, allow_redirects=True)
    except requests.Timeout:
        logger.warning("%s %s [%s]: Vision download timed out", symbol, date_str, market)
        return None
    except Exception as exc:
        logger.warning("%s %s [%s]: Vision download error: %s", symbol, date_str, market, exc)
        return None

    if resp.status_code == 404:
        logger.debug("%s %s [%s]: Vision 404 — not yet published", symbol, date_str, market)
        return None
    if resp.status_code != 200:
        logger.warning("%s %s [%s]: Vision HTTP %d", symbol, date_str, market, resp.status_code)
        return None

    # Select columns and header handling based on market
    if market == "um_futures":
        columns = _UM_COLUMNS
        # UM futures CSVs have a header row from 2022-08-11 onward
        has_header = date_str >= _UM_HEADER_CUTOFF
    else:
        columns = _SPOT_COLUMNS
        has_header = False

    schema_overrides = {
        "agg_trade_id": pl.Int64,
        "price": pl.Float64,
        "quantity": pl.Float64,
        "first_trade_id": pl.Int64,
        "last_trade_id": pl.Int64,
        "timestamp": pl.Int64,
        "is_buyer_maker": pl.Boolean,
    }

    try:
        with zipfile.ZipFile(io.BytesIO(resp.content)) as zf:
            csv_name = f"{symbol}-aggTrades-{date_str}.csv"
            with zf.open(csv_name) as csv_f:
                if has_header:
                    # UM futures with header: read with header, then rename
                    df = pl.read_csv(
                        csv_f,
                        has_header=True,
                        schema_overrides={
                            # Use the Vision header names for schema overrides
                            "agg_trade_id": pl.Int64,
                            "price": pl.Float64,
                            "quantity": pl.Float64,
                            "first_trade_id": pl.Int64,
                            "last_trade_id": pl.Int64,
                            "transact_time": pl.Int64,
                            "is_buyer_maker": pl.Boolean,
                        },
                    )
                    # Rename transact_time → timestamp to match our schema
                    if "transact_time" in df.columns:
                        df = df.rename({"transact_time": "timestamp"})
                    # Select only the columns we need (in case of extra columns)
                    df = df.select(columns)
                else:
                    # Spot or early UM futures: no header, positional columns
                    df = pl.read_csv(
                        csv_f,
                        has_header=False,
                        new_columns=columns,
                        schema_overrides=schema_overrides,
                    )
    except (zipfile.BadZipFile, KeyError) as exc:
        logger.warning("%s %s [%s]: ZIP parse error: %s", symbol, date_str, market, exc)
        return None
    except Exception as exc:
        logger.warning("%s %s [%s]: CSV parse error: %s", symbol, date_str, market, exc)
        return None

    # Normalize all timestamps to microseconds for uniform storage.
    # - Post-2025-01-01 spot data: already µs (16-digit), pass through.
    # - Pre-2025 spot data: ms (13-digit), upscale x1000.
    # - UM futures: always ms, upscale x1000.
    if not df.is_empty():
        max_ts = df["timestamp"].max()
        if max_ts is not None and max_ts < 1_500_000_000_000_000:
            # ms timestamps → upscale to µs
            df = df.with_columns((pl.col("timestamp") * 1000).alias("timestamp"))

    if df.is_empty():
        logger.warning("%s %s [%s]: Vision CSV was empty", symbol, date_str, market)
        return None

    # Select only the 7 canonical columns — Vision spot CSVs have an 8th column
    # (was_best_price) that gets auto-named "column_8" by Polars. Drop it to
    # ensure REST-sourced and Vision-sourced files have identical schemas.
    canonical = ["agg_trade_id", "price", "quantity", "first_trade_id",
                 "last_trade_id", "timestamp", "is_buyer_maker"]
    df = df.select([c for c in canonical if c in df.columns])

    # Sort by agg_trade_id for deterministic ordering within each daily file
    df = df.sort("agg_trade_id")

    return df


# ---------------------------------------------------------------------------
# Checksum verification (Phase 10 — SEEDER-01, SEEDER-02, SEEDER-03)
# ---------------------------------------------------------------------------


def _download_vision_day_raw(
    symbol: str,
    date_str: str,
    *,
    timeout_s: int,
    market: str = "spot",
) -> tuple[bytes, pl.DataFrame | None]:
    """Download Vision ZIP and return (raw_bytes, parsed_df).

    Thin wrapper around _download_vision_day that also returns the raw
    response bytes needed for SHA-256 checksum verification.
    """
    url = _vision_url(symbol, date_str, market=market)
    try:
        resp = requests.get(url, timeout=timeout_s, allow_redirects=True)
    except requests.Timeout:
        logger.warning("%s %s [%s]: Vision download timed out", symbol, date_str, market)
        return b"", None
    except Exception as exc:
        logger.warning("%s %s [%s]: Vision download error: %s", symbol, date_str, market, exc)
        return b"", None

    if resp.status_code == 404:
        logger.debug("%s %s [%s]: Vision 404 — not yet published", symbol, date_str, market)
        return b"", None
    if resp.status_code != 200:
        logger.warning("%s %s [%s]: Vision HTTP %d", symbol, date_str, market, resp.status_code)
        return b"", None

    raw_bytes = resp.content

    # Delegate CSV parsing to _download_vision_day, but we already have the
    # response content. Re-parse using the same logic as _download_vision_day.
    if market == "um_futures":
        columns = _UM_COLUMNS
        has_header = date_str >= _UM_HEADER_CUTOFF
    else:
        columns = _SPOT_COLUMNS
        has_header = False

    schema_overrides = {
        "agg_trade_id": pl.Int64,
        "price": pl.Float64,
        "quantity": pl.Float64,
        "first_trade_id": pl.Int64,
        "last_trade_id": pl.Int64,
        "timestamp": pl.Int64,
        "is_buyer_maker": pl.Boolean,
    }

    try:
        with zipfile.ZipFile(io.BytesIO(raw_bytes)) as zf:
            csv_name = f"{symbol}-aggTrades-{date_str}.csv"
            with zf.open(csv_name) as csv_f:
                if has_header:
                    df = pl.read_csv(
                        csv_f,
                        has_header=True,
                        schema_overrides={
                            "agg_trade_id": pl.Int64,
                            "price": pl.Float64,
                            "quantity": pl.Float64,
                            "first_trade_id": pl.Int64,
                            "last_trade_id": pl.Int64,
                            "transact_time": pl.Int64,
                            "is_buyer_maker": pl.Boolean,
                        },
                    )
                    if "transact_time" in df.columns:
                        df = df.rename({"transact_time": "timestamp"})
                    df = df.select(columns)
                else:
                    df = pl.read_csv(
                        csv_f,
                        has_header=False,
                        new_columns=columns,
                        schema_overrides=schema_overrides,
                    )
    except (zipfile.BadZipFile, KeyError) as exc:
        logger.warning("%s %s [%s]: ZIP parse error: %s", symbol, date_str, market, exc)
        return raw_bytes, None
    except Exception as exc:
        logger.warning("%s %s [%s]: CSV parse error: %s", symbol, date_str, market, exc)
        return raw_bytes, None

    # Normalize timestamps to microseconds
    if not df.is_empty():
        max_ts = df["timestamp"].max()
        if max_ts is not None and max_ts < 1_500_000_000_000_000:
            df = df.with_columns((pl.col("timestamp") * 1000).alias("timestamp"))

    if df.is_empty():
        logger.warning("%s %s [%s]: Vision CSV was empty", symbol, date_str, market)
        return raw_bytes, None

    canonical = [
        "agg_trade_id", "price", "quantity", "first_trade_id",
        "last_trade_id", "timestamp", "is_buyer_maker",
    ]
    df = df.select([c for c in canonical if c in df.columns])
    df = df.sort("agg_trade_id")

    return raw_bytes, df


def _verify_vision_checksum(
    zip_bytes: bytes,
    symbol: str,
    date_str: str,
    timeout_s: int,
    *,
    market: str = "spot",
) -> dict:
    """Verify ZIP bytes against the Vision .CHECKSUM file.

    Returns:
        {"verified": True, "sha256": hex_hash}   on match
        {"verified": False, "sha256": None, "reason": "404"|"timeout"|"invalid_format"}
            on soft-skip
    Raises:
        RuntimeError on hash mismatch (caller handles retry)
    """
    checksum_url = _vision_url(symbol, date_str, market=market) + ".CHECKSUM"

    try:
        resp = requests.get(checksum_url, timeout=timeout_s)
    except (requests.Timeout, requests.ConnectionError):
        logger.warning("%s %s: CHECKSUM fetch timed out / network error", symbol, date_str)
        return {"verified": False, "sha256": None, "reason": "timeout"}
    except Exception:
        logger.warning("%s %s: CHECKSUM fetch failed", symbol, date_str, exc_info=True)
        return {"verified": False, "sha256": None, "reason": "timeout"}

    if resp.status_code == 404:
        logger.debug("%s %s: no .CHECKSUM file (404) — soft-skip", symbol, date_str)
        return {"verified": False, "sha256": None, "reason": "404"}

    if resp.status_code != 200:
        logger.warning("%s %s: CHECKSUM HTTP %d — soft-skip", symbol, date_str, resp.status_code)
        return {"verified": False, "sha256": None, "reason": "404"}

    try:
        expected_hash = parse_checksum_file_content(resp.text)
    except ValueError:
        logger.warning("%s %s: invalid CHECKSUM format — soft-skip", symbol, date_str)
        return {"verified": False, "sha256": None, "reason": "invalid_format"}

    # verify_checksum_hex raises RuntimeError on mismatch
    verify_checksum_hex(zip_bytes, expected_hash)
    return {"verified": True, "sha256": expected_hash}


def _download_and_verify_vision_day(
    symbol: str,
    date_str: str,
    timeout_s: int,
    *,
    registry: ChecksumRegistry,
    stats: dict[str, int],
    stats_lock: threading.Lock,
    market: str = "spot",
    max_retries: int = 2,
) -> pl.DataFrame | None:
    """Download Vision ZIP, verify checksum, record in registry.

    On checksum mismatch, retries up to *max_retries* times.
    On persistent failure, sends LOUD Telegram alert and returns None.
    On soft-skip (404/timeout), proceeds without verification.
    """
    for attempt in range(1 + max_retries):
        raw_bytes, df = _download_vision_day_raw(
            symbol, date_str, timeout_s=timeout_s, market=market,
        )
        if df is None:
            return None  # Vision 404 or parse error — no retry needed

        try:
            result = _verify_vision_checksum(
                raw_bytes, symbol, date_str, timeout_s, market=market,
            )
        except RuntimeError:
            # Checksum mismatch — retry
            if attempt < max_retries:
                logger.warning(
                    "%s %s: checksum mismatch (attempt %d/%d) — retrying",
                    symbol, date_str, attempt + 1, 1 + max_retries,
                )
                log_checksum_event(
                    "verify_retry", symbol, date_str,
                    source="seeder",
                    reason=f"attempt {attempt + 1}/{1 + max_retries}",
                )
                continue
            # All retries exhausted
            logger.warning(
                "%s %s: checksum mismatch after %d retries — LOUD alert",
                symbol, date_str, max_retries,
            )
            log_checksum_event(
                "verify_fail", symbol, date_str,
                source="seeder",
                reason="max_retries_exhausted",
            )
            with stats_lock:
                stats["fail"] += 1
            try:
                send_telegram(
                    f"<b>CHECKSUM FAIL</b>\n\n"
                    f"Symbol: {symbol}\n"
                    f"Date: {date_str}\n\n"
                    f"All {max_retries} retries exhausted. "
                    f"Day skipped — manual investigation required.",
                    disable_notification=False,
                )
            except Exception:
                logger.warning("Failed to send Telegram alert", exc_info=True)
            return None

        # Verification result received (no exception)
        if result.get("verified"):
            sha256 = result["sha256"]
            registry.add(symbol=symbol, date=date_str, sha256=sha256, source="seeder")
            with stats_lock:
                stats["pass"] += 1
            logger.info("%s %s: checksum verified (%s...)", symbol, date_str, sha256[:12])
            log_checksum_event(
                "verify_pass", symbol, date_str,
                source="seeder", expected_hash=sha256[:16],
            )
        else:
            # Soft-skip (404/timeout/invalid_format)
            with stats_lock:
                stats["skip"] += 1
            logger.debug(
                "%s %s: checksum soft-skip (reason=%s)", symbol, date_str, result.get("reason"),
            )
            log_checksum_event(
                "verify_skip", symbol, date_str,
                source="seeder", reason=result.get("reason", "unknown"),
            )

        return df

    return None  # unreachable, but keeps linters happy


# ---------------------------------------------------------------------------
# Day gap discovery
# ---------------------------------------------------------------------------

def _find_missing_days(
    symbol: str,
    start_date: date,
    end_date: date,
    storage: TickStorage,
) -> list[str]:
    """Return date strings needing seeding, newest-first.

    Issue #277: With daily Parquet files, checking is simple:
    for each day in [start_date, end_date], check if the daily file exists.
    """
    missing: list[str] = []

    current = start_date
    while current <= end_date:
        date_str = current.isoformat()
        day_path = storage._get_parquet_path(symbol, date_str)
        if not day_path.exists():
            missing.append(date_str)
        elif day_path.with_suffix(".parquet.meta").exists():
            # REST-sourced file — include so Vision can overwrite (D-08)
            missing.append(date_str)
        # else: Parquet exists without .meta = kintsugi/Vision already processed (SEEDER-04), skip
        current += timedelta(days=1)

    # Newest-first (matches kintsugi's priority ordering)
    missing.sort(reverse=True)
    return missing


# ---------------------------------------------------------------------------
# Per-day download worker
# ---------------------------------------------------------------------------

def _download_day(
    symbol: str,
    date_str: str,
    timeout_s: int,
    market: str = "spot",
) -> tuple[str, pl.DataFrame | None]:
    """Download one Vision day. Returns (date_str, df_or_None)."""
    return date_str, _download_vision_day(symbol, date_str, timeout_s=timeout_s, market=market)


def _download_and_verify_day_worker(
    symbol: str,
    date_str: str,
    timeout_s: int,
    market: str,
    registry: ChecksumRegistry,
    stats: dict[str, int],
    stats_lock: threading.Lock,
) -> tuple[str, pl.DataFrame | None]:
    """Thread pool worker: download + checksum verify. Returns (date_str, df_or_None)."""
    df = _download_and_verify_vision_day(
        symbol, date_str, timeout_s,
        registry=registry, stats=stats, stats_lock=stats_lock, market=market,
    )
    return date_str, df


# ---------------------------------------------------------------------------
# REST fallback (D-01, D-02, D-04)
# ---------------------------------------------------------------------------

def _rest_fallback_day(
    symbol: str,
    date_str: str,
    market: str = "spot",
    resume_from_id: int | None = None,
) -> pl.DataFrame | None:
    """Fetch one day's trades via REST API when Vision 404s.

    If resume_from_id is provided, fetches only trades AFTER that ID
    (incremental — avoids re-downloading the entire day on each run).

    Returns a polars DataFrame matching Vision schema, or None on failure.
    """
    if market != "spot":
        return None  # REST fallback only for spot (primary market)

    from opendeviationbar._core import fetch_aggtrades_parallel, fetch_aggtrades_rest

    target_date = date.fromisoformat(date_str)
    day_start_ms = int(datetime.combine(target_date, datetime.min.time(), tzinfo=UTC).timestamp() * 1000)
    now_ms = int(datetime.now(UTC).timestamp() * 1000)
    # For today: use now as end. For past days: use end-of-day.
    day_end_ms = min(day_start_ms + 86_400_000 - 1, now_ms)

    try:
        # Step 1: Determine first trade ID
        if resume_from_id is not None:
            # Incremental: start from last known trade + 1
            first_trade_id = resume_from_id + 1
        else:
            # Full day: find first trade via time-based query
            first_trades = fetch_aggtrades_rest(symbol, day_start_ms, day_start_ms + 60_000)
            if not first_trades:
                logger.warning("%s %s: REST fallback found no trades at day start", symbol, date_str)
                return None
            first_trade_id = first_trades[0]["agg_trade_id"]

        # Step 2: Get last trade ID — use recent window ending at day_end_ms (or now)
        last_trades = fetch_aggtrades_rest(symbol, max(day_start_ms, day_end_ms - 60_000), day_end_ms)
        if not last_trades:
            logger.debug("%s %s: no trades in recent window", symbol, date_str)
            return None
        last_trade_id = last_trades[-1]["agg_trade_id"]

        # Nothing new since last run
        if first_trade_id > last_trade_id:
            return None

        # Step 3: Bulk fetch using from_id/to_id parallel fetch
        trades = fetch_aggtrades_parallel(symbol, first_trade_id, last_trade_id)
    except Exception as exc:
        logger.warning("%s %s: REST fallback failed: %s", symbol, date_str, exc)
        return None

    if not trades:
        logger.warning("%s %s: REST fallback returned 0 trades", symbol, date_str)
        return None

    # Verify response covers the requested range
    actual_last = trades[-1]["agg_trade_id"]
    if actual_last < last_trade_id:
        logger.warning(
            "%s %s: REST partial — got tid %d, expected %d (%d trades missing)",
            symbol, date_str, actual_last, last_trade_id, last_trade_id - actual_last,
        )
        # Return what we have — write_ticks merges, next run resumes from max_tid

    # Convert to polars DataFrame — Rust bindings already use full column names
    df = pl.DataFrame(trades)

    # Ensure correct types (Rust returns correct names: agg_trade_id, price, etc.)
    df = df.select([
        pl.col("agg_trade_id").cast(pl.Int64),
        pl.col("price").cast(pl.Float64),
        pl.col("quantity").cast(pl.Float64),
        pl.col("first_trade_id").cast(pl.Int64),
        pl.col("last_trade_id").cast(pl.Int64),
        pl.col("timestamp").cast(pl.Int64),
        pl.col("is_buyer_maker").cast(pl.Boolean),
    ])
    df = df.sort("agg_trade_id")
    return df


# ---------------------------------------------------------------------------
# Batch rotation helpers (D-14, D-20)
# ---------------------------------------------------------------------------

def _load_batch_offset(state_path: Path, total_symbols: int) -> int:
    """Load batch offset from state file.  Returns 0 on any error."""
    try:
        data = json.loads(state_path.read_text())
        offset = data.get("batch_offset", 0)
        if offset >= total_symbols:
            return 0
        return offset
    except (FileNotFoundError, json.JSONDecodeError, OSError):
        return 0


def _save_batch_offset(state_path: Path, offset: int) -> None:
    """Save next batch offset to state file."""
    import contextlib

    with contextlib.suppress(OSError):
        state_path.write_text(json.dumps({
            "batch_offset": offset,
            "updated": datetime.now(UTC).isoformat(),
        }))


# ---------------------------------------------------------------------------
# Day-boundary backfill (SEED-01, SEED-02)
# ---------------------------------------------------------------------------



# ---------------------------------------------------------------------------
# Per-symbol seeder
# ---------------------------------------------------------------------------

def _cache_key(symbol: str, market: str) -> str:
    """Build the TickStorage cache key for a (symbol, market) pair.

    Spot uses the plain symbol ("BTCUSDT") for backward compatibility.
    Futures use a prefixed key ("BINANCE_UM_BTCUSDT") to avoid collision.
    """
    if market == "um_futures":
        return f"BINANCE_UM_{symbol}"
    return symbol


def _seed_symbol(
    symbol: str,
    effective_start: date,
    storage: TickStorage,
    max_workers: int,
    vision_timeout_s: int,
    state_path: Path | None = None,
    run_start_iso: str | None = None,
    market: str = "spot",
    cfg: SeederConfig | None = None,
    registry: ChecksumRegistry | None = None,
    checksum_stats: dict[str, int] | None = None,
    checksum_lock: threading.Lock | None = None,
) -> int:
    """Seed all missing days for one symbol. Returns count of newly seeded days.

    Issue #277: Each Vision daily ZIP → one daily Parquet file.
    No more merge/append logic. Each day is one atomic write.

    Issue #278: market parameter selects spot vs um_futures Vision source.
    """
    today = datetime.now(UTC).date()
    yesterday = today - timedelta(days=1)

    cache_sym = _cache_key(symbol, market)

    if effective_start > today:
        logger.debug("%s [%s]: effective_start %s is in the future — skip", symbol, market, effective_start)
        return 0

    # Seed through today — Vision handles historical, REST handles recent days
    # including today. No conflict with sidecar: seeder writes Parquet ticks,
    # sidecar writes ClickHouse bars (different stores).
    missing = _find_missing_days(cache_sym, effective_start, today, storage)

    # Today always re-seeds (incremental — write_ticks merges via anti-join)
    # because the file grows as the day progresses.
    today_str = today.isoformat()
    if today_str not in missing:
        missing.insert(0, today_str)  # newest-first
    if not missing:
        logger.debug("%s [%s]: fully cached up to %s", symbol, market, yesterday)
        return 0

    logger.info("%s [%s]: %d days to seed (newest: %s)", symbol, market, len(missing), missing[0])

    # Ensure symbol directory exists
    symbol_dir = storage._get_symbol_dir(cache_sym)
    symbol_dir.mkdir(parents=True, exist_ok=True)

    newly_seeded = 0
    completed_days = 0
    rest_seeded_days: list[str] = []
    vision_seeded_days: list[str] = []

    # -----------------------------------------------------------------------
    # REST-first for T-1 (yesterday) — seed via REST immediately before
    # attempting Vision download. This ensures kintsugi can repair yesterday
    # from local Parquet cache on its very first pass. Vision will overwrite
    # on the next seeder run once the ZIP is published (D-08).
    # -----------------------------------------------------------------------
    yesterday_str = yesterday.isoformat()
    if (
        cfg is not None
        and cfg.rest_first_t1
        and cfg.rest_fallback_enabled
        and market == "spot"
        and yesterday_str in missing
    ):
        logger.info(
            "%s %s [%s]: REST-seeding T-1 (Vision not yet available)",
            symbol, yesterday_str, market,
        )
        df_t1 = _rest_fallback_day(symbol, yesterday_str, market)
        if df_t1 is not None:
            storage.write_ticks(cache_sym, df_t1)
            newly_seeded += 1
            completed_days += 1
            rest_seeded_days.append(yesterday_str)

            # Create .meta sidecar so Vision can overwrite later (D-08)
            try:
                meta_path = storage._get_parquet_path(cache_sym, yesterday_str).with_suffix(".parquet.meta")
                meta_path.write_text(json.dumps({
                    "source": "REST",
                    "complete": False,
                    "fetched_at": datetime.now(UTC).isoformat(),
                    "trade_count": df_t1.height,
                }))
            except OSError:
                pass  # non-critical metadata

            logger.info(
                "%s %s [%s]: REST-seeded T-1 successfully, %d trades",
                symbol, yesterday_str, market, df_t1.height,
            )
            del df_t1

            # Remove yesterday from missing list — already handled
            missing = [d for d in missing if d != yesterday_str]

    if not missing:
        # Only yesterday was missing and we just seeded it
        if newly_seeded > 0:
            logger.info("Seeded %s [%s]: %d day(s) (REST: %d, Vision: %d)",
                        symbol, market, newly_seeded, len(rest_seeded_days), len(vision_seeded_days))
        return newly_seeded

    # Checksum-aware download worker (Phase 10)
    _cs_registry = registry
    _cs_stats = checksum_stats or {"pass": 0, "fail": 0, "skip": 0}
    _cs_lock = checksum_lock or threading.Lock()

    def _submit_download(sym: str, d: str) -> concurrent.futures.Future:
        """Submit a download task — checksum-aware when registry is available."""
        if _cs_registry is not None:
            return ex.submit(
                _download_and_verify_day_worker,
                sym, d, vision_timeout_s, market,
                _cs_registry, _cs_stats, _cs_lock,
            )
        return ex.submit(_download_day, sym, d, vision_timeout_s, market)

    # MEM-018: Bounded future submission — only max_workers futures in flight
    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as ex:
        pending: dict[concurrent.futures.Future[tuple[str, object]], str] = {}
        days_iter = iter(missing)

        # Seed the pool up to max_workers in-flight futures.
        for _ in range(max_workers):
            d = next(days_iter, None)
            if d is None:
                break
            pending[_submit_download(symbol, d)] = d

        while pending:
            done, _ = concurrent.futures.wait(
                pending,
                return_when=concurrent.futures.FIRST_COMPLETED,
            )
            for fut in done:
                pending.pop(fut)  # Drop strong reference
                try:
                    date_str, df = fut.result()
                    rest_fallback_used = False

                    # REST fallback when Vision 404s — no age cutoff
                    # Seeder writes Parquet ticks, sidecar writes CH bars — no conflict
                    if df is None and cfg is not None and cfg.rest_fallback_enabled:
                            # Incremental: if Parquet exists, resume from last trade ID
                            resume_id = None
                            parquet_path = storage._get_parquet_path(cache_sym, date_str)
                            if parquet_path.exists():
                                try:
                                    existing = pl.read_parquet(parquet_path, columns=["agg_trade_id"])
                                    if not existing.is_empty():
                                        resume_id = existing["agg_trade_id"].max()
                                        logger.info(
                                            "%s %s [%s]: resuming REST from agg_trade_id=%d",
                                            symbol, date_str, market, resume_id,
                                        )
                                    del existing
                                except Exception:
                                    pass  # Corrupt file — full re-fetch

                            df = _rest_fallback_day(symbol, date_str, market, resume_from_id=resume_id)
                            if df is not None:
                                rest_fallback_used = True
                                logger.info(
                                    "%s %s [%s]: REST %s, %d trades",
                                    symbol, date_str, market,
                                    "incremental" if resume_id else "full",
                                    df.height,
                                )

                    if df is not None:
                        # Check if this day was previously REST-sourced (D-08)
                        meta_path = storage._get_parquet_path(cache_sym, date_str).with_suffix(".parquet.meta")
                        if not rest_fallback_used and meta_path.exists():
                            logger.info(
                                "%s %s [%s]: Vision overwriting REST-sourced file",
                                symbol, date_str, market,
                            )
                            meta_path.unlink(missing_ok=True)

                        # Route through write_ticks() for dedup + trade-ID continuity validation (D-09)
                        storage.write_ticks(cache_sym, df)
                        newly_seeded += 1
                        completed_days += 1

                        # Track source for summary logging
                        if rest_fallback_used:
                            rest_seeded_days.append(date_str)
                        else:
                            vision_seeded_days.append(date_str)

                        # Create .meta sidecar for REST-sourced files (D-08)
                        if rest_fallback_used:
                            try:
                                meta_path = storage._get_parquet_path(cache_sym, date_str).with_suffix(".parquet.meta")
                                meta_path.write_text(json.dumps({
                                    "source": "REST",
                                    "complete": False,
                                    "fetched_at": datetime.now(UTC).isoformat(),
                                    "trade_count": df.height,
                                }))
                            except OSError:
                                pass  # non-critical metadata

                        del df  # Only reference gone → immediate GC

                        if completed_days % 50 == 0:
                            logger.info(
                                "%s [%s]: seeded %d/%d days so far",
                                symbol, market, completed_days, len(missing),
                            )
                except Exception as exc:
                    logger.warning("%s [%s]: download/write error: %s", symbol, market, exc)

                # Refill pool: submit next day to keep workers busy.
                d = next(days_iter, None)
                if d is not None:
                    pending[_submit_download(symbol, d)] = d

    if newly_seeded > 0:
        logger.info("Seeded %s [%s]: %d day(s) (REST: %d, Vision: %d)",
                     symbol, market, newly_seeded, len(rest_seeded_days), len(vision_seeded_days))

    # Write progress state if applicable
    if state_path is not None and newly_seeded > 0:
        try:
            state_path.write_text(
                f"running\n"
                f"timestamp={run_start_iso or datetime.now(UTC).isoformat()}\n"
                f"current_symbol={symbol}\n"
                f"market={market}\n"
                f"days_done={newly_seeded}\n"
            )
        except OSError:
            logger.warning("Failed to write progress state file %s", state_path, exc_info=True)

    return newly_seeded


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def _parse_markets(raw: str) -> list[str]:
    """Parse comma-separated markets string into validated list."""
    markets = [m.strip() for m in raw.split(",") if m.strip()]
    for m in markets:
        if m not in VALID_MARKETS:
            msg = f"Invalid market '{m}'. Valid: {', '.join(VALID_MARKETS)}"
            raise ValueError(msg)
    return markets if markets else ["spot"]


def main() -> int:
    """Run seeder over all enabled symbols. Returns exit code."""
    import setproctitle

    setproctitle.setproctitle(
        "odb-tick-cache-seeder: binance-spot vision-download parquet-writer"
    )

    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s %(levelname)s %(name)s %(message)s",
        stream=sys.stdout,
    )

    # Load deploy env (non-secrets: OPENDEVIATIONBAR_SEEDER_* vars live here)
    deploy_env = Path.home() / "opendeviationbar-py/deploy/opendeviationbar.env"
    _load_env_file(deploy_env)

    from opendeviationbar.config.seeder import SeederConfig
    from opendeviationbar.storage.parquet import TickStorage
    from opendeviationbar.symbol_registry import (
        get_effective_start_date,
        get_symbol_entries,
    )

    cfg = SeederConfig()
    storage = TickStorage()
    markets = _parse_markets(cfg.markets)

    # Filter for Binance symbols only — Exness/forex have their own seeder
    # (scripts/exness_tick_cache_seeder.py). For each configured market,
    # include symbols whose registry market matches.
    entries = get_symbol_entries()
    _market_map = {"spot": "spot", "um_futures": "futures-um"}
    binance_symbols: list[str] = []
    skipped_symbols: list[str] = []
    for sym, entry in entries.items():
        if not entry.enabled:
            continue
        if entry.exchange != "binance":
            skipped_symbols.append(sym)
            continue
        # Include if symbol's market matches any configured market
        if any(entry.market == _market_map.get(m, m) for m in markets):
            binance_symbols.append(sym)

    if skipped_symbols:
        logger.info(
            "Skipped %d non-Binance symbols (handled by dedicated seeders): %s",
            len(skipped_symbols), skipped_symbols,
        )

    symbols = tuple(sorted(binance_symbols))

    # Cross-process rate limit coordinator (D-11)
    from opendeviationbar.ratelimit import RateLimitCoordinator

    coordinator = RateLimitCoordinator()

    # Checksum verification registry (Phase 10 — SEEDER-01/02/03)
    from opendeviationbar.checksum_registry import ChecksumRegistry

    checksum_registry = ChecksumRegistry()
    checksum_stats: dict[str, int] = {"pass": 0, "fail": 0, "skip": 0}
    checksum_lock = threading.Lock()

    # Batch rotation (D-14): process cfg.batch_size symbols per run
    # Prioritize by volume: BTCUSDT and ETHUSDT always first
    priority = ["BTCUSDT", "ETHUSDT"]
    ordered = [s for s in priority if s in symbols] + [s for s in symbols if s not in priority]

    batch_state_path = Path(cfg.last_seeded_state_file)
    batch_offset = _load_batch_offset(batch_state_path, len(ordered))

    # Select this run's batch
    batch = ordered[batch_offset:batch_offset + cfg.batch_size]
    if not batch:
        batch = ordered[:cfg.batch_size]
        batch_offset = 0

    t0 = time.monotonic()
    total_seeded = 0
    symbols_processed = 0
    symbol_counts: dict[str, int] = {}
    symbols_failed: list[str] = []

    logger.info(
        "Seeder start: batch %d-%d of %d symbols (%s), markets=%s, historical_start=%s, workers=%d",
        batch_offset,
        batch_offset + len(batch),
        len(ordered),
        list(batch),
        markets,
        cfg.historical_start,
        cfg.max_concurrent_downloads,
    )

    # Write initial state file immediately so monit doesn't see "Does not exist"
    # during long-running seeder passes (thousands of days can take hours).
    run_start_iso = datetime.now(UTC).isoformat()
    state_path = Path(cfg.state_file)
    try:
        state_path.write_text(
            f"running\ntimestamp={run_start_iso}\nsymbols={len(batch)}\nmarkets={','.join(markets)}\n"
        )
    except OSError:
        logger.warning("Failed to write initial state file %s", state_path, exc_info=True)

    for market in markets:
        for symbol in batch:
            # Use symbol's effective_start (or listing_date) when later than config default
            registry_start = get_effective_start_date(symbol)
            config_start = date.fromisoformat(cfg.historical_start)
            if registry_start:
                symbol_start = max(config_start, date.fromisoformat(registry_start))
            else:
                symbol_start = config_start

            task_key = f"{symbol}@{market}"
            try:
                count = _seed_symbol(
                    symbol,
                    symbol_start,
                    storage,
                    cfg.max_concurrent_downloads,
                    cfg.vision_timeout_s,
                    state_path=state_path,
                    run_start_iso=run_start_iso,
                    market=market,
                    cfg=cfg,
                    registry=checksum_registry,
                    checksum_stats=checksum_stats,
                    checksum_lock=checksum_lock,
                )
                total_seeded += count
                symbols_processed += 1
                symbol_counts[task_key] = count

                # Report weight to cross-process coordinator (D-11)
                try:
                    from opendeviationbar._core import get_rest_rate_limiter_status

                    status = get_rest_rate_limiter_status()
                    used = int(status.get("max_weight_per_minute", 6000) * status.get("utilization", 0))
                    coordinator.report_usage("seeder", used)
                except Exception:
                    pass  # non-critical telemetry
            except Exception:
                logger.exception("Seeder failed for %s", task_key)
                symbols_failed.append(task_key)

    # Save next batch offset for rotation (D-14)
    next_offset = batch_offset + cfg.batch_size
    if next_offset >= len(ordered):
        next_offset = 0
    _save_batch_offset(batch_state_path, next_offset)

    # Persist checksum registry (batch save — not per-file)
    try:
        checksum_registry.save()
    except Exception:
        logger.warning("Failed to save checksum registry", exc_info=True)

    elapsed = time.monotonic() - t0
    logger.info(
        "Seeder complete: %d days seeded across %d tasks (batch of %d symbols x %d markets) in %.1fs"
        " | checksum pass=%d fail=%d skip=%d",
        total_seeded,
        symbols_processed,
        len(batch),
        len(markets),
        elapsed,
        checksum_stats["pass"],
        checksum_stats["fail"],
        checksum_stats["skip"],
    )

    # Overwrite state file with final completion status.
    # Per-symbol counts enable heartbeat and monit to detect ETHUSDT-specific failures.
    per_symbol = "".join(f"{s}_seeded={c}\n" for s, c in symbol_counts.items())
    try:
        state_path.write_text(
            f"ok\n"
            f"timestamp={datetime.now(UTC).isoformat()}\n"
            f"symbols={symbols_processed}\n"
            f"batch_offset={batch_offset}\n"
            f"markets={','.join(markets)}\n"
            f"seeded={total_seeded}\n"
            f"elapsed_s={elapsed:.1f}\n"
            f"failed={len(symbols_failed)}\n"
            f"checksum_pass={checksum_stats['pass']}\n"
            f"checksum_fail={checksum_stats['fail']}\n"
            f"checksum_skip={checksum_stats['skip']}\n"
            + per_symbol
        )
    except OSError:
        logger.warning("Failed to write final state file %s", state_path, exc_info=True)

    if symbols_failed:
        logger.error("Seeder failed for %d task(s): %s", len(symbols_failed), symbols_failed)
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main())
