#!/usr/bin/env python3
"""Integrity sweep: audit Parquet cache files against Binance Vision CHECKSUMs.

Retroactively verifies all historical Parquet data that was downloaded before
Phase 10 added checksum verification to the seeder pipeline.

Usage::

    # Dry run -- list work items without downloading
    python scripts/integrity_sweep.py --dry-run

    # Incremental sweep (skip already-verified)
    python scripts/integrity_sweep.py

    # Full sweep (re-verify everything)
    python scripts/integrity_sweep.py --full

    # Single symbol with date range
    python scripts/integrity_sweep.py --symbol BTCUSDT --start-date 2024-01-01 --end-date 2024-06-30

    # With Telegram summary
    python scripts/integrity_sweep.py --telegram
"""

from __future__ import annotations

import argparse
import logging
import re
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path

import requests
from opendeviationbar._core import compute_sha256_hex, parse_checksum_file_content
from opendeviationbar.checksum_registry import ChecksumRegistry
from opendeviationbar.checksum_telemetry import log_checksum_event
from opendeviationbar.storage.parquet import TickStorage

logger = logging.getLogger(__name__)

_VISION_SPOT_BASE = (
    "https://data.binance.vision/data/spot/daily/aggTrades"
)

# Pattern to extract base symbol from directory names like "BINANCE_SPOT_BTCUSDT"
_PREFIX_RE = re.compile(r"^(?:BINANCE_SPOT_|BINANCE_UM_)?(.+)$")


# ---------------------------------------------------------------------------
# Core logic functions (testable, no side effects beyond arguments)
# ---------------------------------------------------------------------------


def _extract_base_symbol(dir_name: str) -> str:
    """Extract the base symbol from a potentially prefixed directory name.

    Examples:
        "BTCUSDT" -> "BTCUSDT"
        "BINANCE_SPOT_BTCUSDT" -> "BTCUSDT"
    """
    m = _PREFIX_RE.match(dir_name)
    return m.group(1) if m else dir_name


def _build_work_items(
    ticks_dir: Path,
    registry: ChecksumRegistry,
    args: argparse.Namespace,
) -> list[dict]:
    """Discover Parquet cache files and filter into work items.

    Parameters
    ----------
    ticks_dir
        Path to the ticks cache directory (e.g., ``~/.cache/opendeviationbar/ticks/``).
    registry
        ChecksumRegistry for incremental skip lookups.
    args
        Parsed CLI arguments with ``full``, ``symbol``, ``start_date``, ``end_date``.

    Returns
    -------
    list[dict]
        Work items, each with keys: ``symbol``, ``date``, ``dir_name``.
    """
    items: list[dict] = []

    if not ticks_dir.exists():
        return items

    for symbol_dir in sorted(ticks_dir.iterdir()):
        if not symbol_dir.is_dir():
            continue

        dir_name = symbol_dir.name
        base_symbol = _extract_base_symbol(dir_name)

        # --symbol filter
        if args.symbol and base_symbol != args.symbol:
            continue

        # List all parquet days in this directory
        days = sorted(f.stem for f in symbol_dir.glob("*.parquet"))

        for day in days:
            # --start-date / --end-date filter
            if args.start_date and day < args.start_date:
                continue
            if args.end_date and day > args.end_date:
                continue

            # Incremental: skip if already verified (unless --full)
            if not args.full and registry.has_verified(base_symbol, day):
                continue

            items.append({
                "symbol": base_symbol,
                "date": day,
                "dir_name": dir_name,
            })

    return items


def _verify_one_day(
    symbol: str,
    date_str: str,
    *,
    timeout_s: int = 30,
) -> dict:
    """Verify a single day against Vision CHECKSUM.

    Downloads the Vision ZIP and its CHECKSUM file, computes SHA-256 of the
    ZIP bytes, and compares. This verifies the original Vision data -- NOT the
    local Parquet (which is processed/transformed data).

    Returns
    -------
    dict
        One of:
        - ``{"status": "pass", "sha256": hex_hash}`` on match
        - ``{"status": "skip", "reason": "404"|"timeout"}`` on soft-skip
        - ``{"status": "fail", "expected": str, "actual": str}`` on mismatch
    """
    checksum_url = (
        f"{_VISION_SPOT_BASE}/{symbol}/{symbol}-aggTrades-{date_str}.zip.CHECKSUM"
    )
    zip_url = (
        f"{_VISION_SPOT_BASE}/{symbol}/{symbol}-aggTrades-{date_str}.zip"
    )

    # 1. Fetch CHECKSUM file
    try:
        checksum_resp = requests.get(checksum_url, timeout=timeout_s)
    except (requests.Timeout, requests.ConnectionError):
        logger.warning("%s %s: CHECKSUM fetch timed out / network error", symbol, date_str)
        log_checksum_event("sweep_skip", symbol, date_str, source="sweep", reason="timeout")
        return {"status": "skip", "reason": "timeout"}
    except Exception:
        logger.warning("%s %s: CHECKSUM fetch failed", symbol, date_str, exc_info=True)
        log_checksum_event("sweep_skip", symbol, date_str, source="sweep", reason="timeout")
        return {"status": "skip", "reason": "timeout"}

    if checksum_resp.status_code == 404:
        logger.debug("%s %s: no .CHECKSUM file (404) -- soft-skip", symbol, date_str)
        log_checksum_event("sweep_skip", symbol, date_str, source="sweep", reason="404")
        return {"status": "skip", "reason": "404"}

    if checksum_resp.status_code != 200:
        logger.warning(
            "%s %s: CHECKSUM HTTP %d -- soft-skip",
            symbol,
            date_str,
            checksum_resp.status_code,
        )
        log_checksum_event("sweep_skip", symbol, date_str, source="sweep", reason="404")
        return {"status": "skip", "reason": "404"}

    try:
        expected_hash = parse_checksum_file_content(checksum_resp.text)
    except ValueError:
        logger.warning("%s %s: invalid CHECKSUM format -- soft-skip", symbol, date_str)
        log_checksum_event("sweep_skip", symbol, date_str, source="sweep", reason="invalid_format")
        return {"status": "skip", "reason": "invalid_format"}

    # 2. Download ZIP and compute hash
    try:
        zip_resp = requests.get(zip_url, timeout=timeout_s)
    except (requests.Timeout, requests.ConnectionError):
        logger.warning("%s %s: ZIP download timed out / network error", symbol, date_str)
        return {"status": "skip", "reason": "timeout"}
    except Exception:
        logger.warning("%s %s: ZIP download failed", symbol, date_str, exc_info=True)
        return {"status": "skip", "reason": "timeout"}

    if zip_resp.status_code != 200:
        logger.warning(
            "%s %s: ZIP HTTP %d -- soft-skip",
            symbol,
            date_str,
            zip_resp.status_code,
        )
        return {"status": "skip", "reason": "404"}

    actual_hash = compute_sha256_hex(zip_resp.content)

    # 3. Compare
    if actual_hash == expected_hash:
        log_checksum_event("sweep_pass", symbol, date_str, source="sweep")
        return {"status": "pass", "sha256": actual_hash}

    logger.warning(
        "%s %s: CHECKSUM MISMATCH expected=%s actual=%s",
        symbol,
        date_str,
        expected_hash,
        actual_hash,
    )
    log_checksum_event(
        "sweep_fail", symbol, date_str,
        source="sweep",
        expected_hash=expected_hash[:16],
        actual_hash=actual_hash[:16],
    )
    return {"status": "fail", "expected": expected_hash, "actual": actual_hash}


def _aggregate_results(results: list[dict]) -> dict:
    """Aggregate verification results into a summary.

    Parameters
    ----------
    results
        List of result dicts from ``_verify_one_day()``, each augmented with
        ``symbol`` and ``date`` keys.

    Returns
    -------
    dict
        Summary with ``total`` (pass/fail/skip counts), ``by_symbol`` (per-symbol
        counts), and ``failed`` (list of (symbol, date) tuples).
    """
    total = {"pass": 0, "fail": 0, "skip": 0}
    by_symbol: dict[str, dict[str, int]] = {}
    failed: list[tuple[str, str]] = []

    for r in results:
        symbol = r["symbol"]
        status = r["status"]

        total[status] = total.get(status, 0) + 1

        if symbol not in by_symbol:
            by_symbol[symbol] = {"pass": 0, "fail": 0, "skip": 0}
        by_symbol[symbol][status] += 1

        if status == "fail":
            failed.append((symbol, r["date"]))

    return {
        "total": total,
        "by_symbol": by_symbol,
        "failed": failed,
    }


# ---------------------------------------------------------------------------
# CLI + orchestration
# ---------------------------------------------------------------------------


def _format_summary(agg: dict, elapsed_s: float) -> str:
    """Format aggregated results into a human-readable summary."""
    lines = []
    lines.append("=" * 60)
    lines.append("INTEGRITY SWEEP SUMMARY")
    lines.append("=" * 60)
    lines.append("")

    t = agg["total"]
    lines.append(f"  Total:  {t['pass'] + t['fail'] + t['skip']}")
    lines.append(f"  Pass:   {t['pass']}")
    lines.append(f"  Fail:   {t['fail']}")
    lines.append(f"  Skip:   {t['skip']}")
    lines.append(f"  Time:   {elapsed_s:.1f}s")
    lines.append("")

    if agg["by_symbol"]:
        lines.append("Per-Symbol Breakdown:")
        lines.append(f"  {'Symbol':<25s} {'Pass':>6s} {'Fail':>6s} {'Skip':>6s}")
        lines.append("  " + "-" * 49)
        for sym in sorted(agg["by_symbol"]):
            s = agg["by_symbol"][sym]
            lines.append(f"  {sym:<25s} {s['pass']:>6d} {s['fail']:>6d} {s['skip']:>6d}")
        lines.append("")

    if agg["failed"]:
        lines.append("FAILED ITEMS:")
        for symbol, date in agg["failed"]:
            lines.append(f"  {symbol} {date}")
        lines.append("")

    lines.append("=" * 60)
    return "\n".join(lines)


def _format_telegram_summary(agg: dict, elapsed_s: float) -> str:
    """Format aggregated results for Telegram HTML message."""
    t = agg["total"]
    total = t["pass"] + t["fail"] + t["skip"]

    msg_parts = [
        "<b>INTEGRITY SWEEP</b>",
        "",
        f"Total: {total}",
        f"Pass: {t['pass']}  |  Fail: {t['fail']}  |  Skip: {t['skip']}",
        f"Time: {elapsed_s:.1f}s",
    ]

    if agg["failed"]:
        msg_parts.append("")
        msg_parts.append("<b>FAILURES:</b>")
        for symbol, date in agg["failed"]:
            msg_parts.append(f"  {symbol} {date}")

    return "\n".join(msg_parts)


def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
    """Parse CLI arguments."""
    parser = argparse.ArgumentParser(
        description="Audit Parquet cache files against Binance Vision CHECKSUMs.",
    )
    parser.add_argument(
        "--full",
        action="store_true",
        help="Verify all files (ignore registry, re-verify everything)",
    )
    parser.add_argument(
        "--symbol",
        type=str,
        default=None,
        help="Restrict to one symbol (e.g., BTCUSDT)",
    )
    parser.add_argument(
        "--start-date",
        type=str,
        default=None,
        dest="start_date",
        help="Filter start date (YYYY-MM-DD)",
    )
    parser.add_argument(
        "--end-date",
        type=str,
        default=None,
        dest="end_date",
        help="Filter end date (YYYY-MM-DD)",
    )
    parser.add_argument(
        "--workers",
        type=int,
        default=8,
        help="Thread pool size (default: 8)",
    )
    parser.add_argument(
        "--timeout",
        type=int,
        default=30,
        help="HTTP timeout seconds (default: 30)",
    )
    parser.add_argument(
        "--telegram",
        action="store_true",
        help="Send summary to Telegram",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        dest="dry_run",
        help="List work items without downloading",
    )
    return parser.parse_args(argv)


def main(argv: list[str] | None = None) -> int:
    """Entry point for the integrity sweep."""
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s %(levelname)-8s %(message)s",
    )

    args = _parse_args(argv)

    # Initialize storage and registry
    storage = TickStorage()
    ticks_dir = storage.ticks_dir
    registry = ChecksumRegistry()

    logger.info("Discovering work items from %s ...", ticks_dir)
    items = _build_work_items(ticks_dir, registry, args)
    logger.info("Found %d work items", len(items))

    if not items:
        logger.info("Nothing to verify. Exiting.")
        return 0

    # Dry run: just list items
    if args.dry_run:
        print(f"\nDry run: {len(items)} items to verify\n")
        for item in items:
            print(f"  {item['symbol']}  {item['date']}")
        return 0

    # Run verification
    start_time = time.monotonic()
    results: list[dict] = []

    try:
        from tqdm import tqdm

        progress = tqdm(total=len(items), desc="Verifying", unit="day")
    except ImportError:
        progress = None

    with ThreadPoolExecutor(max_workers=args.workers) as executor:
        future_to_item = {
            executor.submit(
                _verify_one_day,
                item["symbol"],
                item["date"],
                timeout_s=args.timeout,
            ): item
            for item in items
        }

        for future in as_completed(future_to_item):
            item = future_to_item[future]
            try:
                result = future.result()
            except Exception:
                logger.warning(
                    "%s %s: unexpected error",
                    item["symbol"],
                    item["date"],
                    exc_info=True,
                )
                result = {"status": "skip", "reason": "error"}

            # Augment result with item metadata
            result["symbol"] = item["symbol"]
            result["date"] = item["date"]
            results.append(result)

            # Record pass in registry
            if result["status"] == "pass":
                registry.add(
                    symbol=item["symbol"],
                    date=item["date"],
                    sha256=result["sha256"],
                    source="sweep",
                )

            if progress is not None:
                progress.update(1)

    if progress is not None:
        progress.close()

    # Save registry (batch save)
    registry.save()

    elapsed_s = time.monotonic() - start_time

    # Aggregate and print summary
    agg = _aggregate_results(results)
    summary_text = _format_summary(agg, elapsed_s)
    print(summary_text)

    # Telegram notification
    if args.telegram:
        try:
            from opendeviationbar.notify.telegram import send_telegram

            tg_msg = _format_telegram_summary(agg, elapsed_s)
            is_failure = agg["total"]["fail"] > 0
            send_telegram(tg_msg, disable_notification=not is_failure)
            logger.info("Telegram summary sent")
        except Exception:
            logger.warning("Failed to send Telegram summary", exc_info=True)

    # Exit code: 0 if no failures, 1 if any failures
    if agg["total"]["fail"] > 0:
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main())
