#!/usr/bin/env python3
"""Claude Code Usage Dashboard — FastAPI Backend (SQLite-backed)"""
from __future__ import annotations

import json
import logging
import os
import re
import sqlite3
import threading
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Optional

from fastapi import FastAPI, HTTPException, Query, Request
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse, JSONResponse
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.responses import Response

app = FastAPI(title="Claude Code Usage Dashboard")
log = logging.getLogger("myburn")
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")


# ---------------------------------------------------------------------------
# Security headers middleware (FIX-05)
# ---------------------------------------------------------------------------
class _SecurityHeadersMiddleware(BaseHTTPMiddleware):
    async def dispatch(self, request: Request, call_next):
        response: Response = await call_next(request)
        response.headers["X-Content-Type-Options"] = "nosniff"
        response.headers["X-Frame-Options"] = "DENY"
        response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
        response.headers["Content-Security-Policy"] = (
            "default-src 'self'; "
            "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; "
            "font-src 'self' https://fonts.gstatic.com; "
            "img-src 'self' https://avatars.githubusercontent.com data:; "
            "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; "
            "connect-src 'self'"
        )
        return response


app.add_middleware(_SecurityHeadersMiddleware)


# ---------------------------------------------------------------------------
# Rate limiting (FIX-03) — in-memory, per-IP, no external deps
# ---------------------------------------------------------------------------
_rate_buckets: dict[str, list[float]] = {}
_rate_lock = threading.Lock()

# path prefix → (max_requests, window_seconds)
_RATE_LIMITS: dict[str, tuple[int, int]] = {
    "/api/auth/device/": (20, 60),    # device flow poll: 20 req/min (polls every 5s)
    "/api/auth/":       (5, 60),      # auth: 5 req/min
    "/api/push":        (10, 60),     # push: 10 req/min
    "/api/cache/clear": (2, 60),      # sync trigger: 2/min
    "/leaderboard":     (30, 60),     # leaderboard: 30/min
}
_RATE_DEFAULT = (60, 60)  # everything else: 60 req/min


def _get_client_ip(request: Request) -> str:
    """Get real client IP, respecting X-Real-IP from reverse proxy."""
    real_ip = request.headers.get("X-Real-IP")
    if real_ip:
        return real_ip
    return request.client.host if request.client else "unknown"


class _RateLimitMiddleware(BaseHTTPMiddleware):
    async def dispatch(self, request: Request, call_next):
        client_ip = _get_client_ip(request)
        path = request.url.path

        # Find matching limit
        max_req, window = _RATE_DEFAULT
        for prefix, limits in _RATE_LIMITS.items():
            if path.startswith(prefix):
                max_req, window = limits
                break

        bucket_key = f"{client_ip}:{path.split('?')[0]}"
        now = time.time()

        with _rate_lock:
            hits = _rate_buckets.get(bucket_key, [])
            # Prune old entries
            hits = [t for t in hits if now - t < window]
            if len(hits) >= max_req:
                return JSONResponse(
                    {"detail": "Too many requests"},
                    status_code=429,
                    headers={"Retry-After": str(window)},
                )
            hits.append(now)
            _rate_buckets[bucket_key] = hits

            # Periodic cleanup: if dict grows big, prune stale keys
            if len(_rate_buckets) > 10000:
                stale = [k for k, v in _rate_buckets.items() if not v or now - v[-1] > 300]
                for k in stale:
                    del _rate_buckets[k]

        return await call_next(request)


app.add_middleware(_RateLimitMiddleware)

# ---------------------------------------------------------------------------
# SQLite setup
# ---------------------------------------------------------------------------
DATA_DIR = Path(os.environ.get("MYBURN_DATA_DIR", str(Path.home() / ".myburn" / "data")))
DATA_DIR.mkdir(parents=True, exist_ok=True)
DB_PATH = DATA_DIR / "usage.db"

_db_ready = threading.Event()
_shutdown_event = threading.Event()
_sync_thread: Optional[threading.Thread] = None
_sync_degraded = False
_PID_PATH = DATA_DIR / "myburn.pid"


# ---------------------------------------------------------------------------
# Response cache — TTL-based, invalidated on sync (Operacao BUNKER M.1)
# ---------------------------------------------------------------------------
_CACHE_TTL = 45  # seconds
_cache: dict[str, tuple[float, object]] = {}
_cache_lock = threading.Lock()


def _cache_get(key: str):
    """Return cached value if fresh, else None."""
    with _cache_lock:
        entry = _cache.get(key)
        if entry and (time.time() - entry[0]) < _CACHE_TTL:
            return entry[1]
    return None


def _cache_set(key: str, value: object):
    """Store value in cache with current timestamp."""
    with _cache_lock:
        _cache[key] = (time.time(), value)


def _cache_invalidate():
    """Clear all cached responses (called after sync)."""
    with _cache_lock:
        _cache.clear()


def _acquire_instance_lock():
    """Ensure only one server instance runs against this data directory."""
    if _PID_PATH.exists():
        try:
            old_pid = int(_PID_PATH.read_text(encoding="utf-8").strip())
            os.kill(old_pid, 0)  # Check if process is alive
            log.error(f"Another instance is running (PID {old_pid}). Exiting.")
            raise SystemExit(1)
        except (ValueError, ProcessLookupError, OSError):
            # OSError covers Windows (no signal 0 support) + other stale PID scenarios
            log.warning("Removing stale PID file (PID no longer running)")
        except PermissionError:
            log.error("Another instance is running (PID file exists, access denied). Exiting.")
            raise SystemExit(1)
    _PID_PATH.write_text(str(os.getpid()), encoding="utf-8")


def _release_instance_lock():
    """Remove PID file on shutdown."""
    try:
        if _PID_PATH.exists() and _PID_PATH.read_text(encoding="utf-8").strip() == str(os.getpid()):
            _PID_PATH.unlink()
    except Exception:
        pass


def _get_db() -> sqlite3.Connection:
    conn = sqlite3.connect(str(DB_PATH), timeout=10)
    result = conn.execute("PRAGMA journal_mode=WAL").fetchone()
    if result and result[0] != "wal":
        log.error(
            "SQLite WAL mode failed (got '%s'). Your storage may not support it. "
            "Use a local filesystem for MYBURN_DATA_DIR (NFS/SMB volumes don't support WAL).",
            result[0],
        )
    conn.execute("PRAGMA busy_timeout=5000")
    conn.row_factory = sqlite3.Row
    return conn


def _migrate_drop_ccusage():
    """One-time migration: rename native shadow tables to primary, drop ccusage artifacts."""
    if not DB_PATH.exists():
        return
    conn = _get_db()
    try:
        # Check if old native_daily_usage shadow table exists (migration needed)
        tables = [r[0] for r in conn.execute(
            "SELECT name FROM sqlite_master WHERE type='table'"
        ).fetchall()]
        if "native_daily_usage" not in tables:
            return  # Already migrated or fresh DB

        log.info("Migrating: dropping ccusage tables, renaming native → primary")

        # Atomic migration — all-or-nothing (FIX-17)
        conn.isolation_level = None  # explicit transaction control for DDL
        conn.execute("BEGIN IMMEDIATE")
        try:
            conn.execute("DROP TABLE IF EXISTS blocks")
            conn.execute("DROP TABLE IF EXISTS comparison_log")

            if "daily_usage" in tables:
                conn.execute("DROP TABLE daily_usage")
            if "sessions" in tables:
                conn.execute("DROP TABLE sessions")

            conn.execute("ALTER TABLE native_daily_usage RENAME TO daily_usage")
            conn.execute("ALTER TABLE native_sessions RENAME TO sessions")

            conn.execute("DELETE FROM sync_meta WHERE key IN ('initial_sync_done', 'last_sync')")
            conn.execute("COMMIT")
        except Exception:
            conn.execute("ROLLBACK")
            raise
        log.info("Migration complete: native tables are now primary")
    finally:
        conn.close()


def _init_db():
    conn = _get_db()
    conn.executescript("""
        CREATE TABLE IF NOT EXISTS daily_usage (
            date TEXT NOT NULL,
            project TEXT NOT NULL,
            model_name TEXT NOT NULL,
            input_tokens INTEGER DEFAULT 0,
            output_tokens INTEGER DEFAULT 0,
            cache_creation_tokens INTEGER DEFAULT 0,
            cache_read_tokens INTEGER DEFAULT 0,
            cost REAL DEFAULT 0,
            PRIMARY KEY (date, project, model_name)
        );

        CREATE TABLE IF NOT EXISTS sessions (
            session_id TEXT NOT NULL,
            project_path TEXT NOT NULL,
            input_tokens INTEGER DEFAULT 0,
            output_tokens INTEGER DEFAULT 0,
            cache_creation_tokens INTEGER DEFAULT 0,
            cache_read_tokens INTEGER DEFAULT 0,
            total_tokens INTEGER DEFAULT 0,
            total_cost REAL DEFAULT 0,
            last_activity TEXT,
            models_used TEXT,
            model_breakdowns TEXT,
            PRIMARY KEY (session_id, project_path)
        );

        -- NOTE: PK (timestamp, model) can collide when two API calls for the same
        -- model land in the same millisecond. INSERT OR IGNORE silently drops the
        -- second entry. Accepted trade-off: adding rowid PK would require rewriting
        -- the entire sync pipeline. Collision probability is very low in practice. (H8)
        CREATE TABLE IF NOT EXISTS live_entries (
            timestamp TEXT NOT NULL,
            model TEXT NOT NULL,
            input_tokens INTEGER DEFAULT 0,
            output_tokens INTEGER DEFAULT 0,
            cache_read_tokens INTEGER DEFAULT 0,
            cache_creation_tokens INTEGER DEFAULT 0,
            cost REAL DEFAULT 0,
            project TEXT DEFAULT '',
            PRIMARY KEY (timestamp, model)
        );
        CREATE INDEX IF NOT EXISTS idx_live_entries_ts ON live_entries(timestamp);

        CREATE TABLE IF NOT EXISTS sync_meta (
            key TEXT PRIMARY KEY,
            value TEXT
        );

        -- Turn-level metrics (Pass 2: YOLO mode, tools, duration)
        CREATE TABLE IF NOT EXISTS native_turns (
            session_file TEXT NOT NULL,
            turn_index INTEGER NOT NULL,
            timestamp TEXT NOT NULL,
            model TEXT,
            permission_mode TEXT DEFAULT 'default',
            duration_ms INTEGER,
            input_tokens INTEGER DEFAULT 0,
            output_tokens INTEGER DEFAULT 0,
            cache_read_tokens INTEGER DEFAULT 0,
            cache_creation_tokens INTEGER DEFAULT 0,
            cost REAL DEFAULT 0,
            has_thinking BOOLEAN DEFAULT 0,
            tool_count INTEGER DEFAULT 0,
            tools_used TEXT DEFAULT '[]',
            is_error BOOLEAN DEFAULT 0,
            is_interrupted BOOLEAN DEFAULT 0,
            project TEXT,
            is_subagent BOOLEAN DEFAULT 0,
            PRIMARY KEY (session_file, turn_index)
        );
        CREATE INDEX IF NOT EXISTS idx_native_turns_project ON native_turns(project);
        CREATE INDEX IF NOT EXISTS idx_native_turns_ts ON native_turns(timestamp);
    """)
    # Migrations: add columns to existing live_entries tables
    for col, default in [("project", "''"), ("session_id", "''")]:
        try:
            conn.execute(f"ALTER TABLE live_entries ADD COLUMN {col} TEXT DEFAULT {default}")
        except sqlite3.OperationalError:
            pass  # column already exists
    conn.commit()
    conn.close()


def _wal_checkpoint():
    """Force WAL checkpoint so data actually hits the DB file on disk."""
    conn = _get_db()
    try:
        conn.execute("PRAGMA wal_checkpoint(PASSIVE)")
    finally:
        conn.close()


def _sync_loop():
    """Background sync loop — native parser only."""
    global _sync_degraded

    # Initial sync — always set _db_ready in finally (FIX-11)
    try:
        try:
            _native_sync(full=True)
        except Exception as e:
            log.error(f"Initial native sync failed: {e}")
            _sync_degraded = True

        try:
            _recalculate_live_entry_costs()
        except Exception as e:
            log.error(f"Cost recalculation failed: {e}")

        try:
            _wal_checkpoint()
        except Exception as e:
            log.error(f"WAL checkpoint failed: {e}")
    finally:
        _cache_invalidate()
        _db_ready.set()
        if _sync_degraded:
            log.warning("DB marked ready with DEGRADED data — initial sync failed")

    # Periodic loop — interruptible via _shutdown_event (FIX-10)
    cycle = 0
    while not _shutdown_event.is_set():
        _shutdown_event.wait(timeout=300)
        if _shutdown_event.is_set():
            break
        cycle += 1
        try:
            do_full = (cycle % 12 == 0)
            _native_sync(full=do_full)
            _cache_invalidate()
            if do_full:
                log.info("Periodic full sync completed (hourly self-heal)")
        except Exception as e:
            log.error(f"Periodic native sync failed: {e}")
        try:
            _cleanup_old_live_entries()
        except Exception as e:
            log.error(f"Live entries cleanup failed: {e}")
        try:
            _wal_checkpoint()
        except Exception as e:
            log.error(f"WAL checkpoint failed: {e}")
        try:
            from myburn.push import check_auto_push
            check_auto_push()
        except Exception as e:
            log.error(f"Auto-push check failed: {e}")
    log.info("Sync loop exiting (shutdown signalled)")


# ---------------------------------------------------------------------------
# Query helpers — reconstruct exact ccusage JSON shapes
# ---------------------------------------------------------------------------
def _aggregate_rows_to_period(rows, period_key: str) -> list[dict]:
    """
    Aggregate daily_usage rows into period objects with modelBreakdowns.
    rows must have: period, model_name, input_tokens, output_tokens,
                    cache_creation_tokens, cache_read_tokens, cost
    """
    periods: dict[str, dict] = {}
    for r in rows:
        p = r["period"]
        if p not in periods:
            periods[p] = {"models": {}, "totals": {
                "inputTokens": 0, "outputTokens": 0,
                "cacheCreationTokens": 0, "cacheReadTokens": 0,
                "totalTokens": 0, "totalCost": 0,
            }}
        entry = periods[p]
        t = entry["totals"]
        t["inputTokens"] += r["input_tokens"]
        t["outputTokens"] += r["output_tokens"]
        t["cacheCreationTokens"] += r["cache_creation_tokens"]
        t["cacheReadTokens"] += r["cache_read_tokens"]
        t["totalCost"] += r["cost"]
        t["totalTokens"] += (
            r["input_tokens"] + r["output_tokens"]
            + r["cache_creation_tokens"] + r["cache_read_tokens"]
        )

        mn = r["model_name"]
        if mn not in entry["models"]:
            entry["models"][mn] = {
                "modelName": mn, "inputTokens": 0, "outputTokens": 0,
                "cacheCreationTokens": 0, "cacheReadTokens": 0, "cost": 0,
            }
        m = entry["models"][mn]
        m["inputTokens"] += r["input_tokens"]
        m["outputTokens"] += r["output_tokens"]
        m["cacheCreationTokens"] += r["cache_creation_tokens"]
        m["cacheReadTokens"] += r["cache_read_tokens"]
        m["cost"] += r["cost"]

    result = []
    for p in sorted(periods.keys()):
        entry = periods[p]
        breakdowns = list(entry["models"].values())
        models_used = [mb["modelName"] for mb in breakdowns]
        t = entry["totals"]
        obj = {
            period_key: p,
            "inputTokens": t["inputTokens"],
            "outputTokens": t["outputTokens"],
            "cacheCreationTokens": t["cacheCreationTokens"],
            "cacheReadTokens": t["cacheReadTokens"],
            "totalTokens": t["totalTokens"],
            "totalCost": t["totalCost"],
            "modelsUsed": models_used,
            "modelBreakdowns": breakdowns,
        }
        result.append(obj)
    return result


def _compute_totals(items: list[dict]) -> dict:
    """Compute totals from a list of period objects."""
    totals = {
        "inputTokens": 0, "outputTokens": 0,
        "cacheCreationTokens": 0, "cacheReadTokens": 0,
        "totalCost": 0, "totalTokens": 0,
    }
    for item in items:
        totals["inputTokens"] += item.get("inputTokens", 0)
        totals["outputTokens"] += item.get("outputTokens", 0)
        totals["cacheCreationTokens"] += item.get("cacheCreationTokens", 0)
        totals["cacheReadTokens"] += item.get("cacheReadTokens", 0)
        totals["totalCost"] += item.get("totalCost", 0)
        totals["totalTokens"] += item.get("totalTokens", 0)
    return totals


def _query_daily(conn: sqlite3.Connection, since: str = None, until: str = None, project: str = None) -> dict:
    """Query daily_usage for aggregate daily data."""
    # FIX-30: honor project param (defaults to __all__ for aggregated view)
    proj = project if project else "__all__"
    sql = """
        SELECT date AS period, model_name, input_tokens, output_tokens,
               cache_creation_tokens, cache_read_tokens, cost
        FROM daily_usage WHERE project = ?
    """
    params = [proj]
    if since:
        sql += " AND date >= ?"
        params.append(_normalize_date(since))
    if until:
        sql += " AND date <= ?"
        params.append(_normalize_date(until))
    sql += " ORDER BY date"
    rows = conn.execute(sql, params).fetchall()
    items = _aggregate_rows_to_period(rows, "date")
    return {"daily": items, "totals": _compute_totals(items)}


def _query_monthly(conn: sqlite3.Connection) -> dict:
    """Aggregate daily_usage by month."""
    sql = """
        SELECT substr(date, 1, 7) AS period, model_name,
               SUM(input_tokens) AS input_tokens, SUM(output_tokens) AS output_tokens,
               SUM(cache_creation_tokens) AS cache_creation_tokens,
               SUM(cache_read_tokens) AS cache_read_tokens, SUM(cost) AS cost
        FROM daily_usage WHERE project = '__all__'
        GROUP BY period, model_name
        ORDER BY period
    """
    rows = conn.execute(sql).fetchall()
    items = _aggregate_rows_to_period(rows, "month")
    return {"monthly": items, "totals": _compute_totals(items)}


def _query_weekly(conn: sqlite3.Connection, since: str = None, until: str = None) -> dict:
    """Aggregate daily_usage by week start (Sunday, matching ccusage)."""
    sql = """
        SELECT date(date, '-' || strftime('%w', date) || ' days') AS period, model_name,
               SUM(input_tokens) AS input_tokens, SUM(output_tokens) AS output_tokens,
               SUM(cache_creation_tokens) AS cache_creation_tokens,
               SUM(cache_read_tokens) AS cache_read_tokens, SUM(cost) AS cost
        FROM daily_usage WHERE project = '__all__'
    """
    params = []
    if since:
        sql += " AND date >= ?"
        params.append(_normalize_date(since))
    if until:
        sql += " AND date <= ?"
        params.append(_normalize_date(until))
    sql += " GROUP BY period, model_name ORDER BY period"
    rows = conn.execute(sql, params).fetchall()
    items = _aggregate_rows_to_period(rows, "week")
    return {"weekly": items, "totals": _compute_totals(items)}


def _query_sessions(conn: sqlite3.Connection) -> dict:
    """Return all sessions."""
    rows = conn.execute(
        "SELECT * FROM sessions ORDER BY total_cost DESC"
    ).fetchall()
    sessions = []
    for r in rows:
        sessions.append({
            "sessionId": r["session_id"],
            "inputTokens": r["input_tokens"],
            "outputTokens": r["output_tokens"],
            "cacheCreationTokens": r["cache_creation_tokens"],
            "cacheReadTokens": r["cache_read_tokens"],
            "totalTokens": r["total_tokens"],
            "totalCost": r["total_cost"],
            "lastActivity": r["last_activity"],
            "modelsUsed": json.loads(r["models_used"]) if r["models_used"] else [],
            "modelBreakdowns": json.loads(r["model_breakdowns"]) if r["model_breakdowns"] else [],
            "projectPath": r["project_path"],
        })
    totals = {
        "inputTokens": sum(s["inputTokens"] for s in sessions),
        "outputTokens": sum(s["outputTokens"] for s in sessions),
        "cacheCreationTokens": sum(s["cacheCreationTokens"] for s in sessions),
        "cacheReadTokens": sum(s["cacheReadTokens"] for s in sessions),
        "totalCost": sum(s["totalCost"] for s in sessions),
        "totalTokens": sum(s["totalTokens"] for s in sessions),
    }
    return {"sessions": sessions, "totals": totals}


def _query_projects(conn: sqlite3.Connection, since: str = None, until: str = None, project: str = None) -> dict:
    """Return per-project daily data grouped by project name."""
    sql = """
        SELECT project, date AS period, model_name, input_tokens, output_tokens,
               cache_creation_tokens, cache_read_tokens, cost
        FROM daily_usage WHERE project != '__all__'
    """
    params = []
    if since:
        sql += " AND date >= ?"
        params.append(_normalize_date(since))
    if until:
        sql += " AND date <= ?"
        params.append(_normalize_date(until))
    if project:
        sql += " AND project = ?"
        params.append(project)
    sql += " ORDER BY project, date"
    rows = conn.execute(sql, params).fetchall()

    # Group by project, then aggregate each project's daily data
    project_rows: dict[str, list] = {}
    for r in rows:
        p = r["project"]
        if p not in project_rows:
            project_rows[p] = []
        project_rows[p].append(r)

    projects = {}
    all_items = []
    for proj_name, proj_rows in sorted(project_rows.items()):
        items = _aggregate_rows_to_period(proj_rows, "date")
        projects[proj_name] = items
        all_items.extend(items)

    return {"projects": projects, "totals": _compute_totals(all_items)}



def _normalize_date(d: str) -> str:
    """Convert YYYYMMDD to YYYY-MM-DD if needed."""
    if d and len(d) == 8 and d.isdigit():
        return f"{d[:4]}-{d[4:6]}-{d[6:8]}"
    return d


# ---------------------------------------------------------------------------
# API Endpoints
# ---------------------------------------------------------------------------
def _wait_for_db():
    """Wait for initial sync to complete. Returns True if ready."""
    return _db_ready.wait(timeout=120)


@app.get("/api/daily")
def api_daily(
    since: Optional[str] = Query(None),
    until: Optional[str] = Query(None),
    project: Optional[str] = Query(None),
):
    if not _wait_for_db():
        return JSONResponse({"error": "Database not ready"}, status_code=503)
    ck = f"daily:{since}:{until}:{project}"
    cached = _cache_get(ck)
    if cached is not None:
        return JSONResponse(cached)
    conn = _get_db()
    try:
        result = _query_daily(conn, since, until, project)
        _cache_set(ck, result)
        return JSONResponse(result)
    finally:
        conn.close()


@app.get("/api/monthly")
def api_monthly():
    if not _wait_for_db():
        return JSONResponse({"error": "Database not ready"}, status_code=503)
    cached = _cache_get("monthly")
    if cached is not None:
        return JSONResponse(cached)
    conn = _get_db()
    try:
        result = _query_monthly(conn)
        _cache_set("monthly", result)
        return JSONResponse(result)
    finally:
        conn.close()


@app.get("/api/weekly")
def api_weekly(
    since: Optional[str] = Query(None),
    until: Optional[str] = Query(None),
):
    if not _wait_for_db():
        return JSONResponse({"error": "Database not ready"}, status_code=503)
    ck = f"weekly:{since}:{until}"
    cached = _cache_get(ck)
    if cached is not None:
        return JSONResponse(cached)
    conn = _get_db()
    try:
        result = _query_weekly(conn, since, until)
        _cache_set(ck, result)
        return JSONResponse(result)
    finally:
        conn.close()


@app.get("/api/sessions")
def api_sessions():
    if not _wait_for_db():
        return JSONResponse({"error": "Database not ready"}, status_code=503)
    cached = _cache_get("sessions")
    if cached is not None:
        return JSONResponse(cached)
    conn = _get_db()
    try:
        result = _query_sessions(conn)
        _cache_set("sessions", result)
        return JSONResponse(result)
    finally:
        conn.close()


@app.get("/api/projects")
def api_projects(
    since: Optional[str] = Query(None),
    until: Optional[str] = Query(None),
    project: Optional[str] = Query(None),
):
    if not _wait_for_db():
        return JSONResponse({"error": "Database not ready"}, status_code=503)
    ck = f"projects:{since}:{until}:{project}"
    cached = _cache_get(ck)
    if cached is not None:
        return JSONResponse(cached)
    conn = _get_db()
    try:
        result = _query_projects(conn, since, until, project)
        _cache_set(ck, result)
        return JSONResponse(result)
    finally:
        conn.close()



@app.get("/api/analytics")
def api_analytics():
    if not _wait_for_db():
        return JSONResponse({"error": "Database not ready"}, status_code=503)
    cached = _cache_get("analytics")
    if cached is not None:
        return JSONResponse(cached)
    conn = _get_db()
    try:
        monthly = _query_monthly(conn)
        daily = _query_daily(conn)
        sessions = _query_sessions(conn)
        weekly = _query_weekly(conn)
        total_sessions_count = conn.execute(
            "SELECT COUNT(DISTINCT session_id) FROM live_entries WHERE session_id != ''"
        ).fetchone()[0] or 0
    finally:
        conn.close()

    analytics = {}

    # Monthly totals
    totals = monthly.get("totals", {})
    total_cost = totals.get("totalCost", 0)
    total_tokens = totals.get("totalTokens", 0)
    cache_read = totals.get("cacheReadTokens", 0)
    cache_creation = totals.get("cacheCreationTokens", 0)
    input_tokens = totals.get("inputTokens", 0)
    output_tokens = totals.get("outputTokens", 0)

    # Compute cache savings using per-model pricing instead of hardcoded Opus (FIX-32)
    cache_savings = 0.0
    without_caching = 0.0
    for item in monthly.get("monthly", []):
        for mb in item.get("modelBreakdowns", []):
            p = PRICING.get(mb["modelName"], DEFAULT_PRICING)
            cr = mb.get("cacheReadTokens", 0)
            cc = mb.get("cacheCreationTokens", 0)
            inp = mb.get("inputTokens", 0)
            out = mb.get("outputTokens", 0)
            # Savings = what cache_read would have cost at full input price minus actual cache_read price
            cache_savings += cr * (p["input"] - p["cache_read"]) / 1_000_000
            # Without caching = all tokens at full (non-cache) prices
            without_caching += (
                (cr + inp) * p["input"] / 1_000_000
                + cc * p["cache_create"] / 1_000_000
                + out * p["output"] / 1_000_000
            )
    with_caching = total_cost
    analytics["cacheSavings"] = cache_savings
    analytics["withoutCaching"] = without_caching
    analytics["withCaching"] = with_caching

    now = datetime.now()
    days_in_month = (
        (datetime(now.year, now.month % 12 + 1, 1) - timedelta(days=1)).day
        if now.month < 12
        else 31
    )
    days_elapsed = now.day
    days_remaining = days_in_month - days_elapsed

    analytics["daysActive"] = days_elapsed
    analytics["daysRemaining"] = days_remaining
    analytics["daysInMonth"] = days_in_month

    daily_items = daily.get("daily", [])
    daily_costs = [d.get("totalCost", 0) for d in daily_items]
    active_costs = [c for c in daily_costs if c > 0]

    daily_avg = sum(active_costs) / max(len(active_costs), 1)
    analytics["dailyAvgCost"] = daily_avg

    # --- 3 projection modes ---
    last_day_cost = daily_costs[-1] if daily_costs else 0
    analytics["projLastDay"] = last_day_cost * 30

    last7 = daily_costs[-7:] if len(daily_costs) >= 7 else daily_costs
    last7_avg = sum(last7) / max(len(last7), 1)
    analytics["projLastWeek"] = last7_avg * 30

    if active_costs:
        decay = 0.85
        ewma = active_costs[-1]
        for c in reversed(active_costs[:-1]):
            ewma = decay * ewma + (1 - decay) * c
        sorted_costs = sorted(active_costs)
        mid = len(sorted_costs) // 2
        median = (
            sorted_costs[mid]
            if len(sorted_costs) % 2
            else (sorted_costs[mid - 1] + sorted_costs[mid]) / 2
        )
        smart_avg = 0.6 * ewma + 0.2 * median + 0.2 * daily_avg
    else:
        smart_avg = 0

    analytics["projSmart"] = smart_avg * 30
    analytics["smartDailyAvg"] = smart_avg
    analytics["lastDayDailyAvg"] = last_day_cost
    analytics["lastWeekDailyAvg"] = last7_avg
    analytics["projectedMonthlyCost"] = analytics["projSmart"]

    session_list = sessions.get("sessions", [])

    # Count real Claude Code sessions (distinct session_ids), not projects
    analytics["totalSessions"] = total_sessions_count

    unique_projects = set()
    for s in session_list:
        sid = s.get("sessionId", "")
        if sid and "subagent" not in sid.lower():
            unique_projects.add(sid)
    analytics["uniqueProjects"] = len(unique_projects)

    if daily_items:
        peak = max(daily_items, key=lambda d: d.get("totalCost", 0))
        analytics["peakDay"] = {"date": peak.get("date", ""), "cost": peak.get("totalCost", 0)}
    else:
        analytics["peakDay"] = {"date": "", "cost": 0}

    analytics["tokensPerDollar"] = output_tokens / max(total_cost, 0.01)

    sparkline = []
    if daily_items:
        last7 = daily_items[-7:]
        sparkline = [d.get("totalCost", 0) for d in last7]
    analytics["costSparkline"] = sparkline

    weekly_items = weekly.get("weekly", [])
    if len(weekly_items) >= 2:
        this_week = weekly_items[-1].get("totalCost", 0)
        last_week = weekly_items[-2].get("totalCost", 0)
        if last_week > 0:
            analytics["weekOverWeekDelta"] = ((this_week - last_week) / last_week) * 100
        else:
            analytics["weekOverWeekDelta"] = 0
        analytics["thisWeekCost"] = this_week
        analytics["lastWeekCost"] = last_week
    else:
        analytics["weekOverWeekDelta"] = 0
        analytics["thisWeekCost"] = 0
        analytics["lastWeekCost"] = 0

    model_economics = []
    monthly_items = monthly.get("monthly", [])
    if monthly_items:
        breakdowns = monthly_items[0].get("modelBreakdowns", [])
        for mb in breakdowns:
            cost = mb.get("cost", 0)
            total_t = (
                mb.get("cacheReadTokens", 0)
                + mb.get("cacheCreationTokens", 0)
                + mb.get("inputTokens", 0)
                + mb.get("outputTokens", 0)
            )
            out_t = mb.get("outputTokens", 0)
            model_economics.append({
                "modelName": mb.get("modelName", ""),
                "cost": cost,
                "totalTokens": total_t,
                "outputTokens": out_t,
                "costPerMillionTokens": cost / max(total_t / 1_000_000, 0.001),
                "outputTokensPerDollar": out_t / max(cost, 0.01),
            })
    analytics["modelEconomics"] = model_economics

    _cache_set("analytics", analytics)
    return JSONResponse(analytics)


# --- Live endpoint: reads JSONL files directly (unchanged) ---

CLAUDE_DIR = Path.home() / ".claude" / "projects"

PRICING = {
    # Opus 4.6 (current) — $5/$25 per MTok
    "claude-opus-4-6": {"input": 5.0, "output": 25.0, "cache_read": 0.50, "cache_create": 6.25},
    # Opus 4.5 — same pricing as 4.6
    "claude-opus-45": {"input": 5.0, "output": 25.0, "cache_read": 0.50, "cache_create": 6.25},
    # Opus 4.0/4.1 — legacy pricing
    "claude-opus-4-20250514": {"input": 15.0, "output": 75.0, "cache_read": 1.50, "cache_create": 18.75},
    # Sonnet 4.5/4.6 — $3/$15 per MTok
    "claude-sonnet-4-5-20250514": {"input": 3.0, "output": 15.0, "cache_read": 0.30, "cache_create": 3.75},
    "claude-sonnet-4-5-20250929": {"input": 3.0, "output": 15.0, "cache_read": 0.30, "cache_create": 3.75},
    "claude-sonnet-4-6": {"input": 3.0, "output": 15.0, "cache_read": 0.30, "cache_create": 3.75},
    # Haiku 4.5 — $1/$5 per MTok
    "claude-haiku-4-5-20251001": {"input": 1.0, "output": 5.0, "cache_read": 0.10, "cache_create": 1.25},
}
DEFAULT_PRICING = {"input": 5.0, "output": 25.0, "cache_read": 0.50, "cache_create": 6.25}


def _estimate_cost(model: str, usage: dict) -> float:
    p = PRICING.get(model, DEFAULT_PRICING)
    inp = usage.get("input_tokens", 0)
    out = usage.get("output_tokens", 0)
    cr = usage.get("cache_read_input_tokens", 0)
    cc = usage.get("cache_creation_input_tokens", 0)
    return (inp * p["input"] + out * p["output"] + cr * p["cache_read"] + cc * p["cache_create"]) / 1_000_000


def _scan_jsonl_entries(full: bool = False) -> list[dict]:
    """Pass 1 scanner: reads JSONL files and returns deduplicated entry dicts.
    NOTE: Pass 1 and Pass 2 (_scan_jsonl_turns) read the same files independently.
    Merging them into a single pass would halve I/O but requires a large refactor
    of the sync pipeline. Deferred to v2. (L12)
    full=True  → scans all JSONL files up to 30 days
    full=False → scans only files modified in last 2h

    Dedup strategy: uses message.id:requestId as a unique key per API call,
    exactly like ccusage does. Claude Code writes one JSONL line per content
    block (thinking, text, tool_use) but all blocks from the same API response
    share the same message.id and requestId. We keep only the first occurrence
    and skip duplicates globally across all files (including subagent files)."""
    lookback = timedelta(days=30) if full else timedelta(hours=2)
    cutoff = datetime.now(timezone.utc) - lookback
    cutoff_ts = cutoff.timestamp()
    entries = []
    seen_hashes: set[str] = set()  # global dedup across all files

    if not CLAUDE_DIR.exists():
        return []

    jsonl_files = []  # (path, project_name, session_id)
    for project_dir in CLAUDE_DIR.iterdir():
        if not project_dir.is_dir():
            continue
        proj_name = project_dir.name
        for f in project_dir.rglob("*.jsonl"):
            try:
                if f.stat().st_mtime > cutoff_ts:
                    rel = f.relative_to(project_dir)
                    session_id = rel.parts[0].replace(".jsonl", "") if rel.parts else ""
                    jsonl_files.append((f, proj_name, session_id))
            except OSError:
                continue

    for f, proj_name, session_id in jsonl_files:
        try:
            with open(f, "r", encoding="utf-8", errors="replace") as fh:
                for raw_line in fh:
                    # Partial line (no trailing \n) = file being written, skip (FIX-13)
                    if not raw_line.endswith("\n"):
                        log.debug("Skipping partial line in %s (still being written)", f.name)
                        continue
                    line = raw_line.strip()
                    if not line:
                        continue
                    try:
                        entry = json.loads(line)
                    except json.JSONDecodeError:
                        continue
                    if entry.get("type") != "assistant":
                        continue
                    ts_str = entry.get("timestamp", "")
                    if not ts_str:
                        continue
                    try:
                        ts = datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
                        if ts.timestamp() < cutoff_ts:
                            continue
                    except (ValueError, OSError):
                        continue
                    msg = entry.get("message", {})
                    usage = msg.get("usage", {})
                    if not usage:
                        continue
                    model = msg.get("model", "unknown")
                    if model.startswith("<") or model == "unknown":
                        continue
                    # Dedup: message.id:requestId, fallback to msg_id alone (FIX-16)
                    msg_id = msg.get("id", "")
                    req_id = entry.get("requestId", "")
                    if msg_id:
                        dedup_key = f"{msg_id}:{req_id}" if req_id else msg_id
                        if dedup_key in seen_hashes:
                            continue
                        seen_hashes.add(dedup_key)
                    inp = usage.get("input_tokens", 0)
                    out = usage.get("output_tokens", 0)
                    cr = usage.get("cache_read_input_tokens", 0)
                    cc = usage.get("cache_creation_input_tokens", 0)
                    cost = _estimate_cost(model, usage)
                    entries.append({
                        "timestamp": ts_str,
                        "model": model,
                        "input_tokens": inp,
                        "output_tokens": out,
                        "cache_read_tokens": cr,
                        "cache_creation_tokens": cc,
                        "cost": cost,
                        "project": proj_name,
                        "session_id": session_id,
                    })
        except Exception:
            continue

    return entries


def _scan_jsonl_turns(full: bool = False) -> list[dict]:
    """Pass 2 scanner: reads ALL entry types from JSONL files and extracts turn-level metrics.
    A turn = one user→assistant cycle. Tracks permission mode state, merges streaming
    content blocks, extracts tool names, links turn_duration entries.

    Safety: extracts field names/types/counts only — NEVER content/text/stdout."""
    lookback = timedelta(days=30) if full else timedelta(hours=2)
    cutoff = datetime.now(timezone.utc) - lookback
    cutoff_ts = cutoff.timestamp()

    if not CLAUDE_DIR.exists():
        return []

    # Discover JSONL files (same logic as Pass 1)
    jsonl_files = []
    for project_dir in CLAUDE_DIR.iterdir():
        if not project_dir.is_dir():
            continue
        proj_name = project_dir.name
        for f in project_dir.rglob("*.jsonl"):
            try:
                if f.stat().st_mtime > cutoff_ts:
                    is_subagent = "subagent" in str(f)
                    jsonl_files.append((f, proj_name, is_subagent))
            except OSError:
                continue

    all_turns = []

    for filepath, proj_name, is_subagent in jsonl_files:
        try:
            # Read all entries from file
            entries = []
            with open(filepath, "r", encoding="utf-8", errors="replace") as fh:
                for raw_line in fh:
                    # Partial line = file being written, skip (FIX-13)
                    if not raw_line.endswith("\n"):
                        continue
                    line = raw_line.strip()
                    if not line:
                        continue
                    try:
                        entries.append(json.loads(line))
                    except json.JSONDecodeError:
                        continue

            if not entries:
                continue

            session_file = filepath.relative_to(CLAUDE_DIR).as_posix()

            # State tracking
            current_permission_mode = "default"
            turn_index = 0

            # Index: group assistant entries by message.id for streaming dedup
            assistant_groups: dict[str, list] = {}  # message.id → [entries]
            user_entries = []
            system_entries = []

            for entry in entries:
                etype = entry.get("type")
                if etype == "assistant":
                    msg = entry.get("message", {})
                    msg_id = msg.get("id", "")
                    if msg_id:
                        if msg_id not in assistant_groups:
                            assistant_groups[msg_id] = []
                        assistant_groups[msg_id].append(entry)
                elif etype == "user":
                    user_entries.append(entry)
                elif etype == "system":
                    system_entries.append(entry)

            # Build turn_duration lookup: parentUuid → durationMs
            duration_by_parent: dict[str, int] = {}
            for se in system_entries:
                if se.get("subtype") == "turn_duration":
                    parent = se.get("parentUuid", "")
                    dur = se.get("durationMs")
                    if parent and dur is not None:
                        duration_by_parent[parent] = dur

            # Build merged assistant messages in file order (by first entry timestamp)
            merged_messages = []
            for msg_id, group in assistant_groups.items():
                # Sort group by timestamp to get correct order
                group.sort(key=lambda e: e.get("timestamp", ""))
                first = group[0]
                msg = first.get("message", {})
                model = msg.get("model", "unknown")
                if model.startswith("<") or model == "unknown":
                    continue

                # Merge content blocks from all streaming entries
                all_content = []
                for entry in group:
                    content = entry.get("message", {}).get("content", [])
                    if isinstance(content, list):
                        all_content.extend(content)

                # Usage from any entry (identical across streaming entries)
                usage = msg.get("usage", {})
                if not usage:
                    continue

                # Extract tool names and detect thinking (from merged content)
                tool_names = []
                has_thinking = False
                for block in all_content:
                    if not isinstance(block, dict):
                        continue
                    btype = block.get("type")
                    if btype == "tool_use":
                        name = block.get("name", "")
                        if name:
                            tool_names.append(name)
                    elif btype == "thinking":
                        has_thinking = True

                # Get UUIDs for duration linkage (last entry's uuid)
                last_uuid = group[-1].get("uuid", "")
                duration_ms = duration_by_parent.get(last_uuid)

                # Timestamp from first streaming entry
                ts = first.get("timestamp", "")

                merged_messages.append({
                    "msg_id": msg_id,
                    "timestamp": ts,
                    "model": model,
                    "usage": usage,
                    "tool_names": tool_names,
                    "has_thinking": has_thinking,
                    "duration_ms": duration_ms,
                    "first_uuid": first.get("uuid", ""),
                    "last_uuid": last_uuid,
                })

            # Sort merged messages by timestamp
            merged_messages.sort(key=lambda m: m["timestamp"])

            # Build user entry index by timestamp for permission mode tracking
            # Sort user entries by timestamp
            user_entries.sort(key=lambda e: e.get("timestamp", ""))

            # Track which user entry precedes each assistant message
            user_idx = 0
            for mm in merged_messages:
                # Advance user_idx to the last user entry before this assistant message
                while user_idx < len(user_entries) and user_entries[user_idx].get("timestamp", "") <= mm["timestamp"]:
                    ue = user_entries[user_idx]
                    pm = ue.get("permissionMode")
                    if pm:
                        current_permission_mode = pm
                    user_idx += 1

                # Check for interruption: user entry right after this assistant with interrupt text
                is_interrupted = False
                if user_idx < len(user_entries):
                    next_user = user_entries[user_idx]
                    next_content = next_user.get("message", {}).get("content", [])
                    if isinstance(next_content, list):
                        for block in next_content:
                            if isinstance(block, dict) and block.get("type") == "text":
                                # Check for interrupt marker without reading full text
                                text = block.get("text", "")
                                if text.startswith("[Request interrupted"):
                                    is_interrupted = True
                                    break

                # Check for errors in subsequent user entries (tool_results)
                is_error = False
                if user_idx < len(user_entries):
                    next_user = user_entries[user_idx]
                    next_content = next_user.get("message", {}).get("content", [])
                    if isinstance(next_content, list):
                        for block in next_content:
                            if isinstance(block, dict) and block.get("type") == "tool_result":
                                if block.get("is_error"):
                                    is_error = True
                                    break

                usage = mm["usage"]
                inp = usage.get("input_tokens", 0)
                out = usage.get("output_tokens", 0)
                cr = usage.get("cache_read_input_tokens", 0)
                cc = usage.get("cache_creation_input_tokens", 0)
                cost = _estimate_cost(mm["model"], usage)

                all_turns.append({
                    "session_file": session_file,
                    "turn_index": turn_index,
                    "timestamp": mm["timestamp"],
                    "model": mm["model"],
                    "permission_mode": current_permission_mode,
                    "duration_ms": mm["duration_ms"],
                    "input_tokens": inp,
                    "output_tokens": out,
                    "cache_read_tokens": cr,
                    "cache_creation_tokens": cc,
                    "cost": cost,
                    "has_thinking": mm["has_thinking"],
                    "tool_count": len(mm["tool_names"]),
                    "tools_used": json.dumps(mm["tool_names"]),
                    "is_error": is_error,
                    "is_interrupted": is_interrupted,
                    "project": proj_name,
                    "is_subagent": is_subagent,
                })
                turn_index += 1

        except Exception as e:
            log.warning(f"Pass 2: error scanning {filepath}: {e}")
            continue

    return all_turns


def _persist_turns(turns: list[dict], full: bool = True):
    """Persist turn-level metrics to native_turns table.
    full=True: wipe and rebuild. full=False: delete only scanned session_files, then insert."""
    conn = _get_db()
    try:
        conn.execute("BEGIN IMMEDIATE")
        try:
            if full:
                conn.execute("DELETE FROM native_turns")
            else:
                # Delta: only remove rows for session files we just re-scanned
                session_files = list(set(t["session_file"] for t in turns))
                for sf in session_files:
                    conn.execute("DELETE FROM native_turns WHERE session_file = ?", (sf,))
            if turns:
                conn.executemany(
                    "INSERT INTO native_turns VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
                    [(t["session_file"], t["turn_index"], t["timestamp"], t["model"],
                      t["permission_mode"], t["duration_ms"], t["input_tokens"],
                      t["output_tokens"], t["cache_read_tokens"], t["cache_creation_tokens"],
                      t["cost"], t["has_thinking"], t["tool_count"], t["tools_used"],
                      t["is_error"], t["is_interrupted"], t["project"], t["is_subagent"])
                     for t in turns],
                )
            conn.commit()
        except Exception:
            conn.rollback()
            raise
        log.info(f"Pass 2: persisted {len(turns)} turns to native_turns (full={full})")
    finally:
        conn.close()


def _scan_jsonl_and_persist(full: bool = False) -> int:
    """Thin wrapper: scans JSONL entries and persists to live_entries table.
    Returns number of new entries inserted."""
    entries = _scan_jsonl_entries(full=full)
    if not entries:
        return 0

    rows = [
        (e["timestamp"], e["model"], e["input_tokens"], e["output_tokens"],
         e["cache_read_tokens"], e["cache_creation_tokens"], e["cost"],
         e["project"], e["session_id"])
        for e in entries
    ]
    conn = _get_db()
    try:
        inserted = conn.executemany(
            "INSERT OR IGNORE INTO live_entries (timestamp, model, input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens, cost, project, session_id) VALUES (?,?,?,?,?,?,?,?,?)",
            rows,
        ).rowcount
        conn.commit()
        return inserted
    finally:
        conn.close()


def _recalculate_live_entry_costs():
    """Recalculate all live_entry costs using current PRICING table."""
    conn = _get_db()
    try:
        rows = conn.execute(
            "SELECT rowid, model, input_tokens, output_tokens, "
            "cache_read_tokens, cache_creation_tokens, cost FROM live_entries"
        ).fetchall()
        updates = []
        for r in rows:
            usage = {
                "input_tokens": r["input_tokens"],
                "output_tokens": r["output_tokens"],
                "cache_read_input_tokens": r["cache_read_tokens"],
                "cache_creation_input_tokens": r["cache_creation_tokens"],
            }
            new_cost = _estimate_cost(r["model"], usage)
            if abs(new_cost - r["cost"]) > 0.0001:
                updates.append((new_cost, r["rowid"]))
        if updates:
            conn.executemany("UPDATE live_entries SET cost = ? WHERE rowid = ?", updates)
            conn.commit()
            log.info(f"Recalculated costs for {len(updates)} live entries")
            # Propagate corrected costs to daily_usage
            _rebuild_daily_from_live()
        else:
            log.info("All live entry costs are up to date")
    finally:
        conn.close()


# ---------------------------------------------------------------------------
# Rebuild derived tables from live_entries (single source of truth)
# ---------------------------------------------------------------------------
def _rebuild_daily_from_live():
    """Rebuild daily_usage from live_entries (single source of truth).
    Only deletes+rebuilds dates present in live_entries — historical dates
    outside the 30-day window are never touched."""
    conn = _get_db()
    try:
        # Find which dates have live_entries
        dates = conn.execute(
            "SELECT DISTINCT substr(timestamp, 1, 10) FROM live_entries"
        ).fetchall()
        date_list = [r[0] for r in dates]

        if not date_list:
            return

        # Atomic rebuild — acquire write lock upfront (FIX-09)
        conn.execute("BEGIN IMMEDIATE")
        try:
            # Only delete daily_usage for dates we can rebuild
            placeholders = ",".join("?" * len(date_list))
            conn.execute(
                f"DELETE FROM daily_usage WHERE date IN ({placeholders})",
                date_list,
            )

            # Rebuild per-project rows from live_entries
            conn.execute("""
                INSERT INTO daily_usage (date, project, model_name,
                    input_tokens, output_tokens, cache_creation_tokens,
                    cache_read_tokens, cost)
                SELECT
                    substr(timestamp, 1, 10) AS date,
                    project,
                    model AS model_name,
                    SUM(input_tokens),
                    SUM(output_tokens),
                    SUM(cache_creation_tokens),
                    SUM(cache_read_tokens),
                    SUM(cost)
                FROM live_entries
                GROUP BY substr(timestamp, 1, 10), project, model
            """)

            # __all__ aggregate rows for dates we just rebuilt
            conn.execute(f"""
                INSERT INTO daily_usage (date, project, model_name,
                    input_tokens, output_tokens, cache_creation_tokens,
                    cache_read_tokens, cost)
                SELECT
                    date, '__all__', model_name,
                    SUM(input_tokens),
                    SUM(output_tokens),
                    SUM(cache_creation_tokens),
                    SUM(cache_read_tokens),
                    SUM(cost)
                FROM daily_usage
                WHERE project != '__all__' AND date IN ({placeholders})
                GROUP BY date, model_name
            """, date_list)

            conn.commit()
        except Exception:
            conn.rollback()
            raise
    finally:
        conn.close()


def _rebuild_sessions_from_live():
    """Rebuild sessions from live_entries (single source of truth).
    Uses hybrid SQL+Python approach for JSON fields."""
    from collections import defaultdict

    conn = _get_db()
    try:
        rows = conn.execute("""
            SELECT project, model,
                SUM(input_tokens) AS input_tokens,
                SUM(output_tokens) AS output_tokens,
                SUM(cache_creation_tokens) AS cache_creation_tokens,
                SUM(cache_read_tokens) AS cache_read_tokens,
                SUM(cost) AS cost,
                MAX(timestamp) AS last_activity
            FROM live_entries
            GROUP BY project, model
        """).fetchall()
    finally:
        conn.close()

    if not rows:
        return

    # Build session data in Python (need JSON aggregation)
    groups: dict[str, dict] = defaultdict(lambda: {
        "input_tokens": 0, "output_tokens": 0,
        "cache_creation_tokens": 0, "cache_read_tokens": 0,
        "cost": 0.0, "last_activity": "", "models": set(),
        "model_data": defaultdict(lambda: {
            "inputTokens": 0, "outputTokens": 0,
            "cacheCreationTokens": 0, "cacheReadTokens": 0, "cost": 0.0,
        }),
    })

    for r in rows:
        project = r["project"]
        model = r["model"]
        g = groups[project]
        g["input_tokens"] += r["input_tokens"]
        g["output_tokens"] += r["output_tokens"]
        g["cache_creation_tokens"] += r["cache_creation_tokens"]
        g["cache_read_tokens"] += r["cache_read_tokens"]
        g["cost"] += r["cost"]
        if r["last_activity"] > g["last_activity"]:
            g["last_activity"] = r["last_activity"]
        g["models"].add(model)
        md = g["model_data"][model]
        md["inputTokens"] += r["input_tokens"]
        md["outputTokens"] += r["output_tokens"]
        md["cacheCreationTokens"] += r["cache_creation_tokens"]
        md["cacheReadTokens"] += r["cache_read_tokens"]
        md["cost"] += r["cost"]

    session_rows = []
    for project, g in groups.items():
        total_tokens = (g["input_tokens"] + g["output_tokens"]
                        + g["cache_creation_tokens"] + g["cache_read_tokens"])
        models_used = sorted(g["models"])
        model_breakdowns = []
        for model_name in models_used:
            md = g["model_data"][model_name]
            total_t = (md["inputTokens"] + md["outputTokens"]
                       + md["cacheCreationTokens"] + md["cacheReadTokens"])
            model_breakdowns.append({
                "modelName": model_name,
                "inputTokens": md["inputTokens"],
                "outputTokens": md["outputTokens"],
                "cacheCreationTokens": md["cacheCreationTokens"],
                "cacheReadTokens": md["cacheReadTokens"],
                "totalTokens": total_t,
                "cost": md["cost"],
            })
        session_rows.append((
            project,  # session_id = project dir name
            "Unknown Project",
            g["input_tokens"], g["output_tokens"],
            g["cache_creation_tokens"], g["cache_read_tokens"],
            total_tokens, g["cost"],
            g["last_activity"],
            json.dumps(models_used),
            json.dumps(model_breakdowns),
        ))

    conn = _get_db()
    try:
        conn.execute("BEGIN IMMEDIATE")
        try:
            conn.execute("DELETE FROM sessions")
            if session_rows:
                conn.executemany(
                    "INSERT INTO sessions VALUES (?,?,?,?,?,?,?,?,?,?,?)",
                    session_rows,
                )
            conn.commit()
        except Exception:
            conn.rollback()
            raise
    finally:
        conn.close()


def _check_sync_integrity() -> bool:
    """Compare daily_usage vs live_entries for dates where live_entries exist.
    Only checks overlapping dates — historical daily_usage from before cleanup
    is excluded from the comparison (FIX-14)."""
    conn = _get_db()
    try:
        live = conn.execute(
            "SELECT SUM(input_tokens + output_tokens + cache_creation_tokens + cache_read_tokens) AS tokens, "
            "SUM(cost) AS cost FROM live_entries"
        ).fetchone()
        if not live["tokens"]:
            return True

        # Restrict daily_usage comparison to dates that still have live_entries
        live_dates = conn.execute(
            "SELECT DISTINCT substr(timestamp, 1, 10) FROM live_entries"
        ).fetchall()
        date_list = [r[0] for r in live_dates]
        if not date_list:
            return True

        placeholders = ",".join("?" * len(date_list))
        daily = conn.execute(
            f"SELECT SUM(input_tokens + output_tokens + cache_creation_tokens + cache_read_tokens) AS tokens, "
            f"SUM(cost) AS cost FROM daily_usage WHERE project = '__all__' AND date IN ({placeholders})",
            date_list,
        ).fetchone()

        token_match = (live["tokens"] or 0) == (daily["tokens"] or 0)
        cost_drift = abs((live["cost"] or 0) - (daily["cost"] or 0))
        if not token_match or cost_drift > 0.01:
            log.warning(f"INTEGRITY DRIFT: live={live['tokens']:,.0f}tok/${live['cost']:.2f} "
                        f"vs daily={daily['tokens']:,.0f}tok/${daily['cost']:.2f}")
            return False
        return True
    finally:
        conn.close()


def _native_sync(full: bool = True):
    """Orchestrator: scan JSONL → live_entries → rebuild daily_usage + sessions.
    live_entries is append-only (INSERT OR IGNORE). daily_usage and sessions
    are always derived from live_entries, never from JSONL scans directly."""
    t0 = time.time()
    entries = _scan_jsonl_entries(full=full)
    scan_elapsed = time.time() - t0

    if not entries:
        log.info(f"Native sync: 0 entries found ({scan_elapsed:.1f}s scan)")
        return

    # 1) INSERT OR IGNORE into live_entries (append-only, always safe)
    rows_live = [
        (e["timestamp"], e["model"], e["input_tokens"], e["output_tokens"],
         e["cache_read_tokens"], e["cache_creation_tokens"], e["cost"],
         e["project"], e["session_id"])
        for e in entries
    ]

    conn = _get_db()
    try:
        inserted = conn.executemany(
            "INSERT OR IGNORE INTO live_entries (timestamp, model, input_tokens, "
            "output_tokens, cache_read_tokens, cache_creation_tokens, cost, "
            "project, session_id) VALUES (?,?,?,?,?,?,?,?,?)",
            rows_live,
        ).rowcount
        conn.commit()
    finally:
        conn.close()

    # 2) Rebuild daily_usage from live_entries (source of truth)
    _rebuild_daily_from_live()

    # 3) Rebuild sessions from live_entries
    _rebuild_sessions_from_live()

    # 4) Integrity check — corrective rebuild if drift detected
    if not _check_sync_integrity():
        log.warning("Drift detected — corrective rebuild")
        _rebuild_daily_from_live()

    elapsed = time.time() - t0
    log.info(f"Native sync: {len(entries)} entries scanned, {inserted} new live entries "
             f"({elapsed:.1f}s total, {scan_elapsed:.1f}s scan, full={full})")

    # 5) Pass 2: extract turn-level metrics (YOLO, tools, duration)
    try:
        t2 = time.time()
        turns = _scan_jsonl_turns(full=full)
        _persist_turns(turns, full=full)
        log.info(f"Pass 2: {len(turns)} turns extracted ({time.time() - t2:.1f}s)")
    except Exception as e:
        log.error(f"Pass 2 failed: {e}")



def _get_live_entries_from_db(minutes: int = 60) -> list[dict]:
    """Read live entries from SQLite for the given time window."""
    cutoff = (datetime.now(timezone.utc) - timedelta(minutes=minutes)).strftime("%Y-%m-%dT%H:%M:%S")
    conn = _get_db()
    try:
        rows = conn.execute(
            "SELECT timestamp, model, input_tokens, output_tokens, "
            "cache_read_tokens, cache_creation_tokens, cost, project "
            "FROM live_entries WHERE timestamp >= ? ORDER BY timestamp",
            (cutoff,),
        ).fetchall()
        return [{
            "timestamp": r["timestamp"],
            "model": r["model"],
            "inputTokens": r["input_tokens"],
            "outputTokens": r["output_tokens"],
            "cacheReadTokens": r["cache_read_tokens"],
            "cacheCreationTokens": r["cache_creation_tokens"],
            "cost": r["cost"],
            "project": r["project"] or "",
        } for r in rows]
    finally:
        conn.close()


def _cleanup_old_live_entries():
    """Remove live_entries older than 30 days."""
    cutoff = (datetime.now(timezone.utc) - timedelta(days=30)).strftime("%Y-%m-%dT%H:%M:%S")
    conn = _get_db()
    try:
        deleted = conn.execute(
            "DELETE FROM live_entries WHERE timestamp < ?", (cutoff,)
        ).rowcount
        conn.commit()
        if deleted:
            log.info(f"Cleaned up {deleted} old live entries")
    finally:
        conn.close()


def _bucket_key(ts_str: str, bucket_seconds: int) -> str:
    ts = datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
    if bucket_seconds >= 60:
        # Floor to nearest N-minute boundary
        bucket_minutes = bucket_seconds // 60
        floored_minute = (ts.minute // bucket_minutes) * bucket_minutes
        return ts.replace(minute=floored_minute, second=0, microsecond=0).strftime("%Y-%m-%dT%H:%M")
    floored = (ts.second // bucket_seconds) * bucket_seconds
    return ts.replace(second=floored, microsecond=0).strftime("%Y-%m-%dT%H:%M:%S")


def _day_ratio(conn, day_str: str, from_dt: datetime, to_dt: datetime) -> float:
    """What fraction of a day's activity (by live_entries cost) falls between from_dt and to_dt?
    Falls back to uniform time distribution if no live_entries exist for that day."""
    from_str = from_dt.strftime("%Y-%m-%dT%H:%M:%S")
    to_str = to_dt.strftime("%Y-%m-%dT%H:%M:%S")

    live_full = conn.execute(
        "SELECT SUM(cost) FROM live_entries WHERE substr(timestamp,1,10)=?",
        (day_str,),
    ).fetchone()[0] or 0.0

    if live_full <= 0:
        # No live data — assume uniform distribution across 24h
        hours = (to_dt - from_dt).total_seconds() / 3600
        return min(max(hours / 24.0, 0.0), 1.0)

    live_partial = conn.execute(
        "SELECT SUM(cost) FROM live_entries WHERE substr(timestamp,1,10)=? AND timestamp >= ? AND timestamp < ?",
        (day_str, from_str, to_str),
    ).fetchone()[0] or 0.0

    return min(live_partial / live_full, 1.0)


def _rolling_cost(conn, window_start: datetime, window_end: datetime) -> float:
    """Calculate accurate rolling cost for [window_start, window_end).
    Uses daily_usage for accurate per-day totals, live_entries for intra-day proportions."""
    from datetime import date as date_type

    start_date = window_start.date()
    end_date = window_end.date()
    total = 0.0

    d = start_date
    while d <= end_date:
        day_str = d.strftime("%Y-%m-%d")
        r = conn.execute(
            "SELECT SUM(cost) FROM daily_usage WHERE project='__all__' AND date=?",
            (day_str,),
        ).fetchone()
        day_cost = r[0] or 0.0

        if day_cost > 0:
            # Interior days: 100% of cost
            is_first = (d == start_date)
            is_last = (d == end_date)

            if not is_first and not is_last:
                total += day_cost
            else:
                # Boundary day — compute partial ratio
                day_start = datetime(d.year, d.month, d.day, 0, 0, 0, tzinfo=timezone.utc)
                day_end = datetime(d.year, d.month, d.day, 23, 59, 59, tzinfo=timezone.utc)
                overlap_start = max(day_start, window_start)
                overlap_end = min(day_end, window_end)

                if overlap_end > overlap_start:
                    if overlap_start <= day_start and overlap_end >= day_end:
                        total += day_cost  # Full day within window
                    else:
                        ratio = _day_ratio(conn, day_str, overlap_start, overlap_end)
                        total += day_cost * ratio

        d += timedelta(days=1)

    return total


@app.get("/api/kpi-periods")
def api_kpi_periods():
    """Rolling 24h/7d costs with comparison to previous period.
    Combines daily_usage (accurate costs) with live_entries (intra-day proportions)."""
    _wait_for_db()
    cached = _cache_get("kpi-periods")
    if cached is not None:
        return JSONResponse(cached)
    conn = _get_db()
    try:
        now = datetime.now(timezone.utc)

        cost_24h = _rolling_cost(conn, now - timedelta(hours=24), now)
        cost_prev_24h = _rolling_cost(conn, now - timedelta(hours=48), now - timedelta(hours=24))
        cost_7d = _rolling_cost(conn, now - timedelta(days=7), now)
        cost_prev_7d = _rolling_cost(conn, now - timedelta(days=14), now - timedelta(days=7))

        today_str = now.strftime("%Y-%m-%d")
        projects_today = conn.execute(
            "SELECT COUNT(DISTINCT project) FROM live_entries WHERE substr(timestamp,1,10)=? AND project != ''",
            (today_str,)
        ).fetchone()[0] or 0
        sessions_today = conn.execute(
            "SELECT COUNT(DISTINCT session_id) FROM live_entries WHERE substr(timestamp,1,10)=? AND session_id != ''",
            (today_str,)
        ).fetchone()[0] or 0

        result = {
            "cost24h": round(cost_24h, 4),
            "costPrev24h": round(cost_prev_24h, 4),
            "cost7d": round(cost_7d, 4),
            "costPrev7d": round(cost_prev_7d, 4),
            "projectsToday": projects_today,
            "sessionsToday": sessions_today,
            "serverTime": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
        }
        _cache_set("kpi-periods", result)
        return JSONResponse(result)
    finally:
        conn.close()


_last_live_scan: float = 0.0
_LIVE_SCAN_INTERVAL: float = 10.0
_live_scan_lock = threading.Lock()


@app.get("/api/live")
def api_live(minutes: int = Query(60), bucket: int = Query(60)):
    global _last_live_scan
    with _live_scan_lock:
        now = time.time()
        if now - _last_live_scan >= _LIVE_SCAN_INTERVAL:
            _scan_jsonl_and_persist()
            _last_live_scan = now
    # Read from DB (includes historical data that survived session changes)
    entries = _get_live_entries_from_db(minutes)

    total_tokens = 0
    total_cost = 0.0
    total_input = 0
    total_output = 0
    total_cache_read = 0
    model_counts: dict[str, dict] = {}
    time_buckets: dict[str, dict] = {}

    for e in entries:
        inp = e["inputTokens"]
        out = e["outputTokens"]
        cr = e["cacheReadTokens"]
        cc = e["cacheCreationTokens"]
        t = inp + out + cr + cc
        total_tokens += t
        total_cost += e["cost"]
        total_input += inp
        total_output += out
        total_cache_read += cr

        m = e["model"]
        if m not in model_counts:
            model_counts[m] = {"tokens": 0, "cost": 0.0, "calls": 0}
        model_counts[m]["tokens"] += t
        model_counts[m]["cost"] += e["cost"]
        model_counts[m]["calls"] += 1

        bk = _bucket_key(e["timestamp"], bucket)
        if bk not in time_buckets:
            time_buckets[bk] = {"tokens": 0, "cost": 0.0, "calls": 0, "inputTokens": 0, "outputTokens": 0, "cacheReadTokens": 0, "cacheCreationTokens": 0, "models": {}, "projects": {}}
        time_buckets[bk]["tokens"] += t
        time_buckets[bk]["cost"] += e["cost"]
        time_buckets[bk]["calls"] += 1
        time_buckets[bk]["inputTokens"] += inp
        time_buckets[bk]["outputTokens"] += out
        time_buckets[bk]["cacheReadTokens"] += cr
        time_buckets[bk]["cacheCreationTokens"] += cc
        if m not in time_buckets[bk]["models"]:
            time_buckets[bk]["models"][m] = {"tokens": 0, "cost": 0.0}
        time_buckets[bk]["models"][m]["tokens"] += t
        time_buckets[bk]["models"][m]["cost"] += e["cost"]

        proj = e.get("project", "")
        if proj not in time_buckets[bk]["projects"]:
            time_buckets[bk]["projects"][proj] = {"tokens": 0, "cost": 0.0}
        time_buckets[bk]["projects"][proj]["tokens"] += t
        time_buckets[bk]["projects"][proj]["cost"] += e["cost"]

    return JSONResponse({
        "totalTokens": total_tokens,
        "totalCost": total_cost,
        "totalInput": total_input,
        "totalOutput": total_output,
        "totalCacheRead": total_cache_read,
        "apiCalls": len(entries),
        "models": model_counts,
        "timeline": time_buckets,
        "entries": len(entries),
        "windowMinutes": minutes,
        "bucketSeconds": bucket,
        "serverTime": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S") + "Z",
    })


# --- Sync trigger (replaces old cache clear) ---

def _require_local(request: Request):
    """Reject requests not from localhost/LAN. Used for sensitive local-only endpoints."""
    real_ip = request.headers.get("X-Real-IP")
    if real_ip:
        raise HTTPException(status_code=403, detail="Local-only endpoint")
    host = request.client.host if request.client else ""
    if host.startswith("127.") or host == "::1" or host == "localhost":
        return  # loopback — OK
    if host.startswith("10.") or host.startswith("192.168."):
        return  # private LAN — OK
    if host.startswith("100.") and 64 <= int(host.split(".")[1]) <= 127:
        return  # CGNAT / Tailscale (100.64.0.0/10) — OK
    # 172.16.0.0 – 172.31.255.255
    if host.startswith("172."):
        parts = host.split(".")
        if len(parts) >= 2 and 16 <= int(parts[1]) <= 31:
            return
    raise HTTPException(status_code=403, detail="This endpoint is local-only")


@app.post("/api/cache/clear")
def clear_cache(request: Request):
    """Trigger an immediate delta sync."""
    _require_local(request)
    def _bg_sync():
        try:
            _native_sync(full=False)
            _cache_invalidate()
        except Exception as e:
            log.error(f"Manual sync failed: {e}")
    threading.Thread(target=_bg_sync, daemon=True).start()
    _cache_invalidate()  # Immediate invalidation so next request fetches fresh
    return {"status": "ok", "message": "Delta sync triggered"}



# ---------------------------------------------------------------------------
# Metrics endpoints (Pass 2: turn-level data)
# ---------------------------------------------------------------------------
@app.get("/api/metrics")
def api_metrics():
    """Aggregated metrics from native_turns: YOLO mode, tool usage, turn stats, sessions."""
    _wait_for_db()
    cached = _cache_get("metrics")
    if cached is not None:
        return JSONResponse(cached)
    conn = _get_db()
    try:
        # --- YOLO / Permission Mode ---
        try:
            total_turns = conn.execute("SELECT COUNT(*) FROM native_turns").fetchone()[0] or 0
        except Exception:
            total_turns = 0
        if total_turns == 0:
            return JSONResponse({"yolo": {"totalTurns": 0, "yoloTurns": 0, "yoloPct": 0, "yoloPct24h": 0, "yoloCost": 0, "totalCost": 0, "byPermissionMode": {}}, "tools": {"totalCalls": 0, "avgToolsPerTurn": 0, "byTool": {}, "byCategory": {}}, "turns": {"totalTurns": 0, "avgDurationMs": 0, "medianDurationMs": 0, "thinkingTurns": 0, "thinkingPct": 0, "errorTurns": 0, "errorPct": 0, "interruptedTurns": 0, "interruptedPct": 0, "subagentTurns": 0, "subagentPct": 0, "avgCostPerTurn": 0}, "sessions": {"totalSessions": 0, "avgTurnsPerSession": 0, "avgCostPerSession": 0, "avgDurationPerSession": 0}})

        mode_rows = conn.execute(
            "SELECT permission_mode, COUNT(*) as cnt, SUM(cost) as total_cost "
            "FROM native_turns GROUP BY permission_mode"
        ).fetchall()
        by_permission_mode = {}
        for r in mode_rows:
            by_permission_mode[r["permission_mode"]] = {
                "turns": r["cnt"],
                "cost": round(r["total_cost"] or 0, 4),
            }
        yolo_data = by_permission_mode.get("bypassPermissions", {"turns": 0, "cost": 0})
        total_cost = sum(v["cost"] for v in by_permission_mode.values())

        # 24h YOLO stats
        cutoff_24h = (datetime.now(timezone.utc) - timedelta(hours=24)).strftime("%Y-%m-%dT%H:%M:%S")
        row_24h = conn.execute(
            "SELECT COUNT(*) as total, SUM(CASE WHEN permission_mode = 'bypassPermissions' THEN 1 ELSE 0 END) as yolo "
            "FROM native_turns WHERE timestamp >= ?", (cutoff_24h,)
        ).fetchone()
        total_24h = row_24h["total"] or 0
        yolo_24h = row_24h["yolo"] or 0

        yolo = {
            "totalTurns": total_turns,
            "yoloTurns": yolo_data["turns"],
            "yoloPct": round(yolo_data["turns"] / max(total_turns, 1) * 100, 1),
            "yoloPct24h": round(yolo_24h / max(total_24h, 1) * 100, 1),
            "yoloCost": yolo_data["cost"],
            "totalCost": round(total_cost, 4),
            "byPermissionMode": by_permission_mode,
        }

        # --- Tool usage ---
        tool_rows = conn.execute(
            "SELECT tools_used FROM native_turns WHERE tool_count > 0"
        ).fetchall()
        tool_totals: dict[str, int] = {}
        total_tool_calls = 0
        for r in tool_rows:
            try:
                tools = json.loads(r["tools_used"])
                for t in tools:
                    tool_totals[t] = tool_totals.get(t, 0) + 1
                    total_tool_calls += 1
            except (json.JSONDecodeError, TypeError):
                continue
        # Sort by count descending
        by_tool = dict(sorted(tool_totals.items(), key=lambda x: -x[1]))

        error_count = conn.execute("SELECT COUNT(*) FROM native_turns WHERE is_error = 1").fetchone()[0] or 0
        tools = {
            "totalCalls": total_tool_calls,
            "byTool": by_tool,
            "errorRate": round(error_count / max(total_turns, 1) * 100, 1),
        }

        # --- Turn stats ---
        dur_row = conn.execute(
            "SELECT AVG(duration_ms) as avg_dur, COUNT(*) as cnt "
            "FROM native_turns WHERE duration_ms IS NOT NULL"
        ).fetchone()
        avg_duration = round(dur_row["avg_dur"]) if dur_row["avg_dur"] else None

        # Median duration
        dur_count = dur_row["cnt"] or 0
        median_duration = None
        if dur_count > 0:
            mid = dur_count // 2
            median_row = conn.execute(
                "SELECT duration_ms FROM native_turns WHERE duration_ms IS NOT NULL "
                "ORDER BY duration_ms LIMIT 1 OFFSET ?", (mid,)
            ).fetchone()
            if median_row:
                median_duration = median_row["duration_ms"]

        avg_cost = conn.execute("SELECT AVG(cost) FROM native_turns").fetchone()[0] or 0
        thinking_count = conn.execute("SELECT COUNT(*) FROM native_turns WHERE has_thinking = 1").fetchone()[0] or 0
        interrupted_count = conn.execute("SELECT COUNT(*) FROM native_turns WHERE is_interrupted = 1").fetchone()[0] or 0

        turns = {
            "total": total_turns,
            "avgDurationMs": avg_duration,
            "medianDurationMs": median_duration,
            "avgCost": round(avg_cost, 4),
            "thinkingPct": round(thinking_count / max(total_turns, 1) * 100, 1),
            "interruptedPct": round(interrupted_count / max(total_turns, 1) * 100, 1),
        }

        # --- Session stats ---
        session_count = conn.execute(
            "SELECT COUNT(DISTINCT session_file) FROM native_turns"
        ).fetchone()[0] or 0
        subagent_sessions = conn.execute(
            "SELECT COUNT(DISTINCT session_file) FROM native_turns WHERE is_subagent = 1"
        ).fetchone()[0] or 0

        sessions = {
            "total": session_count,
            "avgTurnsPerSession": round(total_turns / max(session_count, 1), 1),
            "subagentSessions": subagent_sessions,
        }

        result = {
            "yolo": yolo,
            "tools": tools,
            "turns": turns,
            "sessions": sessions,
            "serverTime": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        }
        _cache_set("metrics", result)
        return JSONResponse(result)
    finally:
        conn.close()


@app.get("/api/metrics/yolo")
def api_metrics_yolo():
    """Dedicated YOLO mode endpoint with per-project and per-day breakdowns."""
    _wait_for_db()
    cached = _cache_get("metrics-yolo")
    if cached is not None:
        return JSONResponse(cached)
    conn = _get_db()
    try:
        try:
            total_turns = conn.execute("SELECT COUNT(*) FROM native_turns").fetchone()[0] or 0
        except Exception:
            total_turns = 0
        if total_turns == 0:
            return JSONResponse({"totalTurns": 0, "yoloTurns": 0, "yoloPct": 0, "yoloCost": 0, "totalCost": 0, "byProject": {}, "byDay": []})

        yolo_turns = conn.execute(
            "SELECT COUNT(*) FROM native_turns WHERE permission_mode = 'bypassPermissions'"
        ).fetchone()[0] or 0
        yolo_cost = conn.execute(
            "SELECT SUM(cost) FROM native_turns WHERE permission_mode = 'bypassPermissions'"
        ).fetchone()[0] or 0
        total_cost = conn.execute("SELECT SUM(cost) FROM native_turns").fetchone()[0] or 0

        # Per-project breakdown
        project_rows = conn.execute(
            "SELECT project, permission_mode, COUNT(*) as cnt "
            "FROM native_turns GROUP BY project, permission_mode"
        ).fetchall()
        by_project: dict[str, dict] = {}
        for r in project_rows:
            proj = r["project"]
            if proj not in by_project:
                by_project[proj] = {"yoloPct": 0, "yoloTurns": 0, "turns": 0}
            by_project[proj]["turns"] += r["cnt"]
            if r["permission_mode"] == "bypassPermissions":
                by_project[proj]["yoloTurns"] += r["cnt"]
        for proj, data in by_project.items():
            data["yoloPct"] = round(data["yoloTurns"] / max(data["turns"], 1) * 100, 1)

        # Per-day breakdown
        day_rows = conn.execute(
            "SELECT substr(timestamp, 1, 10) as date, permission_mode, COUNT(*) as cnt "
            "FROM native_turns GROUP BY date, permission_mode ORDER BY date"
        ).fetchall()
        day_data: dict[str, dict] = {}
        for r in day_rows:
            d = r["date"]
            if d not in day_data:
                day_data[d] = {"date": d, "yoloTurns": 0, "totalTurns": 0}
            day_data[d]["totalTurns"] += r["cnt"]
            if r["permission_mode"] == "bypassPermissions":
                day_data[d]["yoloTurns"] += r["cnt"]
        by_day = list(day_data.values())

        result = {
            "yoloPct": round(yolo_turns / max(total_turns, 1) * 100, 1),
            "yoloTurns": yolo_turns,
            "totalTurns": total_turns,
            "yoloCost": round(yolo_cost, 4),
            "totalCost": round(total_cost, 4),
            "yoloCostPct": round(yolo_cost / max(total_cost, 0.01) * 100, 1),
            "byProject": by_project,
            "byDay": by_day,
            "serverTime": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        }
        _cache_set("metrics-yolo", result)
        return JSONResponse(result)
    finally:
        conn.close()


@app.get("/api/hourly-pattern")
def api_hourly_pattern():
    """Hourly activity pattern from native_turns — 24 entries (one per hour)."""
    _wait_for_db()
    cached = _cache_get("hourly-pattern")
    if cached is not None:
        return JSONResponse(cached)
    conn = _get_db()
    try:
        rows = conn.execute(
            "SELECT substr(timestamp, 12, 2) AS hour, "
            "SUM(cost) AS total_cost, COUNT(*) AS turns, "
            "COUNT(DISTINCT substr(timestamp, 1, 10)) AS active_days "
            "FROM native_turns GROUP BY hour ORDER BY hour"
        ).fetchall()
        hour_map = {int(r["hour"]): {"totalCost": r["total_cost"], "turns": r["turns"], "activeDays": r["active_days"]} for r in rows}
        hours = []
        for h in range(24):
            entry = hour_map.get(h, {"totalCost": 0, "turns": 0, "activeDays": 0})
            active = entry["activeDays"] or 1
            hours.append({
                "hour": h,
                "avgCost": round(entry["totalCost"] / active, 4),
                "totalCost": round(entry["totalCost"], 4),
                "turns": entry["turns"],
            })
        result = {"hours": hours}
        _cache_set("hourly-pattern", result)
        return JSONResponse(result)
    finally:
        conn.close()


# ---------------------------------------------------------------------------
# Push endpoints (local — work without GITHUB_CLIENT_ID)
# ---------------------------------------------------------------------------

@app.get("/api/push/status")
def api_push_status():
    """Auth status + available months + receipt history."""
    from myburn.auth import load_config
    from myburn.push import list_receipts

    cfg = load_config()
    logged_in = bool(cfg.get("token"))

    conn = _get_db()
    try:
        rows = conn.execute(
            "SELECT DISTINCT substr(date, 1, 7) AS month FROM daily_usage "
            "WHERE project = '__all__' ORDER BY month DESC"
        ).fetchall()
        months = [r["month"] for r in rows]
    finally:
        conn.close()

    return JSONResponse({
        "loggedIn": logged_in,
        "username": cfg.get("username") if logged_in else None,
        "avatarUrl": cfg.get("avatar_url") if logged_in else None,
        "availableMonths": months,
        "receipts": list_receipts(),
        "autoPush": cfg.get("auto_push", "off"),
    })


@app.get("/api/push/preview")
def api_push_preview(month: str = Query(...)):
    """Preview the push payload for a month (no side effects)."""
    from myburn.push import build_payload
    payload = build_payload(month)
    return JSONResponse(payload)


@app.post("/api/push/send")
async def api_push_send(request: Request):
    """Build payload, save receipt, optionally cloud-push."""
    _require_local(request)

    # Body size limit — 1MB (FIX-28)
    content_length = request.headers.get("content-length")
    if content_length and int(content_length) > 1_048_576:
        return JSONResponse({"ok": False, "error": "Request body too large (max 1MB)"}, status_code=413)

    from myburn.push import build_payload, save_receipt, mark_receipt_synced, _cloud_push
    from myburn.auth import load_config

    body = await request.json()
    month = body.get("month")
    if not month:
        return JSONResponse({"ok": False, "error": "month is required"}, status_code=400)

    payload = build_payload(month)
    if payload["cost"] == 0 and payload["turns"]["total"] == 0:
        return JSONResponse({"ok": False, "error": f"No data for {month}"}, status_code=400)

    # Always save locally
    save_receipt(payload)

    # Attempt cloud push if logged in
    cfg = load_config()
    token = cfg.get("token")
    cloud_synced = False
    profile_url = None
    cloud_error = None

    if token:
        try:
            # Add hash chain link (FIX-29 — same as CLI push)
            from myburn.push import _get_prev_receipt_hash
            payload["_prevHash"] = _get_prev_receipt_hash(month) or "genesis"
            result = _cloud_push(payload, token)
            cloud_synced = True
            profile_url = result.get("profile_url")
            mark_receipt_synced(month)
        except Exception as e:
            cloud_error = str(e)

    return JSONResponse({
        "ok": True,
        "month": month,
        "cost": payload["cost"],
        "cloudSynced": cloud_synced,
        "profileUrl": profile_url,
        "error": cloud_error,
    })


@app.get("/api/push/auto")
def api_push_auto_get(request: Request):
    """Get auto-push setting."""
    _require_local(request)
    from myburn.auth import load_config
    cfg = load_config()
    return JSONResponse({
        "autoPush": cfg.get("auto_push", "off"),
        "lastAutoPushAt": cfg.get("last_auto_push_at"),
    })


@app.post("/api/push/auto")
async def api_push_auto_set(request: Request):
    """Set auto-push mode (off/daily/weekly/monthly)."""
    _require_local(request)
    from myburn.auth import load_config, save_config
    body = await request.json()
    mode = body.get("auto_push", "off")
    if mode not in ("off", "daily", "weekly", "monthly"):
        return JSONResponse({"ok": False, "error": f"Invalid mode: {mode}"}, status_code=400)

    cfg = load_config()
    cfg["auto_push"] = mode
    save_config(cfg)
    return JSONResponse({"ok": True, "autoPush": mode})


@app.get("/api/push/privacy")
def api_push_privacy_get(request: Request):
    """Proxy GET /api/privacy from cloud — returns privacy settings for logged-in user."""
    _require_local(request)
    from myburn.auth import load_config, MYBURN_API
    import urllib.request as _urllib_req

    cfg = load_config()
    token = cfg.get("token")
    if not token:
        return JSONResponse({})

    api_url = os.environ.get("MYBURN_API", MYBURN_API)
    try:
        req = _urllib_req.Request(
            f"{api_url}/api/privacy",
            headers={"Authorization": f"Bearer {token}", "User-Agent": "MyBurn-Server/0.1"},
        )
        with _urllib_req.urlopen(req, timeout=15) as resp:
            return JSONResponse(json.loads(resp.read()))
    except Exception:
        return JSONResponse({})


@app.post("/api/push/privacy")
async def api_push_privacy_set(request: Request):
    """Proxy POST /api/privacy to cloud — updates privacy settings."""
    _require_local(request)
    from myburn.auth import load_config, MYBURN_API
    import urllib.request as _urllib_req

    cfg = load_config()
    token = cfg.get("token")
    if not token:
        return JSONResponse({"ok": False, "error": "Not logged in"}, status_code=401)

    body = await request.json()
    api_url = os.environ.get("MYBURN_API", MYBURN_API)
    try:
        data = json.dumps(body).encode()
        req = _urllib_req.Request(
            f"{api_url}/api/privacy",
            data=data,
            headers={
                "Authorization": f"Bearer {token}",
                "Content-Type": "application/json",
                "User-Agent": "MyBurn-Server/0.1",
            },
            method="POST",
        )
        with _urllib_req.urlopen(req, timeout=15) as resp:
            return JSONResponse(json.loads(resp.read()))
    except Exception as e:
        return JSONResponse({"ok": False, "error": str(e)}, status_code=502)


# --- Browser-based Device Flow auth (local-only) ---

@app.post("/api/auth/device/begin")
def api_auth_device_begin(request: Request):
    """Start GitHub Device Flow — proxy to GitHub (local-only)."""
    _require_local(request)
    from myburn.auth import GITHUB_CLIENT_ID
    import urllib.request as _urllib_req
    import urllib.parse as _urllib_parse
    import urllib.error as _urllib_err
    import ssl
    import socket

    body = _urllib_parse.urlencode({"client_id": GITHUB_CLIENT_ID, "scope": "read:user"}).encode()
    req = _urllib_req.Request("https://github.com/login/device/code", data=body,
                              headers={"Accept": "application/json"}, method="POST")
    try:
        with _urllib_req.urlopen(req, timeout=15) as resp:
            data = json.loads(resp.read())
    except _urllib_err.URLError as e:
        if isinstance(e.reason, ssl.SSLError):
            raise HTTPException(502, "SSL error — your network may be blocking connections (corporate proxy?)")
        raise HTTPException(502, "Can't reach GitHub — check your internet connection")
    except socket.timeout:
        raise HTTPException(504, "GitHub request timed out — try again")
    return JSONResponse({
        "device_code": data["device_code"],
        "user_code": data["user_code"],
        "verification_uri": data["verification_uri"],
        "interval": data.get("interval", 5),
        "expires_in": data.get("expires_in", 900),
    })


@app.post("/api/auth/device/check")
async def api_auth_device_check(request: Request):
    """Poll GitHub Device Flow. On success: exchange with myburn.cc, save to config (local-only)."""
    _require_local(request)
    body = await request.json()
    device_code = body.get("device_code")
    if not device_code:
        raise HTTPException(400, "Missing device_code")

    from myburn.auth import GITHUB_CLIENT_ID, MYBURN_API, save_config
    import urllib.request as _urllib_req
    import urllib.parse as _urllib_parse
    import urllib.error as _urllib_err
    import ssl
    import socket

    # Poll GitHub for token
    form = _urllib_parse.urlencode({
        "client_id": GITHUB_CLIENT_ID,
        "device_code": device_code,
        "grant_type": "urn:ietf:params:oauth:grant-type:device_code",
    }).encode()
    req = _urllib_req.Request("https://github.com/login/oauth/access_token", data=form,
                              headers={"Accept": "application/json"}, method="POST")
    try:
        with _urllib_req.urlopen(req, timeout=15) as resp:
            gh_resp = json.loads(resp.read())
    except _urllib_err.URLError as e:
        if isinstance(e.reason, ssl.SSLError):
            raise HTTPException(502, "SSL error — your network may be blocking connections (corporate proxy?)")
        raise HTTPException(502, "Can't reach GitHub — check your internet connection")
    except socket.timeout:
        raise HTTPException(504, "GitHub request timed out — try again")

    if "error" in gh_resp:
        error = gh_resp["error"]
        if error in ("authorization_pending", "slow_down"):
            return JSONResponse({"status": "pending", "interval": gh_resp.get("interval", 5)})
        return JSONResponse({"status": error})

    gh_token = gh_resp.get("access_token")
    if not gh_token:
        raise HTTPException(502, "No access_token from GitHub")

    # Exchange GH token for MyBurn token on myburn.cc
    api_url = os.environ.get("MYBURN_API", MYBURN_API)
    exchange_body = json.dumps({"github_token": gh_token}).encode()
    exc_req = _urllib_req.Request(
        f"{api_url}/api/auth/github", data=exchange_body,
        headers={
            "Content-Type": "application/json",
            "Accept": "application/json",
            "User-Agent": "MyBurn-Server/0.1",
        },
        method="POST",
    )
    try:
        with _urllib_req.urlopen(exc_req, timeout=15) as resp:
            result = json.loads(resp.read())
    except _urllib_err.URLError as e:
        if isinstance(e.reason, ssl.SSLError):
            raise HTTPException(502, "SSL error connecting to myburn.cc")
        raise HTTPException(502, "Can't reach myburn.cc — check your internet connection")
    except socket.timeout:
        raise HTTPException(504, "myburn.cc request timed out — try again")

    # Save to ~/.myburn/config.json — push endpoints read from here
    save_config({
        "token": result["token"],
        "username": result["username"],
        "avatar_url": result.get("avatar_url", ""),
        "logged_in_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
    })

    return JSONResponse({
        "status": "ok",
        "username": result["username"],
        "avatar_url": result.get("avatar_url", ""),
    })


@app.post("/api/auth/logout")
def api_auth_logout(request: Request):
    """Clear local auth + revoke on cloud (local-only)."""
    _require_local(request)
    from myburn.auth import load_config, save_config, MYBURN_API
    import urllib.request as _urllib_req

    cfg = load_config()
    token = cfg.get("token")
    if token:
        try:
            api_url = os.environ.get("MYBURN_API", MYBURN_API)
            req = _urllib_req.Request(
                f"{api_url}/api/auth/revoke", data=b"{}",
                headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json", "User-Agent": "MyBurn-Server/0.1"},
                method="POST",
            )
            _urllib_req.urlopen(req, timeout=10)
        except Exception:
            pass
    for key in ("token", "username", "avatar_url", "logged_in_at"):
        cfg.pop(key, None)
    save_config(cfg)
    return JSONResponse({"ok": True})


# --- Waitlist (temporary — SQLite, separate DB) ---

_waitlist_conn_local = threading.local()
WAITLIST_DB_PATH = DATA_DIR / "waitlist.db"

def _get_waitlist_db() -> sqlite3.Connection:
    """Lazy-init waitlist DB. Separate from usage.db — easy to kill later."""
    conn = getattr(_waitlist_conn_local, "conn", None)
    if conn is None:
        conn = sqlite3.connect(str(WAITLIST_DB_PATH), timeout=10)
        result = conn.execute("PRAGMA journal_mode=WAL").fetchone()
        if result and result[0] != "wal":
            log.error("WAL mode failed for waitlist.db (got '%s')", result[0])
        conn.execute("PRAGMA busy_timeout=5000")
        conn.execute(
            "CREATE TABLE IF NOT EXISTS waitlist ("
            "id INTEGER PRIMARY KEY AUTOINCREMENT, "
            "email TEXT UNIQUE NOT NULL, "
            "created_at TEXT NOT NULL)"
        )
        conn.commit()
        _waitlist_conn_local.conn = conn
    return conn

_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")

@app.post("/api/waitlist")
async def api_waitlist_add(request: Request):
    """Add email to waitlist."""
    body = await request.json()
    email = (body.get("email") or "").strip().lower()
    if not email or not _EMAIL_RE.match(email) or len(email) > 254:
        raise HTTPException(status_code=400, detail="Invalid email")
    conn = _get_waitlist_db()
    try:
        conn.execute(
            "INSERT OR IGNORE INTO waitlist (email, created_at) VALUES (?, ?)",
            (email, datetime.now(timezone.utc).isoformat()),
        )
        conn.commit()
    except Exception:
        conn.rollback()
        raise
    return JSONResponse({"ok": True})


@app.get("/api/waitlist/count")
def api_waitlist_count():
    """Return waitlist size."""
    conn = _get_waitlist_db()
    count = conn.execute("SELECT COUNT(*) FROM waitlist").fetchone()[0]
    return JSONResponse({"count": count})


# --- Health ---

@app.get("/api/health")
def api_health():
    """Sync health: entry counts, integrity check, db size."""
    conn = _get_db()
    try:
        live = conn.execute(
            "SELECT COUNT(*) AS cnt, "
            "SUM(input_tokens+output_tokens+cache_read_tokens+cache_creation_tokens) AS tokens, "
            "SUM(cost) AS cost FROM live_entries"
        ).fetchone()
        daily = conn.execute(
            "SELECT COUNT(*) AS cnt, "
            "SUM(input_tokens+output_tokens+cache_read_tokens+cache_creation_tokens) AS tokens, "
            "SUM(cost) AS cost FROM daily_usage WHERE project='__all__'"
        ).fetchone()
        tokens_match = (live["tokens"] or 0) == (daily["tokens"] or 0)
        result = {
            "status": "ok" if tokens_match else "drift",
            "db_ready": _db_ready.is_set(),
            "sync_alive": _sync_thread.is_alive() if _sync_thread else False,
            "sync_degraded": _sync_degraded,
            "live_entries": live["cnt"],
            "daily_rows": daily["cnt"],
            "live_tokens": live["tokens"] or 0,
            "daily_tokens": daily["tokens"] or 0,
            "integrity": tokens_match,
            "db_bytes": DB_PATH.stat().st_size if DB_PATH.exists() else 0,
        }
    finally:
        conn.close()

    # Cloud DB health (only when cloud routes are enabled)
    if os.environ.get("GITHUB_CLIENT_ID"):
        try:
            from myburn.cloud import _get_cloud_db, CLOUD_DB_PATH
            cconn = _get_cloud_db()
            try:
                users = cconn.execute("SELECT COUNT(*) FROM users").fetchone()[0]
                pushes = cconn.execute("SELECT COUNT(*) FROM pushes").fetchone()[0]
                result["cloud_ready"] = True
                result["cloud_users"] = users
                result["cloud_pushes"] = pushes
                result["cloud_db_bytes"] = CLOUD_DB_PATH.stat().st_size if CLOUD_DB_PATH.exists() else 0
            finally:
                cconn.close()
        except Exception as e:
            result["cloud_ready"] = False
            result["cloud_error"] = str(e)

    return result


# --- Static Files ---

static_dir = Path(__file__).parent / "static"
static_dir.mkdir(exist_ok=True)

app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")


@app.get("/")
def landing():
    return FileResponse(str(static_dir / "landing.html"))


@app.get("/dashboard")
def dashboard():
    return FileResponse(str(static_dir / "index.html"))


@app.get("/changelog")
def changelog():
    return FileResponse(str(static_dir / "changelog.html"))


@app.get("/wrapped")
def wrapped():
    return FileResponse(str(static_dir / "wrapped.html"))


@app.get("/login")
def login_page():
    return FileResponse(str(static_dir / "login.html"))


@app.get("/testlogo")
def testlogo():
    return FileResponse(str(static_dir / "testlogo.html"))


@app.get("/manifest.json")
def manifest():
    return FileResponse(str(static_dir / "manifest.json"), media_type="application/manifest+json")


@app.get("/sw.js")
def service_worker():
    return FileResponse(str(static_dir / "sw.js"), media_type="application/javascript")


# --- Startup ---

@app.on_event("startup")
def startup():
    global _sync_thread
    _acquire_instance_lock()
    _migrate_drop_ccusage()
    _init_db()
    _sync_thread = threading.Thread(target=_sync_loop, daemon=True, name="myburn-sync")
    _sync_thread.start()
    log.info("Sync thread started")


@app.on_event("shutdown")
def shutdown():
    log.info("Shutting down — stopping sync thread...")
    _shutdown_event.set()
    if _sync_thread and _sync_thread.is_alive():
        _sync_thread.join(timeout=10)
    try:
        _wal_checkpoint()
        log.info("Final WAL checkpoint complete")
    except Exception as e:
        log.error(f"Final WAL checkpoint failed: {e}")
    _release_instance_lock()
    log.info("Clean shutdown complete")


# --- Cloud routes (only on VPS with GITHUB_CLIENT_ID set) ---
if os.environ.get("GITHUB_CLIENT_ID"):
    from myburn.cloud import register_cloud_routes
    register_cloud_routes(app)


