"""
scrylo/api/routers/web.py  (Week 3 — replaces Week 2 version)
--------------------------------------------------------------
HTMX Web UI routes + scrape job management.

New in Week 3:
  GET  /leads                   → real Lead Finder page (was stub)
  POST /api/leads/find          → start background find job, returns {job_id}
  GET  /api/leads/find-status/{job_id} → poll job progress
  GET  /api/leads/export-csv    → download all leads as CSV
  DELETE /api/leads/{id}        → delete a lead (already in leads.py, wired here too)

All other routes unchanged from Week 2.
"""

from __future__ import annotations

import asyncio
import csv
import io
import uuid
from datetime import datetime
from typing import Any

from fastapi import APIRouter, BackgroundTasks, Depends, File, Form, Request, UploadFile
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, StreamingResponse
from fastapi.templating import Jinja2Templates
from sqlalchemy.orm import Session

from scrylo.core.config import settings as cfg, save as save_cfg
from scrylo.db.models import Lead, LeadStatus
from scrylo.db.session import get_db

import pathlib

TEMPLATES_DIR = pathlib.Path(__file__).parent.parent.parent / "web" / "templates"
templates = Jinja2Templates(directory=str(TEMPLATES_DIR))

router = APIRouter(tags=["web"])

# ── In-memory job registry (persists for session, resets on restart) ──
_jobs: dict[str, dict] = {}

# ── Shared base context ────────────────────────────────────────
def _base_ctx(db: Session) -> dict[str, Any]:
    return {
        "total_leads":   db.query(Lead).count(),
        "current_time":  datetime.now().strftime("%Y-%m-%d %H:%M"),
        "deps_missing":  _check_deps(),   # passes dep warnings to every page/template
    }

# ── Auto-installer ────────────────────────────────────────────
import sys as _sys
import subprocess as _subprocess
import threading as _threading

_install_jobs: dict[str, dict] = {}

_DEP_INSTALL_STEPS = {
    "patchright": [
        {"label": "Installing patchright...",            "cmd": [_sys.executable, "-m", "pip", "install", "patchright"]},
        {"label": "Downloading Chromium (~170 MB)...",   "cmd": [_sys.executable, "-m", "patchright", "install", "chromium"]},
    ],
    "patchright_browser": [
        {"label": "Downloading Chromium (~170 MB)...",   "cmd": [_sys.executable, "-m", "patchright", "install", "chromium"]},
    ],
    "easyocr": [
        {"label": "Installing EasyOCR...",               "cmd": [_sys.executable, "-m", "pip", "install", "easyocr"]},
    ],
}

_DEP_INFO = {
    "patchright":         {"name": "Patchright + Chromium", "size": "~170 MB",  "note": "Scrylo uses a real browser to find leads from JavaScript-heavy sites. Without it, lead discovery cannot start.\n\nInstalling takes ~2 minutes. It downloads once and runs silently in the background — no window ever opens."},
    "patchright_browser": {"name": "Chromium browser",      "size": "~170 MB",  "note": "Patchright is installed but the Chromium browser hasn't been downloaded yet. Scrylo needs it to find leads from JavaScript-heavy sites.\n\nDownloads once (~170 MB) and runs silently in the background — no window ever opens."},
    "easyocr":            {"name": "EasyOCR",               "size": "~500 MB",  "note": "Required for business card import. The pip package is ~500 MB. The AI recognition model (~1.5 GB) downloads automatically on your first card scan."},
}

import time as _time
_deps_cache: dict = {}
_deps_checked_at: float = 0.0
_DEPS_TTL = 2.0  # re-check at most once per 2 seconds

def _check_deps() -> dict[str, str]:
    """
    Returns a dict of {dep_key: error_message} for any missing required optional deps.
    Results are cached for _DEPS_TTL seconds so repeated page loads don't re-import.
    """
    global _deps_cache, _deps_checked_at
    now = _time.monotonic()
    if now - _deps_checked_at < _DEPS_TTL:
        return _deps_cache

    missing: dict[str, str] = {}

    # patchright installed?
    from scrylo.scrapers.browser import is_patchright_installed, is_browser_downloaded, recheck_patchright
    recheck_patchright() # Update global flags in browser.py
    if not is_patchright_installed():
        missing["patchright"] = "patchright not installed"
    elif not is_browser_downloaded():
        missing["patchright_browser"] = "Chromium browser not downloaded yet"

    # easyocr installed?
    try:
        import easyocr  # noqa: F401
    except ImportError:
        missing["easyocr"] = "easyocr not installed"

    _deps_cache = missing
    _deps_checked_at = now
    return missing

# Alias — kept for backwards compat with any existing call sites
_get_deps = _check_deps

def _run_install(dep_key: str):
    """Background thread — runs install steps and streams output into _install_jobs."""
    global _deps_checked_at
    steps = _DEP_INSTALL_STEPS.get(dep_key, [])
    if not steps:
        _install_jobs[dep_key] = {"status": "error", "log": [], "error": f"Unknown dep: {dep_key}"}
        return

    _install_jobs[dep_key] = {"status": "running", "log": [], "error": None}

    for step in steps:
        _install_jobs[dep_key]["log"].append(f"▸ {step['label']}")
        try:
            proc = _subprocess.Popen(
                step["cmd"],
                stdout=_subprocess.PIPE,
                stderr=_subprocess.STDOUT,
                text=True, bufsize=1,
            )
            for line in proc.stdout:
                line = line.rstrip()
                if line:
                    _install_jobs[dep_key]["log"].append(line)
            proc.wait()
            if proc.returncode != 0:
                _install_jobs[dep_key].update({
                    "status": "error",
                    "error":  f"Step failed (exit {proc.returncode}): {step['label']}",
                })
                return
        except Exception as e:
            _install_jobs[dep_key].update({"status": "error", "error": str(e)})
            return

    # Invalidate dep cache so next page load re-checks
    global _deps_checked_at
    _deps_checked_at = 0.0
    _install_jobs[dep_key]["status"] = "done"
    _install_jobs[dep_key]["log"].append("✓ Installation complete — refresh the page to continue.")

@router.get("/api/dep-info/{dep_key}")
def dep_info_endpoint(dep_key: str):
    info = _DEP_INFO.get(dep_key)
    if not info:
        return JSONResponse({"error": "unknown dep"}, status_code=404)
    return JSONResponse(info)

@router.post("/api/install-dep/{dep_key}")
def install_dep(dep_key: str):
    if dep_key not in _DEP_INSTALL_STEPS:
        return JSONResponse({"error": "unknown dep"}, status_code=400)
    job = _install_jobs.get(dep_key, {})
    if job.get("status") == "running":
        return JSONResponse({"started": False, "reason": "already running"})
    _install_jobs[dep_key] = {"status": "running", "log": [], "error": None}
    t = _threading.Thread(target=_run_install, args=(dep_key,), daemon=True)
    t.start()
    return JSONResponse({"started": True})

@router.get("/api/install-status/{dep_key}")
def install_status(dep_key: str):
    job = _install_jobs.get(dep_key)
    if not job:
        return JSONResponse({"status": "not_started", "log": [], "error": None})
    return JSONResponse(job)

# ── Dashboard ──────────────────────────────────────────────────
@router.get("/", response_class=HTMLResponse)
def dashboard(request: Request, db: Session = Depends(get_db)):
    from scrylo.core.config import load as _load_cfg
    cfg = _load_cfg()
    
    base = _base_ctx(db)
    sent = db.query(Lead).filter(Lead.status == LeadStatus.SENT).count()
    opened = db.query(Lead).filter(Lead.status == LeadStatus.OPENED).count()
    with_email = db.query(Lead).filter(Lead.email.isnot(None)).count()

    # Calculate checklist
    checklist = [
        {
            "label": "Search API Key",
            "ok":    bool((cfg.get("scraper", {}).get("serper_api_key") or "").strip()),
            "hint":  "Required to find leads via Google/Serper.",
            "link":  "/settings#scraper",
        },
        {
            "label": "Ideal Customer Profile",
            "ok":    bool((cfg.get("icp", {}).get("profile") or "").strip()),
            "hint":  "Define who you're looking for to enable lead scoring.",
            "link":  "/settings#icp",
        },
        {
            "label": "Service Description",
            "ok":    bool((cfg.get("pitch", {}).get("service_description") or "").strip()),
            "hint":  "Used by AI to write personalised cold emails.",
            "link":  "/settings#pitch",
        },
        {
            "label": "SMTP Configuration",
            "ok":    bool((cfg.get("email", {}).get("smtp_host") or "").strip()),
            "hint":  "Set up your email provider to send outreach.",
            "link":  "/settings#email",
        },
    ]
    setup_done = all(item["ok"] for item in checklist)

    return templates.TemplateResponse("dashboard.html", {
        "request": request,
        "title": "Dashboard",
        "active_page": "dashboard",
        "stats": {
            "total": base["total_leads"],
            "with_email": with_email,
            "sent": sent,
            "open_rate": round(opened / sent * 100, 1) if sent else 0,
        },
        "checklist": checklist,
        "setup_done": setup_done,
        **base,
    })

# ── Lead Finder (fully wired Week 3) ──────────────────────────
@router.get("/leads", response_class=HTMLResponse)
def leads_page(request: Request, db: Session = Depends(get_db)):
    from scrylo.core.llm_adapter import _ollama_available, _list_ollama_models
    from scrylo.core.config import load as _load_cfg
    cfg = _load_cfg()
    
    # Just check if keys are CONFIGURED, not live verification (saves API calls/network time)
    groq_key = (cfg.get("llm", {}).get("groq_api_key") or "").strip()
    groq_ready = bool(groq_key)
    
    ollama_active = _ollama_available()
    ollama_models = _list_ollama_models() if ollama_active else []

    leads = (
        db.query(Lead)
        .order_by(Lead.icp_score.desc(), Lead.created_at.desc())
        .limit(500)
        .all()
    )
    return templates.TemplateResponse("leads.html", {
        "request":       request,
        "title":         "Lead Finder",
        "active_page":   "leads",
        "leads":         leads,
        "last_query":    "",
        "groq_ready":    groq_ready,
        "ollama_active": ollama_active,
        "ollama_models": ollama_models,
        **_base_ctx(db),
    })

# ── Find job — start ─────────────────────────────────────────
@router.post("/leads/find")
async def start_find(request: Request, background_tasks: BackgroundTasks):
    data    = await request.json()
    query   = (data.get("query") or "").strip()
    limit   = min(int(data.get("limit", 40)), 500)
    # sources: list of strings e.g. ["web", "github"] or legacy "ddg" → ["web"]
    sources_raw = data.get("sources", data.get("source", "web"))
    if isinstance(sources_raw, str):
        # backward compat — single source string
        sources = ["web"] if sources_raw in ("ddg", "web", "") else [sources_raw]
    else:
        sources = [s for s in sources_raw if s in ("web", "github")]
    if not sources:
        sources = ["web"]

    icp_mode = data.get("icp_mode", "auto")  # "auto" | "manual"

    if not query:
        return JSONResponse({"error": "query is required"}, status_code=422)

    # Check patchright dep before starting
    deps = _get_deps()
    if "patchright" in deps or "patchright_browser" in deps:
        key = "patchright" if "patchright" in deps else "patchright_browser"
        return JSONResponse({
            "dep_missing": key,
            "error": deps[key],
        }, status_code=428)  # 428 = Precondition Required

    job_id = str(uuid.uuid4())[:8]
    _jobs[job_id] = {
        "status":  "running",
        "stage":   "starting",
        "label":   "Starting...",
        "found":   0,
        "saved":   0,
        "total":   limit,
        "sources": sources,
        "error":   None,
    }

    background_tasks.add_task(_run_find_job, job_id, query, limit, sources, icp_mode)
    return JSONResponse({"job_id": job_id})

# Animated labels shown during silent processing phases
_WORKING_LABELS = [
    "⚙️  Verifying emails...",
    "⚙️  Resolving domains...",
    "⚙️  Filtering leads...",
    "⚙️  Processing results...",
    "⚙️  Almost there...",
]


async def _run_find_job(job_id: str, query: str, limit: int, sources: list, icp_mode: str = "auto"):
    """
    Background task — runs the scrape in a dedicated thread with its own
    event loop, isolated from uvicorn's event loop.

    Previously ran find_leads directly on uvicorn's event loop, competing
    with every HTTP status poll the browser sends. Now uses run_sources_sync
    in a ThreadPoolExecutor — asyncio.run() inside the thread creates a
    completely fresh event loop matching CLI execution speed.

    A keep-alive task on the uvicorn loop cycles animated labels every 2s
    during silent phases (MX checks, LLM filter, DB saves) so the progress
    bar never appears frozen.
    """
    import time
    from scrylo.scrapers.sources_runner import run_sources_sync

    start_time = time.time()
    loop       = asyncio.get_event_loop()

    # Sync callback — called from worker thread, writes directly to _jobs.
    # Plain dict updates are thread-safe in CPython.
    def _progress(stage: str, count: int, total: int, label: str = ""):
        elapsed = int(time.time() - start_time)
        pct     = min(round(count / max(total, 1) * 100), 98)
        _jobs[job_id].update({
            "stage":       stage,
            "label":       label or stage.replace("_", " ").title(),
            "found":       count,
            "total":       max(total, 1),
            "elapsed":     elapsed,
            "pct":         pct,
            "last_update": time.time(),
        })

    # Keep-alive — ticks every 2s on uvicorn loop.
    # Animates label during silent phases so bar never looks frozen.
    async def _keep_alive():
        label_idx = 0
        while _jobs.get(job_id, {}).get("status") == "running":
            await asyncio.sleep(2)
            job = _jobs.get(job_id)
            if not job or job.get("status") != "running":
                break
            elapsed     = int(time.time() - start_time)
            last_update = job.get("last_update", 0)
            if time.time() - last_update > 3:
                _jobs[job_id].update({
                    "elapsed": elapsed,
                    "label":   _WORKING_LABELS[label_idx % len(_WORKING_LABELS)],
                })
                label_idx += 1
            else:
                _jobs[job_id].update({"elapsed": elapsed})

    keep_alive_task = asyncio.create_task(_keep_alive())

    try:
        # Run scrape in thread pool — asyncio.run() inside thread gives it
        # a fresh isolated event loop, same as CLI
        result = await loop.run_in_executor(
            None,
            lambda: run_sources_sync(
                query=query,
                sources=sources,
                limit=limit,
                progress_callback=_progress,
            ),
        )
        keep_alive_task.cancel()

        if result.leads_saved == 0 and result.errors:
            _jobs[job_id].update({"status": "error", "error": result.errors[0]})
            return

        # ── Auto ICP scoring ──────────────────────────────────────────────────
        # Same function as manual button (score_leads_batch) — only difference
        # is it runs automatically on just the newly found leads.
        icp_scored = 0
        if icp_mode == "auto" and result.lead_ids:
            from scrylo.core.config import load as _load_cfg
            from scrylo.core.icp_scorer import score_leads_batch
            icp_profile = (_load_cfg().get("icp", {}).get("profile") or "").strip()
            if icp_profile:
                _jobs[job_id].update({
                    "stage": "icp_scoring",
                    "label": "🧠 ICP Scoring...",
                })
                scores = await loop.run_in_executor(
                    None, score_leads_batch, result.lead_ids, icp_profile
                )
                icp_scored = len(scores)

        _jobs[job_id].update({
            "status":     "done",
            "stage":      "done",
            "label":      "Done!",
            "found":      result.leads_saved + result.leads_skipped,
            "saved":      result.leads_saved,
            "skipped":    result.leads_skipped,
            "emails":     result.emails_found,
            "icp_scored": icp_scored,
            "total":      result.leads_saved + result.leads_skipped,
            "elapsed":    int(time.time() - start_time),
            "errors":     result.errors,
            "per_source": result.per_source,
        })
    except Exception as e:
        keep_alive_task.cancel()
        _jobs[job_id].update({"status": "error", "error": str(e)})

# ── Find job — poll status ───────────────────────────────────
@router.get("/leads/find-status/{job_id}")
def find_status(job_id: str):
    job = _jobs.get(job_id)
    if not job:
        return JSONResponse({"error": "job not found"}, status_code=404)
    return JSONResponse(job)

# ── Manual lead add ────────────────────────────────────────────
@router.post("/api/leads/add-manual")
async def add_lead_manual(request: Request, db: Session = Depends(get_db)):
    """Add a lead manually from the leads page modal."""
    data = await request.json()
    full_name = (data.get("full_name") or "").strip()
    if not full_name:
        return JSONResponse({"error": "full_name is required"}, status_code=422)

    # Dedup by name
    existing = db.query(Lead).filter(Lead.full_name == full_name).first()
    if existing:
        return JSONResponse({"error": f"A lead named '{full_name}' already exists (ID {existing.id})"}, status_code=409)

    parts      = full_name.split()
    first_name = parts[0] if parts else ""
    last_name  = parts[-1] if len(parts) > 1 else ""

    email   = (data.get("email") or "").strip() or None
    website = (data.get("website") or "").strip() or None
    if website and not website.startswith("http"):
        website = "https://" + website

    new_lead = Lead(
        full_name    = full_name,
        first_name   = first_name,
        last_name    = last_name,
        role         = (data.get("role") or "").strip() or None,
        company      = (data.get("company") or "").strip() or None,
        email        = email,
        email_confidence = 1.0 if email else None,
        email_verified   = True if email else False,
        website      = website,
        location     = (data.get("location") or "").strip() or None,
        linkedin_url = (data.get("linkedin_url") or "").strip() or None,
        source       = "manual",
        sources      = '["manual"]',
        status       = LeadStatus.NEW,
        icp_score    = 0,
    )
    db.add(new_lead)
    db.commit()
    db.refresh(new_lead)
    return JSONResponse({"id": new_lead.id, "name": full_name})

# ── Manual ICP scoring ─────────────────────────────────────────
@router.post("/api/leads/score-icp")
async def score_icp(request: Request, db: Session = Depends(get_db)):
    """Score selected leads (or all) against the ICP profile."""
    from scrylo.core.config import load as _load_cfg
    from scrylo.core.icp_scorer import score_leads_batch

    data        = await request.json()
    lead_ids    = data.get("lead_ids", [])   # empty = all leads
    icp_profile = (_load_cfg().get("icp", {}).get("profile") or "").strip()

    if not icp_profile:
        return JSONResponse({"error": "No ICP profile set. Add one in Settings."}, status_code=422)

    if not lead_ids:
        from scrylo.db.models import Lead
        lead_ids = [r[0] for r in db.query(Lead.id).all()]

    if not lead_ids:
        return JSONResponse({"error": "No leads to score."}, status_code=422)

    loop   = asyncio.get_event_loop()
    scores = await loop.run_in_executor(None, score_leads_batch, lead_ids, icp_profile)
    return JSONResponse({"scored": len(scores), "lead_ids": list(scores.keys())})

# ── CSV export ─────────────────────────────────────────────────
@router.get("/leads/export-csv")
def export_csv(db: Session = Depends(get_db)):
    leads = db.query(Lead).order_by(Lead.icp_score.desc()).all()

    output = io.StringIO()
    writer = csv.writer(output)
    writer.writerow([
        "id", "name", "role", "company", "location",
        "email", "email_verified", "score",
        "linkedin_url", "website", "status", "source", "created_at",
    ])
    for l in leads:
        writer.writerow([
            l.id, l.full_name, l.role, l.company, l.location,
            l.email, l.email_verified, l.icp_score,
            l.linkedin_url, l.website, l.status, l.source,
            l.created_at.isoformat() if l.created_at else "",
        ])

    output.seek(0)
    filename = f"scrylo_leads_{datetime.now().strftime('%Y%m%d_%H%M')}.csv"
    return StreamingResponse(
        io.BytesIO(output.getvalue().encode()),
        media_type="text/csv",
        headers={"Content-Disposition": f"attachment; filename={filename}"},
    )

# ── Pitches export ─────────────────────────────────────────────
@router.get("/leads/export-pitches")
def export_pitches(db: Session = Depends(get_db)):
    """Download all generated pitches as a formatted .txt file."""
    leads = (
        db.query(Lead)
        .filter(Lead.pitch_body.isnot(None))
        .order_by(Lead.created_at.desc())
        .all()
    )

    if not leads:
        return JSONResponse({"error": "No pitches found to export"}, status_code=404)

    output = io.StringIO()
    sep = "=" * 80 + "\n"
    line = "-" * 80 + "\n"

    for l in leads:
        output.write(sep)
        output.write(f"NAME:    {l.full_name or '—'}\n")
        output.write(f"ROLE:    {l.role or '—'}\n")
        output.write(f"COMPANY: {l.company or '—'}\n")
        
        contact = l.email or l.linkedin_url or l.github_url or "—"
        output.write(f"CONTACT: {contact}\n")
        output.write(line)
        
        if l.pitch_subject:
            output.write(f"SUBJECT: {l.pitch_subject}\n\n")
        
        output.write(f"{l.pitch_body}\n")
        output.write(sep + "\n\n")

    output.seek(0)
    filename = f"scrylo_pitches_{datetime.now().strftime('%Y%m%d_%H%M')}.txt"
    return StreamingResponse(
        io.BytesIO(output.getvalue().encode()),
        media_type="text/plain",
        headers={"Content-Disposition": f"attachment; filename={filename}"},
    )

# ── OCR business card upload ────────────────────────────────────
@router.post("/leads/ocr-upload")
async def ocr_upload(file: UploadFile = File(...), db: Session = Depends(get_db)):
    """
    Accept a single business card image, pre-process with Pillow,
    run EasyOCR, parse with regex (+ LLM fallback for name only), save lead.
    Returns {ok, leads_added, leads_skipped}.
    """
    import tempfile, os
    from scrylo.scrapers.card_parser import preprocess_card_image, parse_card_text_with_llm_fallback

    # ── Check EasyOCR dep first ───────────────────────────────
    deps = _get_deps()
    if "easyocr" in deps:
        return JSONResponse({
            "ok": False,
            "dep_missing": "easyocr",
            "error": deps["easyocr"],
        }, status_code=428)

    # ── Validate ──────────────────────────────────────────────
    allowed = {"image/png", "image/jpeg", "image/jpg"}
    ct = (file.content_type or "").lower()
    if ct not in allowed:
        return JSONResponse({"ok": False, "error": f"Unsupported file type: {ct}"}, status_code=400)

    image_bytes = await file.read()
    if len(image_bytes) > 10 * 1024 * 1024:
        return JSONResponse({"ok": False, "error": "File too large (max 10 MB)"}, status_code=400)

    # ── EasyOCR ───────────────────────────────────────────────
    try:
        import easyocr
    except ImportError:
        return JSONResponse({
            "ok": False,
            "error": "EasyOCR not installed. Run: pip install easyocr"
        }, status_code=500)

    suffix = ".jpg" if "jpeg" in ct else ".png"
    with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
        tmp.write(image_bytes)
        tmp_path = tmp.name

    pre_path = None
    try:
        # ── Pillow pre-process → cleaner text for EasyOCR ─────
        pre_path  = preprocess_card_image(tmp_path)
        reader    = easyocr.Reader(["en"], gpu=False, verbose=False)
        raw_lines = reader.readtext(pre_path, detail=0, paragraph=True)
        text      = "\n".join(raw_lines).strip()

        if not text:
            return JSONResponse({"ok": False, "error": "No text detected in image"})

        # ── Regex parse + LLM fallback for name only ──────────
        ld = parse_card_text_with_llm_fallback(text)

        full_name = (ld.get("full_name") or "").strip() or None
        email     = (ld.get("email") or "").strip() or None
        company   = (ld.get("company") or "").strip() or None

        if not full_name and not email:
            return JSONResponse({"ok": False, "error": "Could not extract contact info from image"})

        # ── Deduplicate ───────────────────────────────────────
        existing = None
        if email:
            existing = db.query(Lead).filter(Lead.email == email).first()
        if not existing and full_name and company:
            existing = db.query(Lead).filter(
                Lead.full_name == full_name,
                Lead.company   == company,
            ).first()

        if existing:
            return JSONResponse({"ok": True, "leads_added": 0, "leads_skipped": 1})

        # ── Save ──────────────────────────────────────────────
        words = full_name.split() if full_name else []
        lead  = Lead(
            full_name  = full_name,
            first_name = words[0] if words else None,
            last_name  = words[-1] if len(words) > 1 else None,
            role       = (ld.get("role") or "").strip() or None,
            company    = company,
            email      = email,
            phone      = (ld.get("phone") or "").strip() or None,
            website    = (ld.get("website") or "").strip() or None,
            location   = (ld.get("location") or "").strip() or None,
            source     = "ocr",
            status     = LeadStatus.NEW,
        )
        db.add(lead)
        db.commit()
        return JSONResponse({"ok": True, "leads_added": 1, "leads_skipped": 0})

    finally:
        # Clean up both original and preprocessed temp files
        for p in (tmp_path, pre_path):
            if p and os.path.exists(p):
                os.unlink(p)

MODULES = {
    "pitch": {
        "title": "Pitch Writer", "active_page": "pitch",
        "module_slug": "pitch-writer", "module_icon": "✍️",
        "module_week": "Week 6", "module_name": "Pitch Writer",
        "module_desc": "Groq Llama 3.3 70B (recommended) → Ollama → Jinja2 smart templates.\nAdd a free Groq key in Settings for AI-written pitches.",
        "cli_command": "scrylo pitch --llm jinja2 --preview",
    },
    "send": {
        "title": "Send Queue", "active_page": "send",
        "module_slug": "send-queue", "module_icon": "🚀",
        "module_week": "Week 7", "module_name": "Send Queue",
        "module_desc": "Async SMTP with randomised delays.\nTracking pixel injection. Unsubscribe handled automatically.",
        "cli_command": "scrylo send --campaign my_campaign --dry-run",
    },
}

def _stub(page_key: str):
    module_data = MODULES[page_key]
    async def handler(request: Request, db: Session = Depends(get_db)):
        return templates.TemplateResponse("module_stub.html", {
            "request": request, **module_data, **_base_ctx(db),
        })
    return handler


# ── Bulk Sender (fully wired) ──────────────────────────────────
@router.get("/send", response_class=HTMLResponse)
def send_page(request: Request, db: Session = Depends(get_db)):
    from scrylo.core.config import load as _load_cfg
    cfg = _load_cfg()

    smtp_configured = bool(
        (cfg.get("email", {}).get("smtp_host") or "").strip()
        and (cfg.get("email", {}).get("smtp_user") or "").strip()
        and (cfg.get("email", {}).get("smtp_password") or "").strip()
    )

    from datetime import datetime as _dt
    default_campaign_name = f"campaign_{_dt.now().strftime('%Y%m%d_%H%M')}"

    pitched_leads = (
        db.query(Lead)
        .filter(
            Lead.pitch_body.isnot(None),
            Lead.email.isnot(None),
            Lead.email.like("%@%"),
            Lead.status.notin_(["bounced", "unsubscribed", "sent"]),
        )
        .order_by(Lead.icp_score.desc())
        .limit(500)
        .all()
    )

    class _CfgProxy:
        def __init__(self, d): self._d = d
        def __getattr__(self, k): return _AttrDict(self._d.get(k, {}))

    class _AttrDict:
        def __init__(self, d): self.__dict__.update(d)
        def __getattr__(self, k): return ""

    return templates.TemplateResponse("send.html", {
        "request":              request,
        "smtp_configured":      smtp_configured,
        "pitched_leads":        pitched_leads,
        "default_campaign_name": default_campaign_name,
        "cfg":                  _CfgProxy(cfg),
        **_base_ctx(db),
    })

@router.post("/api/send/start")
async def send_start(request: Request, background_tasks: BackgroundTasks):
    data          = await request.json()
    campaign_name = (data.get("campaign_name") or "campaign").strip()
    dry_run       = bool(data.get("dry_run", False))
    limit         = int(data.get("limit", 50))
    delay_min     = int(data.get("delay_min", 90))
    delay_max     = int(data.get("delay_max", 180))
    lead_ids      = data.get("lead_ids") or None   # None = all pitched

    if not campaign_name:
        return JSONResponse({"error": "campaign_name is required"}, status_code=422)

    # Quick subject_warnings preview (check first pitched lead)
    from scrylo.db.session import db_session as _dbs
    from scrylo.email_tools.smtp_sender import _warn_subject
    subject_warnings: list[str] = []
    with _dbs() as db:
        sample = (
            db.query(Lead)
            .filter(Lead.pitch_subject.isnot(None))
            .first()
        )
        if sample and sample.pitch_subject:
            subject_warnings = _warn_subject(sample.pitch_subject)

    job_id = str(uuid.uuid4())[:8]
    _jobs[job_id] = {
        "status":   "running",
        "label":    "Starting...",
        "total":    0,
        "sent":     0,
        "skipped":  0,
        "failed":   0,
        "dry_run":  dry_run,
        "leads":    [],
        "error":    None,
    }

    background_tasks.add_task(
        _run_send_job, job_id, campaign_name, lead_ids,
        dry_run, delay_min, delay_max, limit,
    )

    return JSONResponse({"job_id": job_id, "subject_warnings": subject_warnings})

@router.get("/api/send/status/{job_id}")
def send_status(job_id: str):
    job = _jobs.get(job_id)
    if not job:
        return JSONResponse({"error": "job not found"}, status_code=404)
    return JSONResponse(job)

async def _run_send_job(
    job_id: str,
    campaign_name: str,
    lead_ids: list[int] | None,
    dry_run: bool,
    delay_min: int,
    delay_max: int,
    limit: int,
):
    import time as _time
    from scrylo.email_tools.smtp_sender import run_bulk_send_async

    def _progress(current: int, total: int, label: str):
        _jobs[job_id].update({
            "label": label,
            "total": total,
        })

    try:
        result = await run_bulk_send_async(
            campaign_name     = campaign_name,
            lead_ids          = lead_ids,
            dry_run           = dry_run,
            delay_min         = delay_min,
            delay_max         = delay_max,
            limit             = limit,
            progress_callback = _progress,
        )

        _jobs[job_id].update({
            "status":  "done",
            "label":   f"{'Dry run complete' if dry_run else 'Done'} — {result.sent} {'would send' if dry_run else 'sent'}",
            "total":   result.total,
            "sent":    result.sent,
            "skipped": result.skipped,
            "failed":  result.failed,
            "dry_run": dry_run,
            "leads":   [
                {
                    "lead_id": lr.lead_id,
                    "name":    lr.name,
                    "email":   lr.email,
                    "status":  lr.status,
                    "reason":  lr.reason,
                    "subject": lr.subject,
                }
                for lr in result.leads
            ],
            "errors": result.errors,
        })

    except Exception as e:
        _jobs[job_id].update({
            "status": "error",
            "error":  str(e),
            "label":  f"Error: {e}",
        })

# ── Pitch Writer (fully wired) ─────────────────────────────────
@router.get("/pitch", response_class=HTMLResponse)
def pitch_page(request: Request, db: Session = Depends(get_db)):
    from scrylo.core.llm_adapter import verify_groq_key, _ollama_available, _list_ollama_models
    from scrylo.core.config import load as _load_cfg
    cfg = _load_cfg()
    groq_key  = (cfg.get("llm", {}).get("groq_api_key") or "").strip()
    groq_ready, _ = verify_groq_key(groq_key) if groq_key else (False, "")
    
    ollama_active = _ollama_available()
    ollama_models = _list_ollama_models() if ollama_active else []

    from sqlalchemy import or_
    leads = (
        db.query(Lead)
        .filter(or_(
            Lead.email.isnot(None),
            Lead.linkedin_url.isnot(None),
            Lead.github_url.isnot(None)
        ))
        .order_by(Lead.icp_score.desc(), Lead.created_at.desc())
        .limit(500)
        .all()
    )
    pitched_count = sum(1 for l in leads if l.pitch_body)
    return templates.TemplateResponse("pitch.html", {
        "request":       request,
        "title":         "Pitch Writer",
        "active_page":   "pitch",
        "leads":         leads,
        "pitched_count": pitched_count,
        "groq_ready":    groq_ready,
        "ollama_active": ollama_active,
        "ollama_models": ollama_models,
        **_base_ctx(db),
    })

@router.post("/api/leads/generate-pitch")
async def generate_pitch_api(request: Request, db: Session = Depends(get_db)):
    """Generate + save a pitch for one lead. Returns pitch text + provider."""
    import asyncio
    from scrylo.core.llm_adapter import generate_pitch
    from scrylo.core.config import load as _load_cfg

    data    = await request.json()
    lead_id = int(data.get("lead_id", 0))
    lead    = db.query(Lead).filter(Lead.id == lead_id).first()
    if not lead:
        return JSONResponse({"ok": False, "error": "lead not found"}, status_code=404)

    cfg = _load_cfg()
    svc = (cfg.get("pitch", {}).get("service_description") or "").strip()

    # ── Detect contact method preference ───────────────────
    # If no email, but has LinkedIn/GitHub, we route to social prompt
    contact_method = "email"
    if not lead.email and (lead.linkedin_url or lead.github_url):
        contact_method = "social"

    lead_data = {
        "full_name":           lead.full_name or "",
        "company":             lead.company   or "",
        "role":                lead.role      or "",
        "location":            lead.location  or "",
        "industry":            lead.industry  or "",
        "contact_method":      contact_method,
        "pain_points":         getattr(lead, "pain_points", ""),
        "website":             lead.website   or "",
        "service_description": svc,
    }

    loop   = asyncio.get_event_loop()
    result = await loop.run_in_executor(None, generate_pitch, lead_data)

    # Parse body and subject — LLM writes "SUBJECT: ..." as last line
    body    = result.text
    subject = ""
    if result.text:
        # Split on SUBJECT: wherever it appears
        import re as _re
        subj_match = _re.search(r"\nSUBJECT:\s*(.+)$", result.text, _re.IGNORECASE)
        if subj_match:
            subject = subj_match.group(1).strip()
            body    = result.text[:subj_match.start()].strip()
        else:
            # LLM ignored the instruction — derive from name + company
            fn = (lead.full_name or "").split()[0]
            co = lead.company or ""
            subject = f"Quick question for {fn}" if not co else f"Idea for {co}"

    # Persist
    lead.pitch_subject      = subject
    lead.pitch_body         = body
    lead.pitch_generated_by = result.provider
    if lead.status == LeadStatus.NEW:
        lead.status = LeadStatus.PITCHED
    db.commit()

    return JSONResponse({
        "ok":              True,
        "pitch_subject":   subject,
        "pitch_body":      body,
        "provider":        result.provider,
        "no_service_warn": not bool(svc),   # warn UI if no service description
    })

@router.get("/api/leads/{lead_id}")
def get_lead_details(lead_id: int, db: Session = Depends(get_db)):
    """Fetch all fields for one lead."""
    lead = db.query(Lead).filter(Lead.id == lead_id).first()
    if not lead:
        return JSONResponse({"error": "lead not found"}, status_code=404)
    
    import json as _json
    
    # ── Force absolute keys to ensure browser never sees 'undefined'
    payload = {
        "id":               int(lead.id),
        "full_name":        str(lead.full_name or ""),
        "first_name":       str(lead.first_name or ""),
        "last_name":        str(lead.last_name or ""),
        "role":             str(lead.role or ""),
        "company":          str(lead.company or ""),
        "location":         str(lead.location or ""),
        "website":          str(lead.website or ""),
        "linkedin_url":     str(lead.linkedin_url or ""),
        "github_url":       str(lead.github_url or ""),
        "email":            str(lead.email or ""),
        "email_verified":   bool(lead.email_verified),
        "email_confidence": float(lead.email_confidence or 0.0),
        "icp_score":        float(lead.icp_score or 0.0),
        "source":           str(lead.source or ""),
        "status":           str(lead.status or "new"),
        "created_at":       str(lead.created_at.isoformat() if lead.created_at else ""),
        "pitch_body":       str(lead.pitch_body or ""),
        "pitch_subject":    str(lead.pitch_subject or ""),
        "raw_data":         {}
    }
    
    try:
        if lead.raw_data:
            payload["raw_data"] = _json.loads(lead.raw_data)
    except Exception:
        pass

    return JSONResponse(payload)

@router.get("/api/leads/{lead_id}/pitch")
def get_pitch(lead_id: int, db: Session = Depends(get_db)):
    """Fetch saved pitch for one lead (used by drawer on open)."""
    lead = db.query(Lead).filter(Lead.id == lead_id).first()
    if not lead:
        return JSONResponse({"ok": False}, status_code=404)
    return JSONResponse({
        "ok":                True,
        "pitch_subject":     lead.pitch_subject     or "",
        "pitch_body":        lead.pitch_body        or "",
        "pitch_generated_by": lead.pitch_generated_by or "",
    })

@router.post("/api/leads/{lead_id}/pitch")
async def save_pitch(lead_id: int, request: Request, db: Session = Depends(get_db)):
    """Save manually edited pitch back to DB."""
    data = await request.json()
    lead = db.query(Lead).filter(Lead.id == lead_id).first()
    if not lead:
        return JSONResponse({"ok": False}, status_code=404)
    lead.pitch_subject = data.get("pitch_subject", lead.pitch_subject)
    lead.pitch_body    = data.get("pitch_body",    lead.pitch_body)
    db.commit()
    return JSONResponse({"ok": True})

# ── Settings ───────────────────────────────────────────────────
@router.get("/settings", response_class=HTMLResponse)
def settings_page(request: Request, db: Session = Depends(get_db)):
    # Reload config fresh each time — ensures new keys (groq_api_key etc.)
    # added since server startup are visible without restarting scrylo serve
    from scrylo.core.config import load as _load_cfg
    from scrylo.core.llm_adapter import _ollama_available, _list_ollama_models
    fresh_cfg = _load_cfg()
    return templates.TemplateResponse("settings.html", {
        "request": request, "title": "Settings",
        "active_page": "settings", "cfg": fresh_cfg, 
        "ollama_active": _ollama_available(),
        "ollama_models": _list_ollama_models(),
        **_base_ctx(db),
    })

@router.post("/settings/save", response_class=HTMLResponse)
async def settings_save(
    request: Request,
    smtp_host: str = Form(""), smtp_port: int = Form(587),
    smtp_user: str = Form(""), smtp_password: str = Form(""),
    from_name: str = Form(""), from_address: str = Form(""),
    daily_send_limit: int = Form(50),
    min_delay_seconds: int = Form(90), max_delay_seconds: int = Form(180),
    n_ctx: int = Form(2048), n_threads: int = Form(4), temperature: float = Form(0.7),
    max_parallel_requests: int = Form(10), rate_limit_delay: float = Form(1.5),
    serper_api_key: str = Form(""),
    groq_api_key: str = Form(""),
    physical_address: str = Form(""),
    icp_profile: str = Form(""),
    github_token: str = Form(""),
    service_description: str = Form(""),
):
    try:
        cfg["email"].update({
            "smtp_host": smtp_host, "smtp_port": smtp_port,
            "smtp_user": smtp_user, "from_name": from_name,
            "from_address": from_address, "daily_send_limit": daily_send_limit,
            "min_delay_seconds": min_delay_seconds, "max_delay_seconds": max_delay_seconds,
        })
        cfg["llm"].update({"n_ctx": n_ctx, "n_threads": n_threads, "temperature": temperature})
        cfg["scraper"].update({
            "max_parallel_requests": max_parallel_requests,
            "rate_limit_delay": rate_limit_delay,
        })
        if serper_api_key:
            cfg["scraper"]["serper_api_key"] = serper_api_key
        if groq_api_key:
            cfg["llm"]["groq_api_key"] = groq_api_key
        if github_token:
            cfg["scraper"]["github_token"] = github_token
        cfg["compliance"]["physical_address"] = physical_address
        cfg.setdefault("icp", {})["profile"] = icp_profile
        cfg.setdefault("pitch", {})["service_description"] = service_description
        if smtp_password:
            cfg["email"]["smtp_password"] = smtp_password
        save_cfg(cfg)
        return HTMLResponse("""
            <div style="padding:4rem;text-align:center;font-family:'JetBrains Mono',monospace;">
              <div style="font-size:0.8rem;color:#22c55e;margin-bottom:1rem;">✓ Config saved successfully.</div>
              <a href="/settings" style="color:var(--accent);font-size:0.8rem;">← Go Back</a>
            </div>
        """)
    except Exception as e:
        return HTMLResponse(f"""
            <div style="padding:4rem;text-align:center;font-family:'JetBrains Mono',monospace;">
              <div style="font-size:0.8rem;color:#f43f5e;margin-bottom:1rem;">✗ Error: {str(e)}</div>
              <a href="/settings" style="color:var(--accent);font-size:0.8rem;">← Go Back</a>
            </div>
        """, status_code=400)
