# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# ORCID: 0009-0006-4957-0540
# Email: Georgescottfoley@proton.me
# Licensed under the MIT License - see LICENSE file for details

"""
Familiar Web Dashboard

A web-based control panel for managing the agent.

Features:
- Chat interface
- View/manage memory
- Monitor connected nodes
- Configure skills
- View tasks and deadlines
- Email triage overview
- System status

Run:
    python -m familiar.dashboard
    # Opens at http://localhost:5000

Security:
    Set FAMILIAR_DASHBOARD_KEY environment variable to enable authentication.
    Set FAMILIAR_ALLOWED_ORIGINS for CORS (comma-separated list).
"""

import collections
import json
import logging
import os
import queue
import re
import subprocess
import threading
import time
from datetime import datetime
from functools import wraps
from pathlib import Path

from flask import (
    Flask,
    Response,
    abort,
    jsonify,
    make_response,
    redirect,
    render_template,
    request,
    send_file,
)
from flask_cors import CORS

from familiar.channels.connect_wizard._browse import (
    SEARCHABLE_SERVICES,
    _extract_video_id,
    browse_all,
    is_video_url,
    search_service,
)

logger = logging.getLogger(__name__)

# Data paths - use centralized paths
try:
    from familiar.core.paths import AGENT_DIR, CONFIG_FILE, DATA_DIR
except ImportError:
    AGENT_DIR = Path.home() / ".familiar"
    CONFIG_FILE = AGENT_DIR / "config.yaml"
    DATA_DIR = AGENT_DIR / "data"

TEMPLATES_DIR = Path(__file__).parent / "templates"
STATIC_DIR = Path(__file__).parent / "static"

app = Flask(__name__, template_folder=str(TEMPLATES_DIR), static_folder=str(STATIC_DIR))
app.config["MAX_CONTENT_LENGTH"] = 16 * 1024 * 1024  # 16 MB request body limit

# Configure CORS with restricted origins
_allowed_origins = os.environ.get(
    "FAMILIAR_ALLOWED_ORIGINS", "http://localhost:5000,http://127.0.0.1:5000"
)
_origins_list = [o.strip() for o in _allowed_origins.split(",") if o.strip()]
CORS(app, origins=_origins_list)


# ============================================================
# CSRF Protection
# ============================================================


@app.before_request
def csrf_protect():
    """Validate Origin/Referer on state-changing requests."""
    if request.method in ("POST", "PUT", "DELETE", "PATCH"):
        origin = request.headers.get("Origin") or request.headers.get("Referer", "")
        if not origin:
            # Reject requests with no Origin/Referer — prevents CSRF from
            # scripts that strip these headers.
            logger.warning(f"CSRF check failed: no Origin/Referer from {request.remote_addr}")
            abort(403, description="Origin header required")

        from urllib.parse import urlparse

        parsed = urlparse(origin)
        origin_host = f"{parsed.scheme}://{parsed.netloc}"
        if origin_host not in _origins_list:
            logger.warning(
                f"CSRF check failed: origin={origin_host} "
                f"not in allowed={_origins_list} from {request.remote_addr}"
            )
            abort(403, description="Origin not allowed")


# ============================================================
# Authentication
# ============================================================


def require_auth(f):
    """
    Require API key authentication for dashboard endpoints.

    API key can be provided via:
    - X-API-Key header
    - api_key query parameter

    If FAMILIAR_DASHBOARD_KEY is not set, authentication is disabled
    (with a warning logged on first request).
    """

    @wraps(f)
    def decorated(*args, **kwargs):
        # First-run bypass — no auth needed during onboarding setup
        if _first_run_mode:
            return f(*args, **kwargs)

        expected_key = os.environ.get("FAMILIAR_DASHBOARD_KEY")

        if not expected_key:
            # Persist auto-generated key via keyring so restarts don't break sessions
            if not getattr(require_auth, "_auto_key", None):
                import secrets as _secrets

                try:
                    from familiar.core.credential_store import keyring_get, keyring_set

                    stored = keyring_get("dashboard", "api_key")
                    if stored:
                        require_auth._auto_key = stored
                        logger.info("Loaded dashboard auth key from keyring")
                    else:
                        require_auth._auto_key = _secrets.token_urlsafe(32)
                        keyring_set("dashboard", "api_key", require_auth._auto_key)
                        logger.info("Generated and persisted dashboard auth key in keyring")
                except Exception:
                    # Fallback to plain file if keyring fails
                    _key_path = Path.home() / ".familiar" / "data" / "dashboard_key"
                    if _key_path.exists():
                        require_auth._auto_key = _key_path.read_text().strip()
                    else:
                        require_auth._auto_key = _secrets.token_urlsafe(32)
                        _key_path.parent.mkdir(parents=True, exist_ok=True)
                        _key_path.write_text(require_auth._auto_key)
                        _key_path.chmod(0o600)
                print(f"\n{'=' * 60}")
                print("  Dashboard auth key:")
                print(f"   {require_auth._auto_key}")
                print("   Add to requests as X-API-Key header or ?api_key= param")
                print("   Set FAMILIAR_DASHBOARD_KEY env var to override")
                print(f"{'=' * 60}\n")
            expected_key = require_auth._auto_key

        # Check for API key in header or query param
        provided_key = request.headers.get("X-API-Key") or request.args.get("api_key")

        if not provided_key:
            logger.warning(f"Unauthenticated request to {request.path} from {request.remote_addr}")
            abort(401, description="API key required. Provide via X-API-Key header.")

        # Constant-time comparison to prevent timing attacks
        import hmac

        if not hmac.compare_digest(provided_key, expected_key):
            logger.warning(f"Invalid API key from {request.remote_addr}")
            abort(401, description="Invalid API key")

        return f(*args, **kwargs)

    return decorated


# Optional: Rate limiting (requires flask-limiter)
try:
    from flask_limiter import Limiter
    from flask_limiter.util import get_remote_address

    limiter = Limiter(
        get_remote_address, app=app, default_limits=["200 per minute"], storage_uri="memory://"
    )
    _has_limiter = True
    logger.info("Rate limiting enabled")
except ImportError:
    _has_limiter = False

    # Create a no-op decorator
    class _NoOpLimiter:
        def limit(self, *args, **kwargs):
            def decorator(f):
                return f

            return decorator

    limiter = _NoOpLimiter()
    logger.debug("flask-limiter not installed - rate limiting disabled")

# Global references (set by run_dashboard)
_agent = None
_gateway = None
_wizard_host = None
_wizard_lock = threading.Lock()

# Scheduled message queue — briefings, heartbeats, reminders pushed here
_scheduled_messages = collections.deque(maxlen=50)

# Persistent latest briefing — survives page refreshes, shows until replaced
_BRIEFING_FILE = DATA_DIR / "latest_briefing.json"


def _save_briefing(message: str, source: str = "scheduled", sections: dict = None):
    """Persist the latest briefing to disk so it survives restarts.

    When sections is provided (from _generate_briefing_full), the full
    structured data is saved so /api/briefing can serve it without
    regenerating.  Plain-text saves (from scheduler) omit sections.
    """
    data = {
        "message": message,
        "source": source,
        "timestamp": datetime.now().isoformat(),
    }
    if sections is not None:
        data["sections"] = sections
    try:
        _BRIEFING_FILE.parent.mkdir(parents=True, exist_ok=True)
        _BRIEFING_FILE.write_text(json.dumps(data, indent=2, default=str))
    except OSError as e:
        logger.warning("Failed to save briefing: %s", e)


def _load_briefing() -> dict:
    """Load the latest briefing from disk."""
    if _BRIEFING_FILE.exists():
        try:
            return json.loads(_BRIEFING_FILE.read_text())
        except (json.JSONDecodeError, OSError):
            pass
    return {}


def get_agent():
    global _agent
    if _agent is None:
        from familiar.core.agent import Agent

        _agent = Agent()
    return _agent


_marketplace_catalog = None
_service_manager = None
_service_lock = threading.Lock()


def _get_service_manager():
    """Get or create the service manager singleton."""
    global _service_manager
    with _service_lock:
        if _service_manager is None:
            from familiar.services.manager import ServiceManager

            _service_manager = ServiceManager()
    return _service_manager


def _get_marketplace_catalog():
    global _marketplace_catalog
    if _marketplace_catalog is None:
        from familiar.marketplace.catalog import MarketplaceCatalog

        _marketplace_catalog = MarketplaceCatalog(auto_seed=True)
    return _marketplace_catalog


def _get_message_bridge():
    """Get the message bridge from the agent's gateway, or None."""
    try:
        agent = get_agent()
        gw = getattr(agent, "_gateway", None)
        if gw and hasattr(gw, "message_bridge") and gw.message_bridge:
            return gw.message_bridge
    except Exception:
        pass
    return None


def get_wizard_host():
    """Get or create the connect-wizard adapter for the dashboard."""
    global _wizard_host
    with _wizard_lock:
        if _wizard_host is None:
            from familiar.channels.dashboard_wizard import DashboardWizardHost

            _wizard_host = DashboardWizardHost(get_agent())
    return _wizard_host


# ============================================================
# API Routes
# ============================================================


@app.route("/")
def index():
    """Main dashboard page. Redirects to onboarding on first run."""
    if _first_run_mode and not request.args.get("api_key"):
        return redirect("/onboard")

    # Load creature state for server-side template rendering
    creature_name = "Familiar"
    creature_species_label = ""
    creature_color_name = ""
    owner_name = ""
    is_new_setup = False
    try:
        from familiar.creature.palette import hex_to_palette_name
        from familiar.creature.species import SPECIES_INFO
        from familiar.creature.state import load_creature as _load_creature

        creature = _load_creature()
        if creature and creature.name:
            creature_name = creature.name
            info = SPECIES_INFO.get(creature.species)
            creature_species_label = info.label if info else creature.species
            owner_name = creature.owner_name
            creature_color_name = hex_to_palette_name(creature.color_body)
            # First launch: creature exists but hasn't completed dashboard tutorial.
            # Check a server-side flag (not just API keys, since users may have
            # keys set but still be going through onboarding for the first time).
            tutorial_flag = DATA_DIR / ".dashboard_tutorial_done"
            if not tutorial_flag.exists():
                is_new_setup = True
    except Exception as e:
        logger.debug("Error suppressed: %s", e)
    # Version from importlib.metadata, falling back to VERSION file
    version = ""
    try:
        from importlib.metadata import version as _pkg_version

        version = _pkg_version("familiar-agent")
    except Exception:
        pass
    if not version:
        try:
            version = (Path(__file__).resolve().parent.parent / "VERSION").read_text().strip()
        except Exception:
            pass
    # Resolve the dashboard API key so the JS always has it
    injected_api_key = (
        request.args.get("api_key")
        or os.environ.get("FAMILIAR_DASHBOARD_KEY")
        or getattr(require_auth, "_auto_key", "")
        or ""
    )
    resp = make_response(
        render_template(
            "dashboard.html",
            creature_name=creature_name,
            creature_species_label=creature_species_label,
            creature_color_name=creature_color_name,
            owner_name=owner_name,
            is_new_setup=is_new_setup,
            version=version,
            injected_api_key=injected_api_key,
        )
    )
    resp.headers["Cache-Control"] = "no-cache"
    return resp


@app.route("/api/version/check")
@require_auth
def api_version_check():
    """Check PyPI for a newer version of familiar-agent."""
    try:
        from familiar.core.health import get_health_checker

        hc = get_health_checker()
        if "pypi_update" in hc._dependency_checks:
            result = hc._dependency_checks["pypi_update"]()
            return jsonify(result.details)
    except Exception:
        pass
    import familiar

    return jsonify({
        "current_version": familiar.__version__,
        "latest_version": familiar.__version__,
        "update_available": False,
    })


@app.route("/api/status")
@require_auth
def api_status():
    """Get system status."""
    agent = get_agent()
    status = agent.get_status()

    # Add more details
    status["timestamp"] = datetime.now().isoformat()
    status["uptime"] = _get_uptime()

    return jsonify(status)


@app.route("/api/scheduled-messages")
@require_auth
def api_scheduled_messages():
    """Poll for scheduled messages (briefings, heartbeats, reminders).

    Returns and drains queued messages so each is delivered once.
    The frontend polls this every few seconds when the chat panel is visible.
    """
    messages = []
    while _scheduled_messages:
        try:
            messages.append(_scheduled_messages.popleft())
        except IndexError:
            break
    return jsonify({"messages": messages})


@app.route("/api/latest-briefing")
@require_auth
def api_latest_briefing():
    """Get the latest persisted briefing (survives restarts)."""
    return jsonify(_load_briefing())


@app.route("/api/latest-briefing/refresh", methods=["POST"])
@require_auth
def api_latest_briefing_refresh():
    """Trigger a fresh briefing on demand and persist it."""
    try:
        agent = get_agent()
        prompt = (
            "Give me my morning briefing. Include: "
            "today's calendar events, pending tasks and deadlines, "
            "weather if available, and any unread messages or notifications."
        )
        response = agent.chat(prompt, user_id="scheduler", channel="internal")
        if response:
            _save_briefing(response, source="manual")
            return jsonify({"ok": True, "message": response})
        return jsonify({"ok": False, "error": "Empty response from agent"})
    except Exception as e:
        logger.error("Briefing refresh failed: %s", e)
        return jsonify({"ok": False, "error": str(e)}), 500


# ============================================================
# Calendar
# ============================================================


@app.route("/api/calendar/events")
@require_auth
def api_calendar_events():
    """Get calendar events for the dashboard Calendar panel."""
    days = request.args.get("days", "7", type=str)
    view = request.args.get("view", "week", type=str)

    try:
        days_int = int(days)
    except (ValueError, TypeError):
        days_int = 7

    # Clamp days based on view
    if view == "day":
        days_int = max(days_int, 1)
    elif view == "month":
        days_int = max(days_int, 28)

    events: list[dict] = []
    accounts: list[dict] = []
    try:
        from familiar.skills.calendar.skill import get_events_structured

        events = get_events_structured({"date": "today", "days": days_int})
    except Exception as e:
        logger.error("Calendar events fetch failed: %s", e)

    try:
        from familiar.skills.calendar.accounts import get_all_accounts

        raw_accounts = get_all_accounts()
        # Return safe subset (no passwords/tokens)
        accounts = [
            {"type": a.get("type", ""), "label": a.get("label", ""), "id": a.get("id", "")}
            for a in raw_accounts
        ]
    except Exception as e:
        logger.warning("Calendar accounts fetch failed: %s", e)

    return jsonify({"events": events, "accounts": accounts, "view": view})


@app.route("/api/provider-health")
@require_auth
def api_provider_health():
    """Check configured provider health — surfaces billing/credit issues."""
    import httpx

    results = {}
    _checks = {
        "anthropic": {
            "env_key": "ANTHROPIC_API_KEY",
            "url": "https://api.anthropic.com/v1/messages",
            "billing_url": "https://console.anthropic.com/settings/plans",
            "method": "post",
            "headers_fn": lambda k: {
                "x-api-key": k,
                "anthropic-version": "2023-06-01",
                "content-type": "application/json",
            },
            "body": {
                "model": "claude-haiku-4-5-20251001",
                "max_tokens": 1,
                "messages": [{"role": "user", "content": "ping"}],
            },
        },
        "openai": {
            "env_key": "OPENAI_API_KEY",
            "url": "https://api.openai.com/v1/models",
            "billing_url": "https://platform.openai.com/account/billing",
            "method": "get",
            "headers_fn": lambda k: {"Authorization": f"Bearer {k}"},
            "body": None,
        },
        "gemini": {
            "env_key": "GEMINI_API_KEY",
            "url": None,  # key appended as param
            "billing_url": "https://aistudio.google.com/billing",
            "method": "get",
            "headers_fn": lambda k: {},
            "body": None,
        },
    }

    # Load keys from agent env file
    env_path = AGENT_DIR / ".env"
    env_vars = {}
    if env_path.exists():
        for line in env_path.read_text().splitlines():
            if "=" in line and not line.strip().startswith("#"):
                k, _, v = line.partition("=")
                env_vars[k.strip()] = v.strip()

    for provider, cfg in _checks.items():
        api_key = os.environ.get(cfg["env_key"]) or env_vars.get(cfg["env_key"])
        if not api_key:
            results[provider] = {"status": "not_configured"}
            continue

        try:
            with httpx.Client(timeout=10) as client:
                if provider == "gemini":
                    url = (
                        f"https://generativelanguage.googleapis.com"
                        f"/v1beta/models?key={api_key}"
                    )
                    resp = client.get(url)
                else:
                    headers = cfg["headers_fn"](api_key)
                    if cfg["method"] == "post":
                        resp = client.post(
                            cfg["url"], headers=headers, json=cfg["body"]
                        )
                    else:
                        resp = client.get(cfg["url"], headers=headers)

                body_text = resp.text.lower()
                if resp.status_code == 200 or resp.status_code == 201:
                    results[provider] = {"status": "ok"}
                elif any(
                    kw in body_text
                    for kw in [
                        "credit balance",
                        "insufficient",
                        "billing",
                        "payment required",
                        "purchase credits",
                    ]
                ):
                    results[provider] = {
                        "status": "billing_error",
                        "billing_url": cfg["billing_url"],
                        "message": (
                            f"{provider.title()} account has insufficient credits."
                        ),
                    }
                elif resp.status_code == 401:
                    results[provider] = {
                        "status": "auth_error",
                        "message": "Invalid API key",
                    }
                else:
                    results[provider] = {"status": "ok"}
        except Exception as e:
            results[provider] = {
                "status": "connection_error",
                "message": str(e)[:100],
            }

    # Check Ollama
    try:
        with httpx.Client(timeout=5) as client:
            resp = client.get("http://localhost:11434/api/tags")
            results["ollama"] = {
                "status": "ok" if resp.status_code == 200 else "unavailable"
            }
    except Exception:
        results["ollama"] = {"status": "unavailable"}

    # Build warnings list for providers with billing issues
    warnings = []
    for provider, result in results.items():
        if result.get("status") == "billing_error":
            warnings.append({
                "provider": provider,
                "message": result["message"],
                "billing_url": result.get("billing_url"),
            })

    return jsonify({
        "providers": results,
        "warnings": warnings,
        "timestamp": datetime.now().isoformat(),
    })


@app.route("/api/credentials/health")
@require_auth
def api_credentials_health():
    """Credential health summary — lightweight cached-status read."""
    from familiar.core.health import get_credential_health_summary

    summary = get_credential_health_summary()
    return jsonify(summary)


@app.route("/api/credentials/status")
@require_auth
def api_credentials_status():
    """All credential statuses with spec metadata for dashboard cards."""
    from familiar.core.credential_registry import CREDENTIAL_REGISTRY, is_configured
    from familiar.core.credential_health import get_credential_status

    category = request.args.get("category")
    results = []
    for key, spec in CREDENTIAL_REGISTRY.items():
        if category and spec.category != category:
            continue
        status = get_credential_status(key)
        results.append({
            "service_key": key,
            "display_name": spec.display_name,
            "category": spec.category,
            "credential_type": spec.credential_type.value,
            "reauth_type": spec.reauth_type.value,
            "doc_url": spec.doc_url,
            "doc_hint": spec.doc_hint,
            "sensitivity": spec.sensitivity.value,
            "configured": status.configured,
            "valid": status.valid,
            "attention_needed": status.attention_needed,
            "last_validated": status.last_validated,
            "last_error": status.last_error,
        })
    return jsonify({"credentials": results})


@app.route("/api/credentials/status/<service_key>")
@require_auth
def api_credential_status_single(service_key):
    """Single credential status with spec metadata."""
    from familiar.core.credential_registry import get_credential_spec
    from familiar.core.credential_health import get_credential_status

    spec = get_credential_spec(service_key)
    if spec is None:
        return jsonify({"error": f"Unknown service: {service_key}"}), 404
    status = get_credential_status(service_key)
    return jsonify({
        "service_key": service_key,
        "display_name": spec.display_name,
        "category": spec.category,
        "credential_type": spec.credential_type.value,
        "reauth_type": spec.reauth_type.value,
        "doc_url": spec.doc_url,
        "doc_hint": spec.doc_hint,
        "sensitivity": spec.sensitivity.value,
        "configured": status.configured,
        "valid": status.valid,
        "attention_needed": status.attention_needed,
        "last_validated": status.last_validated,
        "last_error": status.last_error,
    })


@app.route("/api/credentials/validate/<service_key>", methods=["POST"])
@require_auth
def api_credential_validate(service_key):
    """Trigger manual validation for a single credential."""
    from familiar.core.credential_registry import get_credential_spec
    from familiar.core.credential_health import ensure_credentials, invalidate_credential

    spec = get_credential_spec(service_key)
    if spec is None:
        return jsonify({"error": f"Unknown service: {service_key}"}), 404

    # Force re-validation by invalidating cache first
    invalidate_credential(service_key)
    status = ensure_credentials(service_key)
    return jsonify({
        "service_key": service_key,
        "valid": status.valid,
        "attention_needed": status.attention_needed,
        "last_validated": status.last_validated,
        "last_error": status.last_error,
        "configured": status.configured,
    })


@app.route("/api/credentials/update/<service_key>", methods=["POST"])
@require_auth
def api_credential_update(service_key):
    """Accept new credential values, validate, save to .env, and clear attention.

    Body: {"values": {"ENV_VAR_NAME": "new_value", ...}}

    Follows the Zapier pattern: validate BEFORE saving — never store untested
    credentials.
    """
    from familiar.core.credential_registry import get_credential_spec, is_configured
    from familiar.core.credential_health import (
        ensure_credentials,
        invalidate_credential,
        clear_attention,
    )

    spec = get_credential_spec(service_key)
    if spec is None:
        return jsonify({"error": f"Unknown service: {service_key}"}), 404

    body = request.get_json(silent=True) or {}
    values = body.get("values", {})
    if not values:
        return jsonify({"error": "No values provided"}), 400

    # Only accept env vars that belong to this service
    allowed_keys = set(spec.env_vars)
    invalid_keys = [k for k in values if k not in allowed_keys]
    if invalid_keys:
        return jsonify({
            "error": f"Invalid keys for {service_key}: {invalid_keys}",
            "allowed": list(allowed_keys),
        }), 400

    # Step 1: Temporarily set env vars for validation
    import os as _os

    old_values = {}
    for k, v in values.items():
        old_values[k] = _os.environ.get(k)
        _os.environ[k] = v

    # Step 2: Validate with new values
    invalidate_credential(service_key)
    status = ensure_credentials(service_key)

    if not status.valid:
        # Restore old values — don't save invalid credentials
        for k, old_v in old_values.items():
            if old_v is None:
                _os.environ.pop(k, None)
            else:
                _os.environ[k] = old_v
        invalidate_credential(service_key)
        return jsonify({
            "error": "Validation failed — credentials not saved",
            "valid": False,
            "last_error": status.last_error,
        }), 422

    # Step 3: Persist to .env file
    try:
        env_path = Path.home() / ".familiar" / ".env"
        from familiar.channels.connect_wizard._helpers import _update_env_file

        _update_env_file(env_path, values)
    except Exception as e:
        logger.warning("Failed to persist credentials to .env: %s", e)
        # Values are already in os.environ and validated — not fatal

    # Step 4: Clear attention flag
    clear_attention(service_key)

    return jsonify({
        "service_key": service_key,
        "valid": True,
        "saved": True,
        "last_validated": status.last_validated,
    })


@app.route("/api/credentials/fields/<service_key>")
@require_auth
def api_credential_fields(service_key):
    """Return the field definitions for a credential update form.

    Used by the dashboard to render the correct modal (simple_token vs
    multi_field) with field labels and masked current values.
    """
    import os as _os
    from familiar.core.credential_registry import get_credential_spec

    spec = get_credential_spec(service_key)
    if spec is None:
        return jsonify({"error": f"Unknown service: {service_key}"}), 404

    fields = []
    for env_var in spec.env_vars:
        current = _os.environ.get(env_var, "")
        masked = ""
        if current:
            if len(current) > 8:
                masked = current[:4] + "•" * (len(current) - 8) + current[-4:]
            else:
                masked = "•" * len(current)
        # Derive a human-friendly label from the env var name
        label = env_var.replace("_", " ").title()
        is_secret = any(
            kw in env_var.lower()
            for kw in ["key", "token", "secret", "password", "pin"]
        )
        fields.append({
            "env_var": env_var,
            "label": label,
            "masked_value": masked,
            "has_value": bool(current),
            "is_secret": is_secret,
        })

    return jsonify({
        "service_key": service_key,
        "display_name": spec.display_name,
        "reauth_type": spec.reauth_type.value,
        "doc_url": spec.doc_url,
        "doc_hint": spec.doc_hint,
        "fields": fields,
    })


@app.route("/api/credentials/audit")
@require_auth
def api_credentials_audit():
    """Return credential audit events, optionally filtered."""
    try:
        from familiar.core.credential_audit import get_credential_events

        service_key = request.args.get("service_key")
        event_type = request.args.get("event_type")
        since = request.args.get("since")
        limit = min(int(request.args.get("limit", 100)), 500)

        events = get_credential_events(
            service_key=service_key,
            event_type=event_type,
            since=since,
            limit=limit,
        )
        return jsonify({
            "count": len(events),
            "events": [e.to_dict() for e in events],
        })
    except Exception as e:
        logger.exception("Failed to get credential audit events")
        return jsonify({"error": str(e)}), 500


@app.route("/api/credentials/attention")
@require_auth
def api_credentials_attention():
    """Lightweight polling endpoint — returns services needing user attention."""
    try:
        from familiar.core.credential_health import get_attention_needed

        items = get_attention_needed()
        return jsonify({
            "count": len(items),
            "services": [
                {
                    "service_key": s.service_key,
                    "last_error": s.last_error,
                }
                for s in items
            ],
        })
    except Exception as e:
        logger.exception("Failed to get credential attention list")
        return jsonify({"error": str(e)}), 500


@app.route("/api/credentials/dismiss/<service_key>", methods=["POST"])
@require_auth
def api_credential_dismiss(service_key):
    """Clear the attention_needed flag for a credential (session dismiss)."""
    try:
        from familiar.core.credential_health import clear_attention

        clear_attention(service_key)
        return jsonify({"status": "dismissed", "service_key": service_key})
    except Exception as e:
        logger.exception("Failed to dismiss credential attention for %s", service_key)
        return jsonify({"error": str(e)}), 500


@app.route("/api/agent/restart", methods=["POST"])
@require_auth
def api_agent_restart():
    """Gracefully restart the agent process via os.execv."""
    import sys
    import threading

    reason = (request.json or {}).get("reason", "manual restart via dashboard")
    logger.info("Agent restart requested: %s", reason)

    def _do_restart():
        import time
        time.sleep(0.5)  # Let the response flush
        # Spawn a trampoline: wait 2s for the parent socket to close,
        # then exec the real process. close_fds=True avoids inheriting
        # the bound socket; start_new_session=True detaches from this group.
        trampoline = (
            "import time, os, sys; "
            "time.sleep(2); "
            "os.execv(sys.argv[1], sys.argv[1:])"
        )
        subprocess.Popen(
            [sys.executable, "-c", trampoline, sys.executable] + sys.argv,
            close_fds=True,
            start_new_session=True,
        )
        os._exit(0)  # Exit this process; socket released, trampoline waits then binds

    threading.Thread(target=_do_restart, daemon=True).start()
    return jsonify({"status": "restarting", "reason": reason})


# ============================================================
# Prometheus Metrics Endpoint
# ============================================================


@app.route("/metrics")
def prometheus_metrics():
    """Expose metrics in Prometheus text exposition format."""
    # Optionally gate behind auth
    if os.environ.get("FAMILIAR_METRICS_AUTH"):
        from functools import wraps

        auth = request.headers.get("Authorization", "")
        if not auth:
            return "Unauthorized", 401

    try:
        from familiar.core.metrics import get_metrics_collector

        mc = get_metrics_collector()
        lines = []

        # Tool metrics
        for tm in mc.get_all_tool_metrics().values():
            safe = tm.tool_name.replace("-", "_").replace(".", "_").replace("/", "_")
            lines.append(f"familiar_tool_{safe}_calls_total {tm.total_calls}")
            lines.append(f"familiar_tool_{safe}_success_total {tm.successful_calls}")
            lines.append(f"familiar_tool_{safe}_duration_ms_avg {tm.avg_duration_ms:.2f}")
            lines.append(f"familiar_tool_{safe}_duration_ms_p95 {tm.p95_duration_ms:.2f}")
            lines.append(f"familiar_tool_{safe}_timeouts_total {tm.timeout_count}")

        # LLM metrics
        for lm in mc.get_all_llm_metrics().values():
            safe = f"{lm.provider}_{lm.model}".replace("-", "_").replace(".", "_").replace(
                "/", "_"
            ).replace(":", "_")
            lines.append(f"familiar_llm_{safe}_calls_total {lm.total_calls}")
            lines.append(f"familiar_llm_{safe}_success_total {lm.successful_calls}")
            lines.append(f"familiar_llm_{safe}_tokens_in_total {lm.total_prompt_tokens}")
            lines.append(f"familiar_llm_{safe}_tokens_out_total {lm.total_completion_tokens}")
            lines.append(f"familiar_llm_{safe}_cost_usd_total {lm.total_cost_usd:.6f}")
            lines.append(f"familiar_llm_{safe}_duration_ms_avg {lm.avg_duration_ms:.2f}")
            lines.append(f"familiar_llm_{safe}_duration_ms_p95 {lm.p95_duration_ms:.2f}")
            lines.append(f"familiar_llm_{safe}_rate_limits_total {lm.rate_limit_errors}")

        body = "\n".join(lines) + "\n"
        return app.response_class(body, mimetype="text/plain; version=0.0.4; charset=utf-8")
    except Exception:
        return app.response_class("", mimetype="text/plain")


# ============================================================
# Activity Monitor Endpoints
# ============================================================


@app.route("/api/activity/metrics")
@require_auth
def api_activity_metrics():
    """Get tool and LLM performance metrics with alerts."""
    try:
        from familiar.core.metrics import get_metrics_collector

        mc = get_metrics_collector()

        # Tool metrics sorted by call count
        tools = []
        for tm in sorted(
            mc.get_all_tool_metrics().values(), key=lambda t: t.total_calls, reverse=True
        ):
            tools.append({
                "tool_name": tm.tool_name,
                "total_calls": tm.total_calls,
                "success_rate": round(tm.success_rate, 4),
                "avg_duration_ms": round(tm.avg_duration_ms, 1),
                "p95_duration_ms": round(tm.p95_duration_ms, 1),
                "timeout_count": tm.timeout_count,
                "last_error": tm.last_error,
                "recent_durations": [round(d, 1) for d in tm.durations[-20:]],
            })

        # LLM provider metrics — merge in-memory (current session) with
        # persistent analytics.db so local Ollama calls always appear.
        llm_providers = []
        _seen_models = set()
        for lm in sorted(
            mc.get_all_llm_metrics().values(), key=lambda m: m.total_calls, reverse=True
        ):
            _seen_models.add(lm.model)
            llm_providers.append({
                "provider": lm.provider,
                "model": lm.model,
                "total_calls": lm.total_calls,
                "success_rate": round(lm.success_rate, 4),
                "avg_duration_ms": round(lm.avg_duration_ms, 1),
                "p95_duration_ms": round(lm.p95_duration_ms, 1),
                "total_prompt_tokens": lm.total_prompt_tokens,
                "total_completion_tokens": lm.total_completion_tokens,
                "total_cost_usd": round(lm.total_cost_usd, 6),
                "rate_limit_errors": lm.rate_limit_errors,
                "last_error": lm.last_error,
                "tool_call_rate": round(lm.tool_call_rate, 4)
                if lm.tool_call_rate is not None
                else None,
                "empty_tool_retries": lm.empty_tool_retries,
                "calls_with_tools": lm.calls_with_tools_provided,
                "tool_calls_produced": lm.total_tool_calls_produced,
                "recent_durations": [round(d, 1) for d in lm.durations[-20:]],
            })

        # Merge persistent analytics.db data for models not in current session
        try:
            from familiar.core.analytics import get_analytics_store
            _astore = get_analytics_store()
            with _astore._connect() as _aconn:
                _db_models = _aconn.execute(
                    """
                    SELECT model,
                           COUNT(*) as calls,
                           COALESCE(SUM(tokens_in), 0) as tok_in,
                           COALESCE(SUM(tokens_out), 0) as tok_out,
                           COALESCE(SUM(cost_usd), 0.0) as cost,
                           COALESCE(AVG(latency_ms), 0) as avg_lat
                    FROM events
                    WHERE event_type = 'message'
                      AND model != '' AND model NOT LIKE '%Mock%'
                    GROUP BY model
                    ORDER BY calls DESC
                    """
                ).fetchall()
                for row in _db_models:
                    _m = row["model"]
                    if _m in _seen_models:
                        continue
                    _prov = "ollama"
                    if "claude" in _m.lower() or "sonnet" in _m.lower() or "haiku" in _m.lower():
                        _prov = "anthropic"
                    elif "gpt" in _m.lower():
                        _prov = "openai"
                    elif "gemini" in _m.lower():
                        _prov = "google"
                    llm_providers.append({
                        "provider": _prov,
                        "model": _m,
                        "total_calls": row["calls"],
                        "success_rate": 1.0,
                        "avg_duration_ms": round(row["avg_lat"], 1),
                        "p95_duration_ms": 0,
                        "total_prompt_tokens": row["tok_in"],
                        "total_completion_tokens": row["tok_out"],
                        "total_cost_usd": round(row["cost"], 6),
                        "rate_limit_errors": 0,
                        "last_error": None,
                        "tool_call_rate": None,
                        "empty_tool_retries": 0,
                        "calls_with_tools": 0,
                        "tool_calls_produced": 0,
                        "recent_durations": [],
                    })
        except Exception:
            pass  # analytics.db unavailable — show in-memory only

        # Alerts
        alerts = [a.to_dict() for a in mc.get_alerts()]

        # Rollup KPIs
        tool_summary = mc.get_tool_summary()
        llm_summary = mc.get_llm_summary()

        # Compute overall tool call rate across all providers
        all_llm = mc.get_all_llm_metrics()
        _tcr_calls = sum(m.calls_with_tools_provided for m in all_llm.values())
        _tcr_produced = sum(m.total_tool_calls_produced for m in all_llm.values())
        _overall_tcr = (_tcr_produced / _tcr_calls) if _tcr_calls > 0 else None
        _total_retries = sum(m.empty_tool_retries for m in all_llm.values())

        return jsonify({
            "tools": tools,
            "llm_providers": llm_providers,
            "alerts": alerts,
            "kpi": {
                "total_tool_calls": tool_summary.get("total_calls", 0),
                "tool_success_rate": round(tool_summary.get("overall_success_rate", 1.0), 4),
                "total_tokens": llm_summary.get("total_tokens", 0),
                "total_cost_usd": round(llm_summary.get("total_cost_usd", 0.0), 6),
                "tool_call_rate": round(_overall_tcr, 4) if _overall_tcr is not None else None,
                "empty_tool_retries": _total_retries,
            },
        })
    except Exception:
        logger.debug("Activity metrics unavailable", exc_info=True)
        return jsonify({
            "tools": [], "llm_providers": [], "alerts": [],
            "kpi": {"total_tool_calls": 0, "tool_success_rate": 1.0,
                    "total_tokens": 0, "total_cost_usd": 0.0},
        })


@app.route("/api/activity/errors")
@require_auth
def api_activity_errors():
    """Get recent error events from analytics store."""
    try:
        from familiar.core.analytics import EventType, get_analytics_store

        store = get_analytics_store()
        events = store.query(event_type=EventType.ERROR, limit=50)
        errors = []
        for e in events:
            _meta = e.metadata if isinstance(e.metadata, dict) else {}
            errors.append({
                "timestamp": e.timestamp.isoformat() if e.timestamp else "",
                "skill": e.skill or "",
                "channel": e.channel or "",
                "model": e.model or "",
                "error_message": e.error_message or "",
                "latency_ms": e.latency_ms,
                "request_id": _meta.get("request_id", ""),
            })
        return jsonify({"errors": errors})
    except Exception:
        logger.debug("Activity errors unavailable", exc_info=True)
        return jsonify({"errors": []})


@app.route("/api/activity/skill-costs")
@require_auth
def api_activity_skill_costs():
    """Get per-skill cost and usage breakdown."""
    try:
        from datetime import timedelta

        from familiar.core.analytics import get_analytics_store

        store = get_analytics_store()
        days = int(request.args.get("days", 30))
        start = datetime.now(timezone.utc) - timedelta(days=days)
        summary = store.get_summary(start=start)
        skills = []
        for name, data in sorted(
            summary.by_skill.items(), key=lambda x: x[1]["cost"], reverse=True
        ):
            skills.append({
                "skill": name,
                "calls": data["calls"],
                "cost": round(data["cost"], 6),
                "avg_latency": round(data["avg_latency"], 1),
            })
        return jsonify({"skills": skills})
    except Exception:
        logger.debug("Skill costs unavailable", exc_info=True)
        return jsonify({"skills": []})


@app.route("/api/activity/timeseries")
@require_auth
def api_activity_timeseries():
    """Get daily cost and token time-series for charting."""
    try:
        from datetime import timedelta

        from familiar.core.analytics import get_analytics_store

        store = get_analytics_store()
        days = int(request.args.get("days", 30))
        start = datetime.now(timezone.utc) - timedelta(days=days)
        costs = store.get_daily_costs(start=start)
        tokens = store.get_daily_tokens(start=start)
        return jsonify({"costs": costs, "tokens": tokens})
    except Exception:
        logger.debug("Timeseries unavailable", exc_info=True)
        return jsonify({"costs": [], "tokens": []})


@app.route("/api/activity/kpis")
@require_auth
def api_activity_kpis():
    """Get persistent KPIs from analytics.db (today/month/all-time)."""
    try:
        from familiar.core.analytics import get_analytics_store

        store = get_analytics_store()
        return jsonify(store.get_period_kpis())
    except Exception:
        logger.debug("KPIs unavailable", exc_info=True)
        return jsonify({
            "today": {"calls": 0, "tokens_in": 0, "tokens_out": 0, "cost": 0},
            "month": {"calls": 0, "tokens_in": 0, "tokens_out": 0, "cost": 0},
            "all_time": {"calls": 0, "tokens_in": 0, "tokens_out": 0, "cost": 0},
            "active_model": "",
        })


@app.route("/api/activity/call-log")
@require_auth
def api_activity_call_log():
    """Get paginated LLM call log with trigger classification."""
    try:
        from familiar.core.analytics import get_analytics_store

        store = get_analytics_store()
        page = int(request.args.get("page", 1))
        per_page = int(request.args.get("per_page", 50))
        start = request.args.get("start")
        end = request.args.get("end")
        trigger = request.args.get("trigger")

        calls, total = store.get_call_log(
            page=page, per_page=per_page, start=start, end=end, trigger=trigger
        )
        pages = max(1, (total + per_page - 1) // per_page)
        return jsonify({"calls": calls, "total": total, "page": page, "pages": pages})
    except Exception:
        logger.debug("Call log unavailable", exc_info=True)
        return jsonify({"calls": [], "total": 0, "page": 1, "pages": 1})


@app.route("/api/activity/monthly-spend")
@require_auth
def api_activity_monthly_spend():
    """Get monthly spend analysis with proactive vs interactive breakdown."""
    try:
        from familiar.core.analytics import get_analytics_store

        store = get_analytics_store()
        return jsonify(store.get_monthly_spend())
    except Exception:
        logger.debug("Monthly spend unavailable", exc_info=True)
        return jsonify({
            "total": 0, "daily_avg": 0, "projected": 0,
            "proactive_cost": 0, "interactive_cost": 0,
            "last_month_total": 0, "change_pct": 0,
            "days_elapsed": 0, "days_in_month": 30,
        })


@app.route("/api/activity/health")
@require_auth
def api_activity_health():
    """Get system health: resources, dependencies, circuit breakers."""
    result = {"status": "unknown", "uptime_seconds": 0,
              "dependencies": [], "resources": {}, "circuits": {},
              "routing": {}}
    try:
        from familiar.core.health import get_health_checker

        hc = get_health_checker()
        health = hc.check_health(include_resources=True)
        hdict = health.to_dict()
        result["status"] = hdict.get("status", "unknown")
        result["uptime_seconds"] = hdict.get("uptime_seconds", 0)
        result["resources"] = hdict.get("resources", {})
        result["dependencies"] = hdict.get("dependencies", [])
    except Exception:
        logger.debug("Health status unavailable", exc_info=True)

    try:
        from familiar.core.resilience import get_all_circuit_status

        result["circuits"] = get_all_circuit_status()
    except Exception:
        logger.debug("Circuit breaker status unavailable", exc_info=True)

    # Routing context: active provider + fallback chain + rate limits
    try:
        agent = get_agent()
        result["routing"]["active_provider"] = agent.provider.name
        result["routing"]["active_provider_key"] = getattr(
            agent.provider, "provider_key", ""
        )
        result["routing"]["active_model"] = getattr(
            agent.provider, "model_name", ""
        )
        if hasattr(agent, "providers") and agent.providers.model_router:
            router = agent.providers.model_router
            result["routing"]["fallback_chain"] = list(
                getattr(router, "_fallback_chain", [])
            )
            router_models = router.get_model_names()
            # Merge in all Ollama models (catalog + installed)
            try:
                from familiar.channels.connect_wizard._llm import OLLAMA_MODEL_CATALOG

                catalog_set = {f"ollama/{tag}" for tag, *_ in OLLAMA_MODEL_CATALOG}
            except ImportError:
                catalog_set = set()
            try:
                import httpx as _httpx

                resp = _httpx.get("http://localhost:11434/api/tags", timeout=3)
                installed_set = {
                    f"ollama/{m.get('name', '')}"
                    for m in resp.json().get("models", [])
                    if m.get("name")
                }
            except Exception:
                installed_set = set()
            all_models = set(router_models) | catalog_set | installed_set
            result["routing"]["available_models"] = sorted(all_models)
        if hasattr(agent, "providers"):
            tracker = agent.providers.capacity_tracker
            result["rate_limits"] = tracker.get_status()
    except Exception:
        logger.debug("Routing info unavailable", exc_info=True)

    return jsonify(result)


@app.route("/api/creature")
@require_auth
def api_creature():
    """Get creature state and rendered sprite for the dashboard."""
    from familiar.creature.palette import build_color_map, hex_to_palette_name
    from familiar.creature.renderer_html import render_frame_html
    from familiar.creature.species import SPECIES_INFO
    from familiar.creature.sprites import get_sprite_frames
    from familiar.creature.state import load_creature

    creature = load_creature()
    if creature is None:
        return jsonify({"exists": False})

    colors = build_color_map(creature.color_body, creature.color_eyes, creature.color_accent)
    frames = get_sprite_frames(creature.species, creature.evolution_stage, "idle")
    frame = frames[0] if frames else []
    sprite_html = render_frame_html(frame, colors)

    species_info = SPECIES_INFO.get(creature.species)
    label = species_info.label if species_info else creature.species
    color_name = hex_to_palette_name(creature.color_body)

    return jsonify(
        {
            "exists": True,
            "name": creature.name,
            "species": label,
            "species_key": creature.species,
            "color_name": color_name,
            "color_body": creature.color_body,
            "color_eyes": creature.color_eyes,
            "color_accent": creature.color_accent,
            "color_voice": creature.color_voice,
            "stage": creature.evolution_stage,
            "interactions": creature.total_interactions,
            "born": creature.born_at[:10] if creature.born_at else "",
            "sprite_html": sprite_html,
            "owner_name": creature.owner_name,
        }
    )


@app.route("/api/creature", methods=["POST"])
@require_auth
def api_creature_update():
    """Update creature name, species, or colors."""
    from familiar.creature.palette import (
        build_color_map,
        hex_to_palette_name,
        resolve_color,
        rgb_to_hex,
    )
    from familiar.creature.renderer_html import render_frame_html
    from familiar.creature.species import SPECIES_INFO, SPECIES_ORDER, get_species_info
    from familiar.creature.sprites import get_sprite_frames
    from familiar.creature.state import load_creature, save_creature

    creature = load_creature()
    if creature is None:
        return jsonify({"error": "No creature exists"}), 404

    data = request.json or {}

    # Validate and apply name
    if "name" in data:
        name = str(data["name"]).strip()
        if not name or len(name) > 32:
            return jsonify({"error": "Name must be 1-32 characters"}), 400
        creature.name = name

    # Validate and apply species
    if "species" in data:
        sp = str(data["species"]).strip().lower()
        if get_species_info(sp) is None:
            return jsonify({"error": f"Unknown species: {sp}"}), 400
        creature.species = sp

    # Validate and apply evolution stage
    if "evolution_stage" in data:
        stage = str(data["evolution_stage"]).strip().lower()
        if stage not in ("baby", "juvenile", "adult"):
            return jsonify({"error": "Stage must be baby, juvenile, or adult"}), 400
        creature.evolution_stage = stage

    # Validate and apply total_interactions (for reset)
    if "total_interactions" in data:
        try:
            count = int(data["total_interactions"])
            if count < 0:
                raise ValueError
            creature.total_interactions = count
        except (ValueError, TypeError):
            return jsonify({"error": "Invalid interaction count"}), 400

    # Validate and apply colors
    for field in ("color_body", "color_eyes", "color_accent", "color_voice"):
        if field in data:
            raw = str(data[field]).strip()
            rgb = resolve_color(raw)
            if rgb is None:
                return jsonify({"error": f"Invalid color for {field}: {raw}"}), 400
            setattr(creature, field, rgb_to_hex(*rgb))

    save_creature(creature)

    # Return refreshed state
    colors = build_color_map(creature.color_body, creature.color_eyes, creature.color_accent)
    frames = get_sprite_frames(creature.species, creature.evolution_stage, "idle")
    frame = frames[0] if frames else []
    sprite_html = render_frame_html(frame, colors)

    species_info = SPECIES_INFO.get(creature.species)
    label = species_info.label if species_info else creature.species
    color_name = hex_to_palette_name(creature.color_body)

    return jsonify(
        {
            "exists": True,
            "name": creature.name,
            "species": label,
            "species_key": creature.species,
            "color_name": color_name,
            "color_body": creature.color_body,
            "color_eyes": creature.color_eyes,
            "color_accent": creature.color_accent,
            "color_voice": creature.color_voice,
            "stage": creature.evolution_stage,
            "interactions": creature.total_interactions,
            "born": creature.born_at[:10] if creature.born_at else "",
            "sprite_html": sprite_html,
            "owner_name": creature.owner_name,
        }
    )


@app.route("/api/creature/preview")
@require_auth
def api_creature_preview():
    """Render a sprite preview without saving. For live preview in settings."""
    from familiar.creature.palette import build_color_map
    from familiar.creature.renderer_html import render_frame_html
    from familiar.creature.species import get_species_info
    from familiar.creature.sprites import get_sprite_frames
    from familiar.creature.state import load_creature

    creature = load_creature()
    stage = request.args.get("stage", creature.evolution_stage if creature else "baby")
    if stage not in ("baby", "juvenile", "adult"):
        stage = "baby"

    species = request.args.get("species", creature.species if creature else "fox")
    if get_species_info(species) is None:
        species = "fox"
    color_body = request.args.get("color_body", creature.color_body if creature else "#4A90D9")
    color_eyes = request.args.get("color_eyes", creature.color_eyes if creature else "#FFD700")
    color_accent = request.args.get(
        "color_accent", creature.color_accent if creature else "#FF6B6B"
    )

    colors = build_color_map(color_body, color_eyes, color_accent)
    frames = get_sprite_frames(species, stage, "idle")
    frame = frames[0] if frames else []
    sprite_html = render_frame_html(frame, colors)

    return jsonify({"sprite_html": sprite_html})


@app.route("/api/chat", methods=["POST"])
@limiter.limit("20 per minute")  # Stricter limit for expensive LLM calls
@require_auth
def api_chat():
    """Send a message to the agent."""
    data = request.json or {}
    message = data.get("message", "")
    files = data.get("files", [])

    if not message and not files:
        return jsonify({"error": "No message provided"}), 400

    # Prepend file context so the agent sees attached file paths
    if files:
        lines = ["[Attached files — use read_document or appropriate filereader tool to process]"]
        for f in files:
            fname = f.get("filename", "unknown")
            fpath = f.get("path", "")
            fsize = f.get("size", 0)
            if fsize >= 1024 * 1024:
                size_str = f"{fsize / (1024 * 1024):.1f} MB"
            elif fsize >= 1024:
                size_str = f"{fsize / 1024:.1f} KB"
            else:
                size_str = f"{fsize} B"
            lines.append(f"  - {fname} ({size_str}): {fpath}")
        file_context = "\n".join(lines) + "\n\n"
        message = file_context + (message or "Please analyze the attached file(s).")

    # Route /connect commands to the wizard
    stripped = message.strip()
    if stripped.lower().startswith("/connect"):
        args = stripped.split()[1:]  # drop the "/connect" prefix
        host = get_wizard_host()
        messages = host.start_wizard(args)
        return jsonify({"wizard_messages": messages, "timestamp": datetime.now().isoformat()})

    # Route /start to the tarot onboarding spread
    if stripped.lower().startswith("/start"):
        host = get_wizard_host()
        messages = host.start()
        return jsonify({"wizard_messages": messages, "timestamp": datetime.now().isoformat()})

    # If wizard is active, intercept text input (unless caller requests agent directly)
    force_agent = data.get("force_agent", False)
    if not force_agent:
        host = get_wizard_host()
        if host.is_active():
            messages = host.send_text(message)
            return jsonify({"wizard_messages": messages, "timestamp": datetime.now().isoformat()})

    # Action Router — handle common messages without involving the LLM.
    # This lets even a 0.5B model work because it never sees "show my email".
    if not force_agent and not files:
        try:
            from familiar.dashboard.action_router import match_action

            action = match_action(stripped)
            if action is not None:
                return jsonify({
                    "action": action,
                    "response": action.get("response", ""),
                    "timestamp": datetime.now().isoformat(),
                })
        except Exception as e:
            logger.debug("Action router error (falling through to LLM): %s", e)

    agent = get_agent()

    # Enforce budget config on every chat request so restarts/reloads can't revert it.
    if hasattr(agent, "sessions"):
        _sess = agent.sessions.get_or_create_session("dashboard", "web")
        _budget_enabled = False
        _budget_amount = 10.0
        try:
            import yaml as _yaml
            _cfg_path = CONFIG_FILE
            if _cfg_path.exists():
                with open(_cfg_path) as _f:
                    _cfg = _yaml.safe_load(_f) or {}
                _budget_enabled = _cfg.get("agent", {}).get("budget_enabled", False)
                _budget_amount = float(_cfg.get("agent", {}).get("daily_budget_amount", 10.0))
        except Exception:
            pass
        _target_budget = _budget_amount if _budget_enabled else 999999.0
        if _sess.daily_budget != _target_budget:
            _sess.daily_budget = _target_budget

    result = agent.chat_with_results(message, user_id="dashboard", channel="web")

    response_data = {"response": result.text, "timestamp": datetime.now().isoformat()}

    # Surface billing warnings when the response indicates a provider issue
    response_lower = (result.text or "").lower()
    if any(
        kw in response_lower
        for kw in ["insufficient credits", "credit balance", "top up"]
    ):
        _billing_urls = {
            "anthropic": "https://console.anthropic.com/settings/plans",
            "openai": "https://platform.openai.com/account/billing",
            "gemini": "https://aistudio.google.com/billing",
        }
        provider_name = getattr(
            getattr(agent, "provider", None), "provider_key", "anthropic"
        )
        response_data["provider_warning"] = {
            "type": "billing",
            "provider": provider_name,
            "message": f"{provider_name.title()} account needs credits",
            "billing_url": _billing_urls.get(provider_name),
        }

    return jsonify(response_data)


@app.route("/api/roundtable", methods=["POST"])
@limiter.limit("10 per minute")
@require_auth
def api_roundtable():
    """Fan a message out to multiple job class agents in parallel and return all responses."""
    import concurrent.futures

    data = request.json or {}
    message = (data.get("message") or "").strip()
    seats = data.get("seats") or []  # list of job_class_key strings

    if not message:
        return jsonify({"error": "No message provided"}), 400
    if not seats:
        return jsonify({"error": "No seats selected"}), 400

    from familiar.jobs import get_job_class, list_job_classes

    valid_keys = {jc.key for jc in list_job_classes()}
    seats = [s for s in seats if s in valid_keys]
    if not seats:
        return jsonify({"error": "No valid job class seats provided"}), 400

    # Hard cap: cloud providers handle 9 fine; local Ollama serialises anyway
    # so no benefit beyond ~3, but we allow up to all 9 job classes.
    MAX_SEATS = 9
    seats = seats[:MAX_SEATS]

    agent = get_agent()

    # Detect whether we're on a local provider so the UI can surface a warning
    provider_name = getattr(getattr(agent, "provider", None), "name", "") or ""
    is_local = "ollama" in provider_name.lower()

    def _ask_seat(job_class_key: str) -> dict:
        jc = get_job_class(job_class_key)
        try:
            result = agent.chat_with_results(
                message,
                user_id=f"roundtable-{job_class_key}",
                channel="web",
                include_history=False,
                job_class_key=job_class_key,
            )
            return {
                "job_class_key": job_class_key,
                "name": jc.name if jc else job_class_key,
                "icon": jc.icon if jc else "fa-user",
                "response": result.text,
                "error": None,
            }
        except Exception as e:
            logger.error("Round table seat %s failed: %s", job_class_key, e)
            return {
                "job_class_key": job_class_key,
                "name": jc.name if jc else job_class_key,
                "icon": jc.icon if jc else "fa-user",
                "response": None,
                "error": str(e),
            }

    # Local Ollama serialises internally; cap parallelism at 3 to avoid piling up
    effective_workers = 3 if is_local else len(seats)

    with concurrent.futures.ThreadPoolExecutor(max_workers=effective_workers) as pool:
        futures = {pool.submit(_ask_seat, key): key for key in seats}
        responses = [f.result() for f in concurrent.futures.as_completed(futures)]

    # Return in same order as seats were requested
    order = {key: i for i, key in enumerate(seats)}
    responses.sort(key=lambda r: order.get(r["job_class_key"], 99))

    return jsonify({
        "responses": responses,
        "timestamp": datetime.now().isoformat(),
        "provider": provider_name,
        "is_local": is_local,
    })


@app.route("/api/roundtable/convene", methods=["POST"])
@limiter.limit("10 per minute")
@require_auth
def api_roundtable_convene():
    """
    Chairman-led Round Table session.

    Phase 1 — Chairman routing: determine which advisors to call and how to frame
              the question for each.
    Phase 2 — Parallel fan-out: each advisor responds with their Chairman framing.
    Phase 3 — Chairman synthesis: Chairman reads all responses and delivers a
              unified recommendation.
    """
    import concurrent.futures
    import json
    import re

    data = request.json or {}
    message = (data.get("message") or "").strip()
    if not message:
        return jsonify({"error": "No message provided"}), 400

    from familiar.jobs import get_job_class, list_job_classes
    from familiar.jobs.chairman import build_team_manifest

    agent = get_agent()
    provider_name = getattr(getattr(agent, "provider", None), "name", "") or ""
    is_local = "ollama" in provider_name.lower()

    # ── Phase 1: Chairman routing ────────────────────────────────────────────
    team_manifest = build_team_manifest()
    routing_message = (
        f"{team_manifest}\n\n"
        f"**User's question:** {message}"
    )

    routing_result = agent.chat_with_results(
        routing_message,
        user_id="roundtable-chairman",
        channel="web",
        include_history=False,
        job_class_key="chairman",
    )

    # Parse Chairman's routing JSON
    raw = routing_result.text or ""
    match = re.search(r"```json\s*(.*?)\s*```", raw, re.DOTALL)
    routing = {}
    if match:
        try:
            routing = json.loads(match.group(1))
        except json.JSONDecodeError:
            pass

    # If Chairman answered directly, return immediately
    if routing.get("action") == "direct":
        return jsonify({
            "mode": "direct",
            "chairman_intro": routing.get("response", raw),
            "responses": [],
            "chairman_synthesis": None,
            "timestamp": datetime.now().isoformat(),
        })

    # Extract seats and per-seat framings
    valid_keys = {jc.key for jc in list_job_classes()}
    seats = [s for s in (routing.get("seats") or []) if s in valid_keys]
    framings = routing.get("framings") or {}
    chairman_intro = routing.get("intro") or raw

    if not seats:
        # Chairman gave a malformed response — fall back to direct
        return jsonify({
            "mode": "direct",
            "chairman_intro": raw,
            "responses": [],
            "chairman_synthesis": None,
            "timestamp": datetime.now().isoformat(),
        })

    # ── Phase 2: Parallel fan-out ────────────────────────────────────────────
    def _ask_seat(job_class_key: str) -> dict:
        jc = get_job_class(job_class_key)
        framing = framings.get(job_class_key, "")
        framed_message = f"{framing}\n\n{message}" if framing else message
        try:
            result = agent.chat_with_results(
                framed_message,
                user_id=f"roundtable-{job_class_key}",
                channel="web",
                include_history=False,
                job_class_key=job_class_key,
            )
            return {
                "job_class_key": job_class_key,
                "name": jc.name if jc else job_class_key,
                "icon": jc.icon if jc else "fa-user",
                "response": result.text,
                "error": None,
            }
        except Exception as e:
            logger.error("Convene seat %s failed: %s", job_class_key, e)
            return {
                "job_class_key": job_class_key,
                "name": jc.name if jc else job_class_key,
                "icon": jc.icon if jc else "fa-user",
                "response": None,
                "error": str(e),
            }

    effective_workers = 3 if is_local else len(seats)
    with concurrent.futures.ThreadPoolExecutor(max_workers=effective_workers) as pool:
        futures = {pool.submit(_ask_seat, key): key for key in seats}
        responses = [f.result() for f in concurrent.futures.as_completed(futures)]

    # Preserve seat order
    order = {key: i for i, key in enumerate(seats)}
    responses.sort(key=lambda r: order.get(r["job_class_key"], 99))

    # ── Phase 3: Chairman synthesis ──────────────────────────────────────────
    synthesis_parts = [
        team_manifest,
        f"\n**Original question:** {message}\n",
        "**Advisor responses:**\n",
    ]
    for resp in responses:
        if resp.get("response"):
            synthesis_parts.append(f"**{resp['name']}:** {resp['response']}\n")

    synthesis_parts.append(
        "\nYou have heard from the advisors. Now provide your synthesis: "
        "what they agreed on, where they diverged, and your clear recommendation. "
        "Write in plain prose, 2-4 sentences, no JSON."
    )

    synthesis_result = agent.chat_with_results(
        "\n".join(synthesis_parts),
        user_id="roundtable-chairman-synthesis",
        channel="web",
        include_history=False,
        job_class_key="chairman",
    )

    return jsonify({
        "mode": "convene",
        "chairman_intro": chairman_intro,
        "responses": responses,
        "chairman_synthesis": synthesis_result.text,
        "timestamp": datetime.now().isoformat(),
        "provider": provider_name,
        "is_local": is_local,
    })


# ── Round Table Groups ────────────────────────────────────────────────────────

_RT_GROUPS_FILE = DATA_DIR / "roundtable_groups.json"


def _load_rt_groups() -> list:
    try:
        if _RT_GROUPS_FILE.exists():
            return json.loads(_RT_GROUPS_FILE.read_text()) or []
    except Exception:
        pass
    return []


def _save_rt_groups(groups: list) -> None:
    _RT_GROUPS_FILE.parent.mkdir(parents=True, exist_ok=True)
    _RT_GROUPS_FILE.write_text(json.dumps(groups, indent=2))


@app.route("/api/roundtable/groups", methods=["GET"])
@require_auth
def api_roundtable_groups_list():
    """List all saved advisor groups."""
    return jsonify({"groups": _load_rt_groups()})


@app.route("/api/roundtable/groups", methods=["POST"])
@require_auth
def api_roundtable_groups_save():
    """Save or update a named advisor group."""
    data = request.json or {}
    name = (data.get("name") or "").strip()
    seats = data.get("seats") or []

    if not name:
        return jsonify({"error": "Group name is required"}), 400
    if not seats:
        return jsonify({"error": "At least one seat is required"}), 400

    from familiar.jobs import list_job_classes
    valid_keys = {jc.key for jc in list_job_classes()}
    seats = [s for s in seats if s in valid_keys]
    if not seats:
        return jsonify({"error": "No valid seats provided"}), 400

    groups = _load_rt_groups()
    # Update if name exists, otherwise append
    for g in groups:
        if g["name"] == name:
            g["seats"] = seats
            _save_rt_groups(groups)
            return jsonify({"saved": True, "group": g})

    group = {"name": name, "seats": seats}
    groups.append(group)
    _save_rt_groups(groups)
    return jsonify({"saved": True, "group": group})


@app.route("/api/roundtable/groups/<name>", methods=["DELETE"])
@require_auth
def api_roundtable_groups_delete(name):
    """Delete a named advisor group."""
    groups = [g for g in _load_rt_groups() if g["name"] != name]
    _save_rt_groups(groups)
    return jsonify({"deleted": True})


@app.route("/api/voice/stt", methods=["POST"])
@limiter.limit("10 per minute")
@require_auth
def api_voice_stt():
    """Transcribe uploaded audio and send transcript to agent chat."""
    import tempfile as _tempfile

    if "audio" not in request.files:
        return jsonify({"error": "No audio file provided"}), 400

    audio_file = request.files["audio"]
    tmp_path = None
    try:
        fd, tmp_path = _tempfile.mkstemp(suffix=".webm")
        os.close(fd)
        audio_file.save(tmp_path)

        from familiar.skills.voice.skill import _transcribe_with_whisper

        transcript = _transcribe_with_whisper(tmp_path)
        text = transcript.text.strip()
        if not text:
            return jsonify({"error": "No speech detected"}), 400

        agent = get_agent()
        result = agent.chat_with_results(text, user_id="dashboard", channel="web")

        return jsonify(
            {
                "transcript": text,
                "response": result.text,
                "timestamp": datetime.now().isoformat(),
            }
        )
    except Exception:
        logger.exception("Voice STT error")
        return jsonify({"error": "Internal server error"}), 500
    finally:
        if tmp_path:
            try:
                os.unlink(tmp_path)
            except OSError as e:
                logger.debug("I/O error suppressed: %s", e)


@app.route("/api/voice/tts", methods=["POST"])
@limiter.limit("20 per minute")
@require_auth
def api_voice_tts():
    """Generate speech audio from text."""
    import tempfile as _tempfile

    data = request.json or {}
    text = data.get("text", "").strip()
    if not text:
        return jsonify({"error": "No text provided"}), 400

    # Try Piper first (local, high-quality WAV)
    try:
        from familiar.skills.voice.document_reader import speak_with_piper

        fd, tmp_path = _tempfile.mkstemp(suffix=".wav")
        os.close(fd)
        success, result = speak_with_piper(text, output_path=tmp_path, play=False)
        if success:
            return send_file(tmp_path, mimetype="audio/wav")
    except Exception:
        logger.debug("Piper TTS unavailable, trying fallbacks", exc_info=True)

    # Fallback: gTTS (MP3)
    try:
        from gtts import gTTS

        fd, tmp_path = _tempfile.mkstemp(suffix=".mp3")
        os.close(fd)
        gTTS(text=text, lang="en").save(tmp_path)
        return send_file(tmp_path, mimetype="audio/mpeg")
    except Exception:
        logger.debug("gTTS unavailable, trying edge-tts", exc_info=True)

    # Fallback: edge-tts (MP3)
    try:
        import asyncio

        import edge_tts

        fd, tmp_path = _tempfile.mkstemp(suffix=".mp3")
        os.close(fd)
        asyncio.run(edge_tts.Communicate(text, "en-US-AriaNeural").save(tmp_path))
        return send_file(tmp_path, mimetype="audio/mpeg")
    except Exception:
        logger.debug("edge-tts unavailable", exc_info=True)

    return jsonify({"error": "No TTS engine available"}), 503


@app.route("/api/terminal/stream", methods=["POST"])
@limiter.limit("20 per minute")
@require_auth
def api_terminal_stream():
    """Stream a chat response via SSE for the terminal panel."""
    from familiar.core.providers import StreamEventType

    data = request.json or {}
    message = data.get("message", "")

    if not message:
        return jsonify({"error": "No message provided"}), 400

    # Route /connect commands to the wizard (non-streaming)
    stripped = message.strip()
    if stripped.lower().startswith("/connect"):
        args = stripped.split()[1:]
        host = get_wizard_host()
        messages = host.start_wizard(args)
        text = "\n".join(m.get("text", str(m)) for m in messages)

        def _wizard_gen():
            yield f"data: {json.dumps({'type': 'text', 'content': text})}\n\n"
            yield f"data: {json.dumps({'type': 'done'})}\n\n"

        resp = Response(_wizard_gen(), mimetype="text/event-stream")
        resp.headers["Cache-Control"] = "no-cache"
        resp.headers["X-Accel-Buffering"] = "no"
        return resp

    # Route /start to tarot onboarding (non-streaming)
    if stripped.lower().startswith("/start"):
        host = get_wizard_host()
        messages = host.start()
        text = "\n".join(m.get("text", str(m)) for m in messages)

        def _start_gen():
            yield f"data: {json.dumps({'type': 'text', 'content': text})}\n\n"
            yield f"data: {json.dumps({'type': 'done'})}\n\n"

        resp = Response(_start_gen(), mimetype="text/event-stream")
        resp.headers["Cache-Control"] = "no-cache"
        resp.headers["X-Accel-Buffering"] = "no"
        return resp

    # If wizard is active, route text input through it
    host = get_wizard_host()
    if host.is_active():
        messages = host.send_text(message)
        text = "\n".join(m.get("text", str(m)) for m in messages)

        def _wizard_input_gen():
            yield f"data: {json.dumps({'type': 'text', 'content': text})}\n\n"
            yield f"data: {json.dumps({'type': 'done'})}\n\n"

        resp = Response(_wizard_input_gen(), mimetype="text/event-stream")
        resp.headers["Cache-Control"] = "no-cache"
        resp.headers["X-Accel-Buffering"] = "no"
        return resp

    agent = get_agent()
    msg_queue = queue.Queue()

    def _run_stream():
        try:
            for event in agent.chat_stream(message, user_id="terminal", channel="terminal"):
                if event.type == StreamEventType.TEXT:
                    msg_queue.put({"type": "text", "content": event.content})
                elif event.type == StreamEventType.TOOL_START:
                    msg_queue.put({"type": "tool_start", "tool_name": event.tool_name or ""})
                elif event.type == StreamEventType.TOOL_END:
                    msg_queue.put({"type": "tool_end", "tool_name": event.tool_name or ""})
                elif event.type == StreamEventType.ERROR:
                    msg_queue.put({"type": "error", "error": event.error or "Unknown error"})
                elif event.type == StreamEventType.DONE:
                    pass  # Handled after loop
        except Exception as e:
            logger.exception("Terminal stream error")
            msg_queue.put({"type": "error", "error": str(e)})
        finally:
            msg_queue.put(None)  # Sentinel

    t = threading.Thread(target=_run_stream, daemon=True)
    t.start()

    def generate():
        while True:
            try:
                item = msg_queue.get(timeout=120)
            except queue.Empty:
                yield f"data: {json.dumps({'type': 'error', 'error': 'Timeout'})}\n\n"
                yield f"data: {json.dumps({'type': 'done'})}\n\n"
                break
            if item is None:
                yield f"data: {json.dumps({'type': 'done'})}\n\n"
                break
            yield f"data: {json.dumps(item)}\n\n"

    resp = Response(generate(), mimetype="text/event-stream")
    resp.headers["Cache-Control"] = "no-cache"
    resp.headers["X-Accel-Buffering"] = "no"
    return resp


# ============================================================
# Host Shell Terminal
# ============================================================

_shell_proc = None
_shell_lock = threading.Lock()
_shell_enabled_override: bool | None = None  # None = use env var


def _shell_enabled() -> bool:
    if _shell_enabled_override is not None:
        return _shell_enabled_override
    return os.environ.get("FAMILIAR_SHELL_ENABLED", "").lower() in ("1", "true", "yes")


# Patterns that should never be executed via the shell terminal
_DANGEROUS_PATTERNS = [
    r"\brm\s+-[^\s]*r[^\s]*f\s+/\s*$",  # rm -rf /
    r"\brm\s+-[^\s]*f[^\s]*r\s+/\s*$",  # rm -fr /
    r"\brm\s+-rf\s+~",  # rm -rf ~
    r"\bmkfs\b",
    r"\bdd\s+if=",
    r">\s*/dev/sd[a-z]",
    r":\(\)\s*\{\s*:\|:\s*&\s*\}\s*;",  # fork bomb
    r"\bcurl\b.*\|\s*\b(sh|bash|zsh)\b",  # curl|sh
    r"\bwget\b.*\|\s*\b(sh|bash|zsh)\b",  # wget|sh
    r"\bsudo\b",
    r"\bchmod\s+777\b",
    r"\bpython[23]?\s+-c\b",
    r"\beval\b",
    r"\bnc\s+-[^\s]*e\b",  # nc -e reverse shell
    r"\bbase64\s+-d\b.*\|\s*\b(sh|bash)\b",  # base64 -d|sh
    r"/dev/tcp/",
    r"\biptables\b",
    r"\bsystemctl\b",
    r">\s*/etc/",
]


@app.route("/api/shell/enabled")
@require_auth
def api_shell_enabled():
    """Check if the host shell terminal is enabled."""
    return jsonify({"enabled": _shell_enabled()})


@app.route("/api/shell/enabled", methods=["POST"])
@require_auth
def api_shell_toggle():
    """Toggle the host shell terminal on or off."""
    global _shell_enabled_override

    data = request.json or {}
    enabled = data.get("enabled")
    if not isinstance(enabled, bool):
        return jsonify({"error": '"enabled" must be a boolean'}), 400

    _shell_enabled_override = enabled

    # Persist to ~/.familiar/.env so the setting survives restarts
    env_path = AGENT_DIR / ".env"
    try:
        from familiar.channels.connect_wizard import _update_env_file

        _update_env_file(env_path, {"FAMILIAR_SHELL_ENABLED": "true" if enabled else "false"})
    except Exception:
        logger.debug("Could not persist shell toggle to .env", exc_info=True)

    return jsonify({"enabled": enabled})


@app.route("/api/settings/pii")
@require_auth
def api_pii_status():
    """Check if PII/PHI detection is enabled."""
    agent = get_agent()
    enabled = getattr(agent.config.agent, "enable_pii_detection", False)
    return jsonify({"enabled": enabled})


@app.route("/api/settings/pii", methods=["POST"])
@require_auth
def api_pii_toggle():
    """Toggle PII/PHI detection on or off. Manual control only."""
    data = request.json or {}
    enabled = data.get("enabled")
    if not isinstance(enabled, bool):
        return jsonify({"error": '"enabled" must be a boolean'}), 400

    agent = get_agent()

    # Update running agent config
    agent.config.agent.enable_pii_detection = enabled

    # Update guardrails live
    from familiar.core.guardrails import PIIConfig, PIIDetector
    if enabled:
        if not agent.guardrails._pii_detector:
            agent.guardrails._pii_detector = PIIDetector(PIIConfig())
            agent.guardrails.pii_config = agent.guardrails._pii_detector.config
    else:
        agent.guardrails._pii_detector = None
        agent.guardrails.pii_config = None

    # Persist to config.yaml
    config_path = CONFIG_FILE
    try:
        import yaml
        config_data = {}
        if config_path.exists():
            with open(config_path) as f:
                config_data = yaml.safe_load(f) or {}
        config_data.setdefault("agent", {})["enable_pii_detection"] = enabled
        with open(config_path, "w") as f:
            yaml.safe_dump(config_data, f, default_flow_style=False)
    except Exception:
        logger.debug("Could not persist PII toggle to config.yaml", exc_info=True)

    logger.info(f"PII detection {'enabled' if enabled else 'disabled'} via Settings toggle")
    return jsonify({"enabled": enabled})


@app.route("/api/settings/budget")
@require_auth
def api_budget_status():
    """Return current budget settings and today's spend."""
    config_path = CONFIG_FILE
    budget_enabled = True  # default: off (no limit)
    daily_budget_amount = 10.0
    try:
        import yaml
        if config_path.exists():
            with open(config_path) as f:
                cfg = yaml.safe_load(f) or {}
            budget_enabled = cfg.get("agent", {}).get("budget_enabled", False)
            daily_budget_amount = cfg.get("agent", {}).get("daily_budget_amount", 10.0)
    except Exception:
        pass

    spent_today = 0.0
    agent = get_agent()
    if agent and hasattr(agent, "sessions"):
        session = agent.sessions.get_or_create_session("dashboard", "web")
        session._check_budget_reset()
        spent_today = round(session.spent_today, 4)

    return jsonify({
        "enabled": budget_enabled,
        "daily_budget": daily_budget_amount,
        "spent_today": spent_today,
    })


@app.route("/api/settings/budget", methods=["POST"])
@require_auth
def api_budget_update():
    """Enable/disable daily budget limit and set amount."""
    data = request.json or {}
    enabled = data.get("enabled")
    if not isinstance(enabled, bool):
        return jsonify({"error": '"enabled" must be a boolean'}), 400

    daily_budget_amount = data.get("daily_budget")
    if daily_budget_amount is not None:
        if not isinstance(daily_budget_amount, (int, float)) or daily_budget_amount < 0:
            return jsonify({"error": '"daily_budget" must be a non-negative number'}), 400
        daily_budget_amount = float(daily_budget_amount)
    else:
        # Read existing value from config
        config_path = CONFIG_FILE
        try:
            import yaml
            if config_path.exists():
                with open(config_path) as f:
                    cfg = yaml.safe_load(f) or {}
                daily_budget_amount = cfg.get("agent", {}).get("daily_budget_amount", 10.0)
            else:
                daily_budget_amount = 10.0
        except Exception:
            daily_budget_amount = 10.0

    # Apply to live session
    agent = get_agent()
    if agent and hasattr(agent, "sessions"):
        session = agent.sessions.get_or_create_session("dashboard", "web")
        if enabled:
            session.daily_budget = daily_budget_amount
        else:
            session.daily_budget = 999999.0  # effectively unlimited
        agent.sessions.save_session(session)

    # Persist to config.yaml
    config_path = CONFIG_FILE
    try:
        import yaml
        config_data = {}
        if config_path.exists():
            with open(config_path) as f:
                config_data = yaml.safe_load(f) or {}
        config_data.setdefault("agent", {})["budget_enabled"] = enabled
        config_data["agent"]["daily_budget_amount"] = daily_budget_amount
        with open(config_path, "w") as f:
            yaml.safe_dump(config_data, f, default_flow_style=False)
    except Exception:
        logger.debug("Could not persist budget settings to config.yaml", exc_info=True)

    logger.info(f"Budget {'enabled ($%.2f/day)' % daily_budget_amount if enabled else 'disabled (unlimited)'} via Settings")
    return jsonify({"enabled": enabled, "daily_budget": daily_budget_amount})


@app.route("/api/settings/briefing")
@require_auth
def api_briefing_settings_get():
    """Get current briefing schedule settings."""
    try:
        from familiar.skills.proactive.skill import _load_config as _load_proactive_config
        cfg = _load_proactive_config()
        return jsonify({
            "enabled": cfg.get("briefing_enabled", False),
            "time": cfg.get("briefing_time", "08:00"),
        })
    except Exception as e:
        logger.warning("Briefing settings get failed: %s", e)
        return jsonify({"enabled": False, "time": "08:00"})


@app.route("/api/settings/briefing", methods=["POST"])
@require_auth
def api_briefing_settings_post():
    """Save briefing schedule settings."""
    data = request.json or {}
    enabled = data.get("enabled", False)
    time_str = data.get("time", "08:00").strip()

    from datetime import datetime as _dt
    try:
        _dt.strptime(time_str, "%H:%M")
    except ValueError:
        return jsonify({"error": "Invalid time — use HH:MM format"}), 400

    try:
        from familiar.skills.proactive.skill import set_briefing_time
        result = set_briefing_time({"time": time_str, "enabled": enabled})
        return jsonify({"ok": True, "message": result, "enabled": enabled, "time": time_str})
    except Exception as e:
        logger.error("briefing settings save failed: %s", e)
        return jsonify({"error": str(e)}), 500


@app.route("/api/settings/news-topics")
@require_auth
def api_news_topics_get():
    """Get saved news topics."""
    try:
        from familiar.skills.media.news import load_topics
        return jsonify({"topics": load_topics()})
    except Exception as e:
        logger.debug("news topics get failed: %s", e)
        return jsonify({"topics": []})


@app.route("/api/settings/news-topics", methods=["POST"])
@require_auth
def api_news_topics_post():
    """Replace the news topics list (up to 5)."""
    data = request.json or {}
    topics = data.get("topics", [])
    if not isinstance(topics, list):
        return jsonify({"error": "topics must be an array"}), 400
    topics = [str(t).strip().lower() for t in topics if str(t).strip()][:5]
    try:
        from familiar.skills.media.news import save_topics
        save_topics(topics)
        return jsonify({"topics": topics})
    except Exception as e:
        logger.error("news topics save failed: %s", e)
        return jsonify({"error": str(e)}), 500


@app.route("/api/shell/stream", methods=["POST"])
@limiter.limit("10 per minute")
@require_auth
def api_shell_stream():
    """Stream shell command output via SSE."""
    global _shell_proc

    if not _shell_enabled():
        return jsonify(
            {"error": "Shell terminal is disabled. Set FAMILIAR_SHELL_ENABLED=true"}
        ), 403

    data = request.json or {}
    command = data.get("command", "").strip()

    if not command:
        return jsonify({"error": "No command provided"}), 400

    # Allowlist: specific Familiar-initiated commands that require sudo
    # but are known-safe (package installs for server bootstrap)
    _SAFE_SUDO_COMMANDS = [
        r"^sudo\s+(apt-get|apt|dnf|yum|pacman|zypper|apk|brew)\s+install\s+-y\s+podman$",
        r"^sudo\s+(apt-get|apt|dnf|yum|pacman|zypper|apk|brew)\s+install\s+-y\s+podman\s*\|",
        r"^sudo\s+apt-get\s+install\s+-y\s+podman\s+\|\|\s+sudo\s+dnf\s+install\s+-y\s+podman$",
        r"^sudo\s+usermod\s+-aG\s+docker\s+\w+$",
    ]
    is_allowlisted = any(re.search(p, command) for p in _SAFE_SUDO_COMMANDS)

    # Defense-in-depth: block catastrophically destructive patterns
    if not is_allowlisted:
        for pattern in _DANGEROUS_PATTERNS:
            if re.search(pattern, command):
                logger.warning(f"Shell blocked dangerous command: {command!r}")
                return jsonify({"error": "Command blocked for safety"}), 400

    # Sudo password passthrough — only for allowlisted commands.
    # The password is piped to sudo -S via stdin (never logged, never stored).
    sudo_password = data.get("sudo_password", "")
    use_sudo_stdin = bool(sudo_password and is_allowlisted and "sudo" in command)
    if use_sudo_stdin:
        # Rewrite sudo → sudo -S so it reads the password from stdin
        command = command.replace("sudo ", "sudo -S ")

    logger.info("Shell command: %s", command[:200])

    msg_queue = queue.Queue()

    def _run_shell():
        global _shell_proc
        try:
            proc = subprocess.Popen(
                command,
                shell=True,
                stdin=subprocess.PIPE if use_sudo_stdin else None,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                text=True,
                bufsize=1,
                cwd=str(Path.home()),
            )
            with _shell_lock:
                _shell_proc = proc

            # Feed the sudo password via stdin, then close it
            if use_sudo_stdin and proc.stdin:
                try:
                    proc.stdin.write(sudo_password + "\n")
                    proc.stdin.flush()
                    proc.stdin.close()
                except OSError:
                    pass

            for line in proc.stdout:
                # Never echo the password back to the UI
                if sudo_password and sudo_password in line:
                    continue
                msg_queue.put({"type": "output", "content": line})

            proc.wait()
            msg_queue.put({"type": "exit", "code": proc.returncode})
        except Exception as e:
            logger.exception("Shell stream error")
            msg_queue.put({"type": "error", "error": str(e)})
        finally:
            with _shell_lock:
                _shell_proc = None
            msg_queue.put(None)  # Sentinel

    t = threading.Thread(target=_run_shell, daemon=True)
    t.start()

    def generate():
        while True:
            try:
                item = msg_queue.get(timeout=300)  # 5 min idle timeout
            except queue.Empty:
                yield f"data: {json.dumps({'type': 'error', 'error': 'Timeout'})}\n\n"
                yield f"data: {json.dumps({'type': 'done'})}\n\n"
                break
            if item is None:
                yield f"data: {json.dumps({'type': 'done'})}\n\n"
                break
            yield f"data: {json.dumps(item)}\n\n"

    resp = Response(generate(), mimetype="text/event-stream")
    resp.headers["Cache-Control"] = "no-cache"
    resp.headers["X-Accel-Buffering"] = "no"
    return resp


@app.route("/api/shell/kill", methods=["POST"])
@require_auth
def api_shell_kill():
    """Terminate the running shell process."""
    global _shell_proc

    if not _shell_enabled():
        return jsonify({"error": "Shell terminal is disabled"}), 403

    with _shell_lock:
        if _shell_proc and _shell_proc.poll() is None:
            _shell_proc.terminate()
            return jsonify({"success": True, "message": "Process terminated"})
    return jsonify({"success": False, "message": "No running process"})


@app.route("/api/memory")
@require_auth
def api_memory():
    """Get all memories."""
    agent = get_agent()
    if not agent.memory:
        return jsonify({"memories": []})

    memories = []
    for key, entry in agent.memory.recall_all().items():
        memories.append(
            {
                "key": entry.key,
                "value": entry.value,
                "category": entry.category,
                "importance": entry.importance,
                "updated_at": entry.updated_at,
            }
        )

    # Sort by importance
    memories.sort(key=lambda x: x["importance"], reverse=True)

    return jsonify({"memories": memories})


@app.route("/api/memory", methods=["POST"])
@require_auth
def api_memory_add():
    """Add a memory."""
    data = request.json or {}
    key = data.get("key", "")
    value = data.get("value", "")
    category = data.get("category", "fact")

    if not key or not value:
        return jsonify({"error": "Key and value required"}), 400

    agent = get_agent()
    if agent.memory:
        agent.memory.remember(key, value, category=category)
        return jsonify({"success": True})

    return jsonify({"error": "Memory not enabled"}), 400


@app.route("/api/memory/<key>", methods=["DELETE"])
@require_auth
def api_memory_delete(key):
    """Delete a memory."""
    agent = get_agent()
    if agent.memory:
        agent.memory.forget(key)
        return jsonify({"success": True})
    return jsonify({"error": "Memory not enabled"}), 400


# ── Working Memory API ──


@app.route("/api/memory/working")
@require_auth
def api_memory_working():
    """Get the agent's working memory buffer."""
    agent = get_agent()
    items = []
    try:
        mm = agent._memory_manager
        # Try to get the default user's episodic memory system
        for uid, ms in mm._episodic_stores.items():
            wm = ms.working_memory
            for m in wm.get_all():
                items.append({
                    "id": m.id,
                    "content": m.content,
                    "type": m.memory_type.value if hasattr(m.memory_type, "value") else str(m.memory_type),
                    "importance": round(m.importance, 2),
                    "created_at": m.created_at.isoformat() if hasattr(m.created_at, "isoformat") else str(m.created_at),
                    "source": m.source or "",
                    "tags": list(m.tags) if m.tags else [],
                    "status": m.status.value if hasattr(m.status, "value") else str(m.status),
                })
            break  # First user only (dashboard is single-user)
    except Exception:
        pass
    return jsonify({"items": items, "count": len(items)})


# ── Downloads Shelf API ──

_downloads_shelf: list = []  # In-memory list; persists per process lifetime
_downloads_lock = threading.Lock()


@app.route("/api/downloads")
@require_auth
def api_downloads_list():
    """List generated downloads."""
    with _downloads_lock:
        return jsonify({"downloads": list(_downloads_shelf)})


@app.route("/api/downloads/track", methods=["POST"])
@require_auth
def api_downloads_track():
    """Track a newly generated download."""
    import uuid
    from datetime import datetime, timezone

    data = request.get_json(silent=True) or {}
    entry = {
        "id": str(uuid.uuid4())[:8],
        "filename": data.get("filename", "download"),
        "url": data.get("url", ""),
        "type": data.get("type", "file"),  # file, csv, pdf, chart, zip
        "source": data.get("source", ""),  # which endpoint generated it
        "created_at": datetime.now(timezone.utc).isoformat(),
        "size_bytes": data.get("size_bytes", 0),
    }
    with _downloads_lock:
        _downloads_shelf.insert(0, entry)
        # Keep last 50
        if len(_downloads_shelf) > 50:
            _downloads_shelf[:] = _downloads_shelf[:50]
    return jsonify({"download": entry}), 201


@app.route("/api/downloads/<dl_id>", methods=["DELETE"])
@require_auth
def api_downloads_delete(dl_id):
    """Remove a download from the shelf."""
    with _downloads_lock:
        before = len(_downloads_shelf)
        _downloads_shelf[:] = [d for d in _downloads_shelf if d["id"] != dl_id]
        found = len(_downloads_shelf) < before
    if found:
        return jsonify({"status": "ok"})
    return jsonify({"error": "Not found"}), 404


# ── Pins API (cross-tab favorites) ──

_pins_db_path = None


def _get_pins_db():
    """Get the pins SQLite database path, creating if needed."""
    global _pins_db_path
    if _pins_db_path is None:
        from familiar.core.paths import DATA_DIR
        _pins_db_path = DATA_DIR / "pins.db"
    import sqlite3
    conn = sqlite3.connect(str(_pins_db_path))
    conn.execute("""
        CREATE TABLE IF NOT EXISTS pins (
            id TEXT PRIMARY KEY,
            pin_type TEXT NOT NULL,
            title TEXT NOT NULL,
            reference TEXT NOT NULL DEFAULT '',
            namespace TEXT NOT NULL DEFAULT '',
            created_at TEXT NOT NULL,
            UNIQUE(pin_type, reference)
        )
    """)
    conn.commit()
    return conn


@app.route("/api/pins")
@require_auth
def api_pins_list():
    """List all pinned items."""
    conn = _get_pins_db()
    rows = conn.execute(
        "SELECT id, pin_type, title, reference, namespace, created_at FROM pins ORDER BY created_at DESC"
    ).fetchall()
    conn.close()
    return jsonify({
        "pins": [
            {"id": r[0], "pin_type": r[1], "title": r[2], "reference": r[3], "namespace": r[4], "created_at": r[5]}
            for r in rows
        ]
    })


@app.route("/api/pins", methods=["POST"])
@require_auth
def api_pins_add():
    """Pin an item."""
    import uuid
    from datetime import datetime, timezone

    data = request.get_json(silent=True) or {}
    pin_type = data.get("pin_type", "")  # episodic, markdown, file, note
    title = data.get("title", "")
    reference = data.get("reference", "")  # key, slug, file_id
    namespace = data.get("namespace", "")

    if not pin_type or not title:
        return jsonify({"error": "pin_type and title required"}), 400

    conn = _get_pins_db()
    pin_id = str(uuid.uuid4())[:8]
    try:
        conn.execute(
            "INSERT INTO pins (id, pin_type, title, reference, namespace, created_at) VALUES (?, ?, ?, ?, ?, ?)",
            (pin_id, pin_type, title, reference, namespace, datetime.now(timezone.utc).isoformat()),
        )
        conn.commit()
    except Exception:
        conn.close()
        return jsonify({"error": "Already pinned"}), 409
    conn.close()
    return jsonify({"id": pin_id}), 201


@app.route("/api/pins/<pin_id>", methods=["DELETE"])
@require_auth
def api_pins_delete(pin_id):
    """Unpin an item."""
    conn = _get_pins_db()
    cur = conn.execute("DELETE FROM pins WHERE id = ?", (pin_id,))
    conn.commit()
    conn.close()
    if cur.rowcount:
        return jsonify({"status": "ok"})
    return jsonify({"error": "Not found"}), 404


@app.route("/api/pins/check")
@require_auth
def api_pins_check():
    """Check if an item is pinned. Pass pin_type and reference as query params."""
    pin_type = request.args.get("pin_type", "")
    reference = request.args.get("reference", "")
    conn = _get_pins_db()
    row = conn.execute(
        "SELECT id FROM pins WHERE pin_type = ? AND reference = ?",
        (pin_type, reference),
    ).fetchone()
    conn.close()
    return jsonify({"pinned": row is not None, "pin_id": row[0] if row else None})


# ── Markdown Memory (Knowledge + Job-Class Notes) API ──

_md_store_cache: dict = {}
_md_store_lock = threading.Lock()


def _get_md_store(namespace: str = ""):
    """Get a MarkdownMemoryStore, optionally namespaced for job-class notes."""
    cache_key = namespace or "__root__"
    if cache_key not in _md_store_cache:
        with _md_store_lock:
            if cache_key not in _md_store_cache:
                from familiar.core.markdown_memory import MarkdownMemoryStore
                from familiar.core.paths import MEMORIES_DIR

                user_id = "dashboard"
                if namespace:
                    # Job-class notes live under _namespace/ subdir
                    store_dir = MEMORIES_DIR / user_id / f"_{namespace}"
                    store_dir.mkdir(parents=True, exist_ok=True)
                    # Create store with the subdir as its base
                    store = MarkdownMemoryStore.__new__(MarkdownMemoryStore)
                    store._base = store_dir
                    store._lock = threading.RLock()
                    store._alias_index = {}
                    store._rebuild_alias_index()
                else:
                    store = MarkdownMemoryStore(MEMORIES_DIR, user_id)
                _md_store_cache[cache_key] = store
    return _md_store_cache[cache_key]


@app.route("/api/memories/markdown")
@require_auth
def api_md_memories_list():
    """List markdown memory topics."""
    namespace = request.args.get("namespace", "")
    store = _get_md_store(namespace)
    limit = int(request.args.get("limit", 50))
    topics = store.list_topics(limit=limit)
    return jsonify({"topics": topics, "namespace": namespace})


@app.route("/api/memories/markdown", methods=["POST"])
@require_auth
def api_md_memories_save():
    """Create or append to a markdown memory topic."""
    data = request.get_json(silent=True) or {}
    topic = data.get("topic", "").strip()
    content = data.get("content", "").strip()
    namespace = data.get("namespace", "")

    if not topic or not content:
        return jsonify({"error": "topic and content are required"}), 400

    store = _get_md_store(namespace)
    tags = data.get("tags", [])
    heading = data.get("heading", "")
    slug, rev = store.save_memory(
        topic=topic, content=content, tags=tags, heading=heading,
    )
    return jsonify({"slug": slug, "revision": rev}), 201


@app.route("/api/memories/markdown/<slug>")
@require_auth
def api_md_memories_read(slug):
    """Read a markdown memory topic."""
    namespace = request.args.get("namespace", "")
    store = _get_md_store(namespace)
    result = store.read_memory(slug)
    if result is None:
        return jsonify({"error": "Topic not found"}), 404
    meta, body = result
    return jsonify({
        "slug": slug,
        "topic": meta.topic if meta else slug,
        "body": body,
        "tags": meta.tags if meta else [],
        "revision": meta.revision if meta else 1,
        "created": meta.created if meta else "",
        "updated": meta.updated if meta else "",
    })


@app.route("/api/memories/markdown/<slug>", methods=["DELETE"])
@require_auth
def api_md_memories_delete(slug):
    """Delete a markdown memory topic."""
    namespace = request.args.get("namespace", "")
    store = _get_md_store(namespace)
    if store.delete_memory(slug):
        return jsonify({"status": "ok"})
    return jsonify({"error": "Topic not found"}), 404


@app.route("/api/memories/markdown/search")
@require_auth
def api_md_memories_search():
    """Search markdown memories."""
    q = request.args.get("q", "")
    namespace = request.args.get("namespace", "")
    store = _get_md_store(namespace)
    results = store.search(q, limit=20)
    return jsonify({
        "results": [
            {"slug": slug, "score": score, "preview": preview}
            for slug, score, preview in results
        ]
    })


# ── Memory Tree API ──


def _get_memory_tree():
    """Get the MemoryTree singleton."""
    from familiar.core.memory_tree import get_memory_tree

    return get_memory_tree()


@app.route("/api/memory/tree")
@require_auth
def api_memory_tree_list():
    """List memories from the MemoryTree, filtered by tier/tag."""
    tier = request.args.get("tier", "working")
    tag = request.args.get("tag", "")
    limit = min(int(request.args.get("limit", 50)), 200)

    tree = _get_memory_tree()

    if tag:
        entries = tree.list_by_tag(tag, tier=tier if tier != "all" else None, limit=limit)
    elif tier == "all":
        entries = []
        for t in ["working", "short", "long", "archival"]:
            entries.extend(tree.list_by_tier(t, limit=limit))
    else:
        entries = tree.list_by_tier(tier, limit=limit)

    return jsonify({
        "entries": [e.to_dict() for e in entries],
        "tier": tier,
        "stats": tree.stats(),
    })


@app.route("/api/memory/tree/<memory_id>")
@require_auth
def api_memory_tree_get(memory_id):
    """Get a single memory by ID."""
    tree = _get_memory_tree()
    entry = tree.get(memory_id)
    if not entry:
        return jsonify({"error": "Memory not found"}), 404
    return jsonify({"entry": entry.to_dict()})


@app.route("/api/memory/tree", methods=["POST"])
@require_auth
def api_memory_tree_store():
    """Store a new memory in the MemoryTree."""
    data = request.json or {}
    title = data.get("title", "").strip()
    body = data.get("body", "").strip()
    tier = data.get("tier", "working")
    tags = data.get("tags", [])
    importance = float(data.get("importance", 0.5))

    if not title:
        return jsonify({"error": "Title required"}), 400

    tree = _get_memory_tree()
    entry = tree.store(
        title=title, body=body, tier=tier,
        tags=tags, importance=importance,
    )
    return jsonify({"entry": entry.to_dict()}), 201


@app.route("/api/memory/tree/<memory_id>", methods=["PUT"])
@require_auth
def api_memory_tree_update(memory_id):
    """Update an existing memory."""
    data = request.json or {}
    tree = _get_memory_tree()

    entry = tree.update(
        memory_id,
        title=data.get("title"),
        body=data.get("body"),
        importance=data.get("importance"),
        tier=data.get("tier"),
        tags=data.get("tags"),
    )
    if not entry:
        return jsonify({"error": "Memory not found"}), 404
    return jsonify({"entry": entry.to_dict()})


@app.route("/api/memory/tree/<memory_id>", methods=["DELETE"])
@require_auth
def api_memory_tree_delete(memory_id):
    """Soft-delete (invalidate) a memory."""
    data = request.json or {}
    reason = data.get("reason", "")
    tree = _get_memory_tree()

    success = tree.invalidate(memory_id, reason=reason)
    if success:
        return jsonify({"status": "ok"})
    return jsonify({"error": "Memory not found or already invalidated"}), 404


@app.route("/api/memory/tree/search")
@require_auth
def api_memory_tree_search():
    """Search the MemoryTree via FTS5."""
    q = request.args.get("q", "").strip()
    tier = request.args.get("tier")
    limit = min(int(request.args.get("limit", 20)), 100)

    if not q:
        return jsonify({"error": "Query parameter 'q' required"}), 400

    tree = _get_memory_tree()
    results = tree.search(q, tier=tier, limit=limit)
    return jsonify({
        "query": q,
        "results": [e.to_dict() for e in results],
        "total": len(results),
    })


@app.route("/api/memory/tree/<memory_id>/graph")
@require_auth
def api_memory_tree_graph(memory_id):
    """Get the subgraph around a memory (for graph visualization)."""
    depth = min(int(request.args.get("depth", 2)), 5)
    tree = _get_memory_tree()

    entry = tree.get(memory_id)
    if not entry:
        return jsonify({"error": "Memory not found"}), 404

    subgraph = tree.get_subgraph(memory_id, depth=depth)
    return jsonify({
        "nodes": [n.to_dict() for n in subgraph["nodes"]],
        "edges": subgraph["edges"],
    })


@app.route("/api/memory/tree/<memory_id>/history")
@require_auth
def api_memory_tree_history(memory_id):
    """Get the version history for a memory (SUPERSEDES chain)."""
    tree = _get_memory_tree()

    entry = tree.get(memory_id)
    if not entry:
        return jsonify({"error": "Memory not found"}), 404

    history = tree.get_history(memory_id)
    return jsonify({
        "history": [e.to_dict() for e in history],
        "current_id": memory_id,
    })


@app.route("/api/files/tree")
@require_auth
def api_files_tree():
    """Get files grouped by job_class then record_type."""
    store = _get_file_store()
    job_class = request.args.get("job_class")

    if job_class:
        files = store.list_for_job_class(job_class)
    else:
        files = store.search(limit=500)

    tree: dict = {}
    for f in files:
        jc = f.job_class
        rt = f.record_type
        if jc not in tree:
            tree[jc] = {}
        if rt not in tree[jc]:
            tree[jc][rt] = []
        tree[jc][rt].append(f.to_dict())

    return jsonify({"tree": tree})


@app.route("/api/memory/search")
@require_auth
def api_memory_unified_search():
    """Unified search across episodic memories, markdown topics, and files."""
    q = request.args.get("q", "").strip()
    if not q:
        return jsonify({"error": "Query parameter 'q' is required"}), 400

    results = []

    # 1. Episodic memories
    try:
        agent = get_agent()
        if hasattr(agent, "memory") and agent.memory:
            memories = agent.memory
            for m in memories:
                key = m.get("key", "")
                value = m.get("value", "")
                if q.lower() in key.lower() or q.lower() in value.lower():
                    results.append({
                        "type": "episodic",
                        "title": key,
                        "preview": value[:200],
                        "category": m.get("category", ""),
                        "source": "brain",
                    })
    except Exception:
        pass

    # 2. Markdown knowledge
    try:
        md_store = _get_md_store("")
        md_results = md_store.search(q, limit=10)
        for slug, score, preview in md_results:
            results.append({
                "type": "markdown",
                "title": slug,
                "preview": preview[:200] if preview else "",
                "score": score,
                "source": "knowledge",
            })
    except Exception:
        pass

    # 3. MemoryTree (FTS5)
    try:
        tree = _get_memory_tree()
        tree_results = tree.search(q, limit=10)
        for entry in tree_results:
            results.append({
                "type": "memory_tree",
                "title": entry.title,
                "preview": entry.body[:200] if entry.body else "",
                "tier": entry.tier,
                "importance": entry.importance,
                "memory_id": entry.memory_id,
                "source": "memory_tree",
            })
    except Exception:
        pass

    # 4. Files
    try:
        file_store = _get_file_store()
        files = file_store.search(query=q, limit=10)
        for f in files:
            results.append({
                "type": "file",
                "title": f.filename,
                "preview": f"{f.job_class}/{f.record_type} · {f.mime_type}",
                "file_id": f.file_id,
                "source": "files",
            })
    except Exception:
        pass

    return jsonify({"query": q, "results": results, "total": len(results)})


@app.route("/api/tasks")
@require_auth
def api_tasks():
    """Get all tasks."""
    tasks_file = DATA_DIR / "tasks.json"
    if not tasks_file.exists():
        return jsonify({"tasks": []})

    try:
        tasks = json.loads(tasks_file.read_text())
        return jsonify({"tasks": tasks})
    except (json.JSONDecodeError, IOError, OSError):
        return jsonify({"tasks": []})


@app.route("/api/tasks", methods=["POST"])
@require_auth
def api_task_add():
    """Add a task."""
    data = request.json or {}

    # Use the tasks skill
    try:
        import sys

        sys.path.insert(0, str(Path(__file__).parent.parent / "skills" / "tasks"))
        from skill import add_task, _load_tasks

        result = add_task(data)
        # Return the newest task for ID access
        tasks = _load_tasks()
        newest = tasks[-1] if tasks else {}
        return jsonify({"result": result, "task": newest})
    except Exception:
        return jsonify({"error": "Failed to add task"}), 500


@app.route("/api/tasks/<task_id>/complete", methods=["POST"])
@require_auth
def api_task_complete(task_id):
    """Complete a task."""
    try:
        import sys

        sys.path.insert(0, str(Path(__file__).parent.parent / "skills" / "tasks"))
        from skill import complete_task

        result = complete_task({"id": task_id})
        return jsonify({"result": result})
    except Exception:
        return jsonify({"error": "Failed to complete task"}), 500


@app.route("/api/skills")
@require_auth
def api_skills():
    """Get loaded skills with prerequisite status."""
    agent = get_agent()
    from familiar.core.skills import check_skill_prerequisites

    prereq_failures = check_skill_prerequisites()
    skills = []
    for s in agent.skills.list_skills():
        name = s["name"]
        failures = prereq_failures.get(name, [])
        if s["enabled"] and not failures:
            status = "connected"
        elif failures:
            status = "setup_required"
        else:
            status = "optional"
        skills.append(
            {
                **s,
                "configured": s["enabled"] and not failures,
                "status": status,
                "tools_count": len(s.get("tools", [])),
                "setup_command": f"/connect {name}",
                "prerequisites": failures,
            }
        )
    return jsonify({"skills": skills})


@app.route("/api/skills/<name>/detail")
@require_auth
def api_skill_detail(name):
    """Get full SKILL.md content for a skill."""
    agent = get_agent()
    skill = agent.skills.get_skill(name)
    if not skill:
        return jsonify({"error": "Skill not found"}), 404

    from familiar.core.skills import check_skill_prerequisites

    prereq_failures = check_skill_prerequisites()
    failures = prereq_failures.get(name, [])
    configured = skill.enabled and not failures

    return jsonify(
        {
            "name": name,
            "description": skill.description,
            "summary": skill.summary,
            "tools": [t.name for t in skill.tools],
            "configured": configured,
            "setup_command": f"/connect {name}",
            "prerequisites": failures,
            "enabled": skill.enabled,
        }
    )


@app.route("/api/skills/<name>/toggle", methods=["POST"])
@require_auth
def api_skill_toggle(name):
    """Enable/disable a skill."""
    agent = get_agent()
    skill = agent.skills.get_skill(name)

    if not skill:
        return jsonify({"error": "Skill not found"}), 404

    if skill.enabled:
        agent.skills.disable_skill(name)
        return jsonify({"enabled": False})
    else:
        agent.skills.enable_skill(name)
        return jsonify({"enabled": True})


@app.route("/api/connect/test/<service>", methods=["POST"])
@require_auth
def api_connect_test(service):
    """Test connection for a configured service."""
    from familiar.channels.connect_wizard import _apis, _selfhosted

    test_fns = {
        "hubspot": _apis.wizard_hubspot_test,
        "todoist": _apis.wizard_todoist_test,
        "stripe": _apis.wizard_stripe_test,
        "square": _apis.wizard_square_test,
        "mailerlite": _apis.wizard_mailerlite_test,
        "shippo": _apis.wizard_shippo_test,
        "calcom": _apis.wizard_calcom_test,
        "brave_search": _apis.wizard_brave_search_test,
        "spoonacular": _apis.wizard_spoonacular_test,
        "reddit": _apis.wizard_reddit_test,
        "etsy": _apis.wizard_etsy_test,
        "clockify": _apis.wizard_clockify_test,
        "quickbooks": _apis.wizard_quickbooks_test,
        "mealie": _selfhosted.wizard_mealie_test,
    }
    test_fn = test_fns.get(service)
    if not test_fn:
        return jsonify({"error": f"No test available for '{service}'"}), 404

    from familiar.channels.connect_wizard._helpers import _test_http_service

    # Use a lightweight approach: check env vars and call _test_http_service directly
    svc_tests = {
        "hubspot": lambda: _test_http_service(
            "https://api.hubapi.com/crm/v3/objects/contacts",
            headers={"Authorization": f"Bearer {os.environ.get('HUBSPOT_ACCESS_TOKEN', '')}"},
            params={"limit": "1"},
        ),
        "todoist": lambda: _test_http_service(
            "https://api.todoist.com/rest/v2/projects",
            headers={"Authorization": f"Bearer {os.environ.get('TODOIST_API_TOKEN', '')}"},
        ),
        "stripe": lambda: _test_http_service(
            "https://api.stripe.com/v1/balance",
            headers={"Authorization": f"Bearer {os.environ.get('STRIPE_SECRET_KEY', '')}"},
        ),
        "square": lambda: _test_http_service(
            "https://connect.squareup.com/v2/locations",
            headers={"Authorization": f"Bearer {os.environ.get('SQUARE_ACCESS_TOKEN', '')}"},
        ),
        "mailerlite": lambda: _test_http_service(
            "https://connect.mailerlite.com/api/subscribers",
            headers={"Authorization": f"Bearer {os.environ.get('MAILERLITE_API_KEY', '')}"},
            params={"limit": "1"},
        ),
        "shippo": lambda: _test_http_service(
            "https://api.goshippo.com/carriers",
            headers={"Authorization": f"ShippoToken {os.environ.get('SHIPPO_API_KEY', '')}"},
        ),
        "calcom": lambda: _test_http_service(
            "https://api.cal.com/v2/me",
            headers={"Authorization": f"Bearer {os.environ.get('CALCOM_API_KEY', '')}"},
        ),
        "brave_search": lambda: _test_http_service(
            "https://api.search.brave.com/res/v1/web/search",
            headers={"X-Subscription-Token": os.environ.get("BRAVE_SEARCH_API_KEY", "")},
            params={"q": "test", "count": "1"},
        ),
        "spoonacular": lambda: _test_http_service(
            "https://api.spoonacular.com/recipes/random",
            params={"number": "1", "apiKey": os.environ.get("SPOONACULAR_API_KEY", "")},
        ),
        "clockify": lambda: _test_http_service(
            "https://api.clockify.me/api/v1/user",
            headers={"X-Api-Key": os.environ.get("CLOCKIFY_API_KEY", "")},
        ),
        "mealie": lambda: _test_http_service(
            f"{os.environ.get('MEALIE_URL', '')}/api/recipes",
            headers={"Authorization": f"Bearer {os.environ.get('MEALIE_TOKEN', '')}"},
            params={"page": "1", "perPage": "1"},
        ),
        "quickbooks": lambda: _test_http_service(
            "https://quickbooks.api.intuit.com/v3/company/{rid}/companyinfo/{rid}".format(
                rid=os.environ.get("QUICKBOOKS_REALM_ID", "")
            ),
            headers={
                "Authorization": f"Bearer {os.environ.get('QUICKBOOKS_ACCESS_TOKEN', '')}",
                "Accept": "application/json",
            },
        ),
        "etsy": lambda: _test_http_service(
            "https://openapi.etsy.com/v3/application/shops/{sid}".format(
                sid=os.environ.get("ETSY_SHOP_ID", "")
            ),
            headers={"x-api-key": os.environ.get("ETSY_API_KEY", "")},
        ),
        "reddit": lambda: _test_reddit(),
    }

    def _test_reddit():
        """Reddit needs OAuth — simplified check."""
        import httpx

        cid = os.environ.get("REDDIT_CLIENT_ID", "")
        csecret = os.environ.get("REDDIT_CLIENT_SECRET", "")
        if not cid or not csecret:
            return False, "Reddit credentials not configured"
        try:
            with httpx.Client(timeout=10) as client:
                resp = client.post(
                    "https://www.reddit.com/api/v1/access_token",
                    auth=(cid, csecret),
                    data={"grant_type": "client_credentials"},
                    headers={"User-Agent": "Familiar/1.0"},
                )
                if resp.status_code == 200 and "access_token" in resp.json():
                    return True, "OK (OAuth token acquired)"
                return False, f"HTTP {resp.status_code}"
        except Exception as e:
            return False, str(e)

    sync_test = svc_tests.get(service)
    if not sync_test:
        return jsonify({"error": f"No sync test for '{service}'"}), 404

    try:
        ok, msg = sync_test()
        return jsonify({"success": ok, "message": msg})
    except Exception as e:
        return jsonify({"success": False, "message": str(e)})


@app.route("/api/tools")
@require_auth
def api_tools():
    """Get available tools."""
    agent = get_agent()
    tools = []

    for tool in agent.tools.get_all():
        tools.append(
            {"name": tool.name, "description": tool.description, "category": tool.category}
        )

    return jsonify({"tools": tools})


@app.route("/api/nodes")
@require_auth
def api_nodes():
    """Get connected mesh nodes."""
    if not _gateway:
        return jsonify({"nodes": [], "gateway_running": False})

    nodes = []
    for node_id, info in _gateway.nodes.items():
        nodes.append(
            {
                "id": info.node_id,
                "name": info.name,
                "type": info.type,
                "connected_at": info.connected_at,
                "last_seen": info.last_seen,
                "tools": info.tools,
            }
        )

    return jsonify({"nodes": nodes, "gateway_running": True})


# ============================================================
# Mesh Network Endpoints (v2.7.6)
# ============================================================


@app.route("/api/mesh/status")
@require_auth
def api_mesh_status():
    """Get full mesh status: gateway, transports, counts, config."""
    if not _gateway:
        return jsonify({"running": False, "error": "Gateway not initialized"})

    try:
        config = _gateway.config
        status = {
            "running": _gateway._running,
            "node_id": config.get("node_id", ""),
            "node_name": config.get("node_name", ""),
            "host": _gateway.host,
            "port": _gateway.port,
            "node_count": len(_gateway.nodes),
            "peer_count": len(_gateway.peer_gateways),
            "transports": {
                "websocket": {
                    "running": _gateway._running,
                    "address": f"ws://{_gateway.host}:{_gateway.port}",
                },
                "mdns": {
                    "running": _gateway.discovery.is_running if _gateway.discovery else False,
                    "available": True,
                    "peer_count": _gateway.discovery.peer_store.count()
                    if _gateway.discovery
                    else 0,
                },
                "tor": _gateway.tor_transport.get_status()
                if _gateway.tor_transport
                else {
                    "running": False,
                    "available": False,
                    "onion_address": "",
                },
            },
            "config": {
                "discovery_enabled": config.get("discovery_enabled", False),
                "tor_enabled": config.get("tor_enabled", False),
                "max_peers": config.get("max_peers", 16),
                "auto_connect_trusted": config.get("auto_connect_trusted", True),
            },
        }

        # Add Tor availability even if transport not initialized
        if not _gateway.tor_transport:
            try:
                from familiar.core.mesh.tor_transport import TorTransport

                status["transports"]["tor"]["available"] = TorTransport.is_available()
            except ImportError:
                status["transports"]["tor"]["available"] = {
                    "available": False,
                    "missing": ["tor_transport module"],
                }

        return jsonify(status)
    except Exception as e:
        return jsonify({"running": False, "error": str(e)})


@app.route("/api/mesh/peers")
@require_auth
def api_mesh_peers():
    """Get all discovered peers from PeerStore."""
    if not _gateway:
        return jsonify({"peers": []})

    try:
        peers = []
        if _gateway.discovery and _gateway.discovery.peer_store:
            from dataclasses import asdict

            for peer in _gateway.discovery.peer_store.get_all():
                p = asdict(peer)
                # Add trust info if available
                if _gateway.trust_manager:
                    rec = _gateway.trust_manager.get_trust_record(peer.node_id)
                    p["trust_level"] = rec.trust_level if rec else "unknown"
                else:
                    p["trust_level"] = "unknown"
                peers.append(p)
        return jsonify({"peers": peers})
    except Exception as e:
        return jsonify({"peers": [], "error": str(e)})


@app.route("/api/mesh/key")
@require_auth
def api_mesh_key():
    """Get mesh secret key."""
    if not _gateway:
        return jsonify({"error": "Gateway not initialized"}), 404

    return jsonify({"key": _gateway.config.get("secret_key", "")})


@app.route("/api/mesh/trust")
@require_auth
def api_mesh_trust():
    """Get all trust records, verification codes, permissions, crypto status."""
    if not _gateway or not _gateway.trust_manager:
        return jsonify({
            "available": False,
            "has_crypto": False,
            "our_fingerprint": "",
            "records": [],
        })

    try:
        tm = _gateway.trust_manager
        records = []
        for rec in tm.get_all_records():
            r = {
                "node_id": rec.node_id,
                "node_name": rec.node_name,
                "identity_fingerprint": rec.identity_fingerprint,
                "trust_level": rec.trust_level,
                "trusted_at": rec.trusted_at or "",
                "blocked_at": rec.blocked_at or "",
                "verification_code": tm.get_verification_code(rec.node_id) or "",
                "has_session": tm.has_session(rec.node_id),
                "permissions": rec.permissions.to_dict() if rec.permissions else {},
            }
            records.append(r)

        return jsonify({
            "available": True,
            "has_crypto": tm.has_crypto,
            "our_fingerprint": tm.get_our_fingerprint(),
            "records": records,
        })
    except Exception as e:
        return jsonify({"available": False, "error": str(e)})


@app.route("/api/mesh/trust/<node_id>/approve", methods=["POST"])
@require_auth
def api_mesh_trust_approve(node_id):
    """Approve a pending peer."""
    if not _gateway or not _gateway.trust_manager:
        return jsonify({"error": "Trust manager not available"}), 404

    try:
        ok = _gateway.trust_manager.approve(node_id)
        return jsonify({"success": ok, "node_id": node_id})
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/mesh/trust/<node_id>/block", methods=["POST"])
@require_auth
def api_mesh_trust_block(node_id):
    """Block a peer."""
    if not _gateway or not _gateway.trust_manager:
        return jsonify({"error": "Trust manager not available"}), 404

    try:
        ok = _gateway.trust_manager.block(node_id)
        return jsonify({"success": ok, "node_id": node_id})
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/mesh/trust/<node_id>/permissions", methods=["POST"])
@require_auth
def api_mesh_trust_permissions(node_id):
    """Update permission flags for a peer."""
    if not _gateway or not _gateway.trust_manager:
        return jsonify({"error": "Trust manager not available"}), 404

    try:
        data = request.get_json() or {}
        permission = data.get("permission", "")
        granted = data.get("granted", False)

        if not permission:
            return jsonify({"error": "permission field required"}), 400

        if granted:
            ok = _gateway.trust_manager.grant_permission(node_id, permission)
        else:
            ok = _gateway.trust_manager.revoke_permission(node_id, permission)

        return jsonify({"success": ok, "node_id": node_id, "permission": permission})
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/mesh/memory")
@require_auth
def api_mesh_memory():
    """Get memory bridge status with full topic listings."""
    if not _gateway or not _gateway.memory_bridge:
        return jsonify({"initialized": False})

    try:
        bridge = _gateway.memory_bridge
        status = {"initialized": True, **bridge.get_status()}

        # Enrich with markdown topic listings and word counts
        md_store = getattr(bridge, "_markdown_store", None)
        shared_store = getattr(bridge, "shared_store", None)

        # Your Knowledge — all local markdown topics with word counts
        markdown_topics = []
        if md_store and hasattr(md_store, "list_topics"):
            for t in md_store.list_topics(limit=100):
                topic_info = {
                    "topic": t["topic"],
                    "tags": t.get("tags", []),
                    "revision": t.get("revision", 1),
                    "updated": t.get("updated", ""),
                    "created": t.get("created", ""),
                }
                # Get word count
                try:
                    body = md_store.read_topic(t["topic"])
                    topic_info["word_count"] = len(body.split()) if body else 0
                except Exception:
                    topic_info["word_count"] = 0
                markdown_topics.append(topic_info)
        status["markdown_topics"] = markdown_topics
        status["total_words"] = sum(t["word_count"] for t in markdown_topics)
        status["total_topics"] = len(markdown_topics)

        # Shared with Peers — topics marked shareable
        shared_topics = []
        if shared_store:
            for key in shared_store.get_shareable_keys():
                meta = shared_store.get(key)
                if meta:
                    shared_topics.append({
                        "key": meta.key,
                        "scope": meta.share_scope,
                        "shared_at": meta.shared_at or "",
                    })
        status["shared_topics"] = shared_topics

        # Received from Peers — topics pushed to us
        received_topics = []
        if shared_store:
            for key in shared_store.get_received_keys():
                meta = shared_store.get(key)
                if meta:
                    received_topics.append({
                        "key": meta.key,
                        "origin_name": meta.mesh_origin_name or "",
                        "origin_id": meta.mesh_origin or "",
                        "origin_genesis": meta.mesh_origin_genesis_hash or "",
                        "received_at": meta.received_at or "",
                    })
        status["received_topics"] = received_topics

        return jsonify(status)
    except Exception as e:
        return jsonify({"initialized": False, "error": str(e)})


@app.route("/api/mesh/models")
@require_auth
def api_mesh_models():
    """Get model bridge status."""
    if not _gateway or not _gateway.model_bridge:
        return jsonify({"initialized": False})

    try:
        return jsonify({"initialized": True, **_gateway.model_bridge.get_status()})
    except Exception as e:
        return jsonify({"initialized": False, "error": str(e)})


@app.route("/api/mesh/usage")
@require_auth
def api_mesh_usage():
    """Get usage/Dross economy summary."""
    if not _gateway or not _gateway._usage_tracker:
        return jsonify({"initialized": False})

    try:
        return jsonify({"initialized": True, **_gateway._usage_tracker.get_usage_summary()})
    except Exception as e:
        return jsonify({"initialized": False, "error": str(e)})


@app.route("/api/mesh/tor/enable", methods=["POST"])
@require_auth
def api_mesh_tor_enable():
    """Start Tor hidden service."""
    if not _gateway:
        return jsonify({"error": "Gateway not initialized"}), 404

    try:
        import asyncio

        from familiar.core.mesh.tor_transport import TorTransport

        avail = TorTransport.is_available()
        if not avail["available"]:
            return jsonify({
                "error": "Tor not available",
                "missing": avail["missing"],
            }), 400

        if _gateway.tor_transport and _gateway.tor_transport.is_running:
            return jsonify({
                "success": True,
                "onion_address": _gateway.tor_transport.onion_address,
                "message": "Already running",
            })

        config = _gateway.config
        transport = TorTransport(
            control_port=config.get("tor_control_port", 9051),
            socks_port=config.get("tor_socks_port", 9050),
            control_password=config.get("tor_control_password", ""),
        )

        loop = asyncio.new_event_loop()
        started = loop.run_until_complete(transport.start(gateway_port=_gateway.port))
        loop.close()

        if started:
            _gateway.tor_transport = transport
            config.set("tor_enabled", True)
            return jsonify({
                "success": True,
                "onion_address": transport.onion_address,
            })
        else:
            return jsonify({"error": "Failed to start Tor hidden service"}), 500
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/mesh/tor/disable", methods=["POST"])
@require_auth
def api_mesh_tor_disable():
    """Stop Tor hidden service."""
    if not _gateway:
        return jsonify({"error": "Gateway not initialized"}), 404

    try:
        if _gateway.tor_transport:
            import asyncio

            loop = asyncio.new_event_loop()
            loop.run_until_complete(_gateway.tor_transport.stop())
            loop.close()
            _gateway.tor_transport = None

        _gateway.config.set("tor_enabled", False)
        return jsonify({"success": True})
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/mesh/discovery/enable", methods=["POST"])
@require_auth
def api_mesh_discovery_enable():
    """Start mDNS discovery."""
    if not _gateway:
        return jsonify({"error": "Gateway not initialized"}), 404

    try:
        if _gateway.discovery and _gateway.discovery.is_running:
            return jsonify({"success": True, "message": "Already running"})

        if not _gateway.discovery:
            from familiar.core.mesh.discovery import MeshDiscovery

            skill_names = []
            try:
                from familiar.core.tools import get_tool_registry

                skill_names = [t.get("name", "") for t in get_tool_registry().get_schemas()]
            except Exception:
                pass

            config = _gateway.config
            _gateway.discovery = MeshDiscovery(
                node_id=config.get("node_id"),
                node_name=config.get("node_name", "familiar"),
                node_type=config.get("node_type", "gateway"),
                port=_gateway.port,
                skills=skill_names,
                fingerprint=_gateway.trust_manager.get_our_fingerprint()
                if _gateway.trust_manager
                else "",
                mesh_dir=None,
                max_peers=config.get("max_peers", 16),
            )

        _gateway.discovery.start()
        _gateway.config.set("discovery_enabled", True)
        return jsonify({"success": True})
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/mesh/discovery/disable", methods=["POST"])
@require_auth
def api_mesh_discovery_disable():
    """Stop mDNS discovery."""
    if not _gateway:
        return jsonify({"error": "Gateway not initialized"}), 404

    try:
        if _gateway.discovery:
            _gateway.discovery.stop()

        _gateway.config.set("discovery_enabled", False)
        return jsonify({"success": True})
    except Exception as e:
        return jsonify({"error": str(e)}), 500


# ============================================================
# MESH — RENDEZVOUS (Phase 9)
# ============================================================


def _get_mesh_config_standalone():
    """Get MeshConfig even without a running gateway — for pre-flight configuration."""
    try:
        agent = get_agent()
        gw = getattr(agent, "_gateway", None)
        if gw and hasattr(gw, "config"):
            return gw.config, gw
    except Exception:
        pass
    # Fallback: load config directly from disk
    try:
        from familiar.core.mesh.gateway import MeshConfig
        return MeshConfig(), None
    except Exception:
        return None, None


@app.route("/api/mesh/rendezvous/status")
@require_auth
def api_mesh_rendezvous_status():
    """Get rendezvous client status."""
    try:
        cfg, gw = _get_mesh_config_standalone()
        if not cfg:
            return jsonify({"enabled": False, "running": False, "servers": []})

        enabled = cfg.get("rendezvous_enabled", False)
        servers = cfg.get("rendezvous_servers", [])
        running = False
        if gw and gw.discovery:
            running = getattr(gw.discovery, "_rendezvous_running", False)

        return jsonify({
            "enabled": enabled,
            "running": running,
            "servers": servers,
            "token_set": bool(cfg.get("rendezvous_token", "")),
        })
    except Exception as e:
        return jsonify({"enabled": False, "error": str(e)})


@app.route("/api/mesh/rendezvous/servers", methods=["POST"])
@require_auth
def api_mesh_rendezvous_add_server():
    """Add a rendezvous server URL."""
    data = request.json or {}
    url = data.get("url", "").strip().rstrip("/")
    if not url:
        return jsonify({"error": "URL is required"}), 400

    try:
        cfg, gw = _get_mesh_config_standalone()
        if not cfg:
            return jsonify({"error": "Cannot load mesh config"}), 500

        servers = cfg.get("rendezvous_servers", [])
        if url not in servers:
            servers.append(url)
            cfg.set("rendezvous_servers", servers)

        return jsonify({"success": True, "servers": servers})
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/mesh/rendezvous/servers", methods=["DELETE"])
@require_auth
def api_mesh_rendezvous_remove_server():
    """Remove a rendezvous server URL."""
    data = request.json or {}
    url = data.get("url", "").strip().rstrip("/")
    if not url:
        return jsonify({"error": "URL is required"}), 400

    try:
        cfg, gw = _get_mesh_config_standalone()
        if not cfg:
            return jsonify({"error": "Cannot load mesh config"}), 500

        servers = cfg.get("rendezvous_servers", [])
        servers = [s for s in servers if s != url]
        cfg.set("rendezvous_servers", servers)

        return jsonify({"success": True, "servers": servers})
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/mesh/rendezvous/enable", methods=["POST"])
@require_auth
def api_mesh_rendezvous_enable():
    """Enable or disable rendezvous discovery."""
    data = request.json or {}
    enable = data.get("enable", True)

    try:
        cfg, gw = _get_mesh_config_standalone()
        if not cfg:
            return jsonify({"error": "Cannot load mesh config"}), 500

        cfg.set("rendezvous_enabled", bool(enable))

        # If gateway is live, start/stop rendezvous immediately
        if gw and gw.discovery:
            if enable:
                servers = cfg.get("rendezvous_servers", [])
                token = cfg.get("rendezvous_token", "")
                gw.discovery.rendezvous_servers = servers
                gw.discovery.rendezvous_token = token
                gw.discovery.start_rendezvous()
            else:
                gw.discovery.stop_rendezvous()

        return jsonify({"success": True, "enabled": bool(enable)})
    except Exception as e:
        return jsonify({"error": str(e)}), 500


# Global ref for the local rendezvous server thread
_local_rdv_thread = None


@app.route("/api/mesh/rendezvous/local/start", methods=["POST"])
@require_auth
def api_mesh_rendezvous_local_start():
    """Start a local rendezvous server on this machine."""
    global _local_rdv_thread
    if _local_rdv_thread and _local_rdv_thread.is_alive():
        return jsonify({"success": True, "already_running": True, "port": 18790})

    try:
        from familiar.core.mesh.rendezvous import start_local_server

        data = request.json or {}
        port = int(data.get("port", 18790))
        token = data.get("token", "")
        _local_rdv_thread = start_local_server(port=port, auth_token=token)
        return jsonify({"success": True, "port": port})
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/mesh/rendezvous/local/status")
@require_auth
def api_mesh_rendezvous_local_status():
    """Check if local rendezvous server is running."""
    running = _local_rdv_thread is not None and _local_rdv_thread.is_alive()
    return jsonify({"running": running})


@app.route("/api/email/summary")
@require_auth
def api_email_summary():
    """Get email triage summary."""
    try:
        import sys

        sys.path.insert(0, str(Path(__file__).parent.parent / "skills" / "triage"))
        from skill import get_category_summary

        result = get_category_summary({"days_back": 7})
        return jsonify({"summary": result})
    except Exception:
        return jsonify({"error": "Failed to generate email summary"}), 500


_triage_cache: dict = {"data": None, "timestamp": 0.0}
_triage_cache_lock = threading.Lock()
_TRIAGE_CACHE_TTL = 300  # 5 minutes
_TRIAGE_CACHE_FILE = Path.home() / ".familiar" / "data" / "cache" / "triage_cache.json"


def _load_triage_cache():
    """Load triage cache from disk on startup."""
    try:
        if _TRIAGE_CACHE_FILE.exists():
            data = json.loads(_TRIAGE_CACHE_FILE.read_text())
            _triage_cache["data"] = data.get("data")
            _triage_cache["timestamp"] = data.get("timestamp", 0.0)
    except Exception as e:
        logger.debug("Could not load triage cache: %s", e)


def _save_triage_cache():
    """Persist triage cache to disk so restarts don't lose state."""
    try:
        _TRIAGE_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
        _TRIAGE_CACHE_FILE.write_text(
            json.dumps({"data": _triage_cache["data"], "timestamp": _triage_cache["timestamp"]})
        )
    except Exception as e:
        logger.debug("Could not save triage cache: %s", e)


# Load on import
_load_triage_cache()


@app.route("/api/email/triage")
@require_auth
def api_email_triage():
    """Get full email triage as structured JSON for the interactive dashboard."""
    force_refresh = request.args.get("refresh", "false").lower() == "true"

    with _triage_cache_lock:
        if (
            not force_refresh
            and _triage_cache["data"] is not None
            and (time.time() - _triage_cache["timestamp"]) < _TRIAGE_CACHE_TTL
        ):
            return jsonify(_triage_cache["data"])

    try:
        import threading

        from familiar.skills.triage.skill import CATEGORIES, triage_inbox

        # Run triage — side-effect: stores ordered_emails on thread-local
        text_result = triage_inbox(
            {
                "limit": int(request.args.get("limit", 25)),
                "unread_only": request.args.get("unread", "false").lower() == "true",
                "days_back": int(request.args.get("days_back", 7)),
            }
        )

        # Retrieve structured data from contextvars (set by triage_inbox)
        from familiar.core.tool_executor import _triage_ctx_var as _dash_triage_ctx

        triage_ctx = _dash_triage_ctx.get()
        if triage_ctx and triage_ctx.get("emails"):
            emails = triage_ctx["emails"]
            # Group by category
            categories = {}
            for em in emails:
                cat = em.get("category", "Other")
                categories.setdefault(cat, []).append(em)
            result = {
                "emails": emails,
                "categories": {cat: categories.get(cat, []) for cat in CATEGORIES},
                "total": triage_ctx.get("total", len(emails)),
                "triage": text_result,  # Keep text fallback
            }
        else:
            # Fallback: text-only response
            result = {"triage": text_result}

        with _triage_cache_lock:
            _triage_cache["data"] = result
            _triage_cache["timestamp"] = time.time()
            _save_triage_cache()
        return jsonify(result)
    except Exception as e:
        logger.warning("Email triage error: %s", e)
        # Serve stale cache rather than error — data is better than nothing
        with _triage_cache_lock:
            if _triage_cache["data"] is not None:
                return jsonify(_triage_cache["data"])
        # Give actionable error when Gmail OAuth is missing
        _has_google = (DATA_DIR / "google_credentials.json").exists()
        _has_gmail = (DATA_DIR / "google_token_gmail.json").exists()
        if _has_google and not _has_gmail:
            return jsonify({
                "error": "Gmail needs authorization",
                "action": "connect_gmail",
                "message": "Click 'Connect Gmail' to authorize email access.",
            }), 401
        return jsonify({"error": "Failed to triage emails"}), 500


@app.route("/api/email/read/<msg_id>")
@require_auth
def api_email_read(msg_id):
    """Read a specific email by Gmail message ID or IMAP ID."""
    try:
        from familiar.skills.email.skill import (
            _gmail_read_email_by_id,
            _imap_read_email_by_id,
        )

        id_source = request.args.get("source", "gmail")
        if id_source == "gmail":
            result = _gmail_read_email_by_id(msg_id)
        else:
            result = _imap_read_email_by_id(msg_id)
        return jsonify({"content": result})
    except Exception as e:
        logger.warning("Email read error: %s", e)
        return jsonify({"error": f"Failed to read email: {e}"}), 500


@app.route("/api/email/delete/<msg_id>", methods=["POST"])
@require_auth
def api_email_delete(msg_id):
    """Move an email to Trash by Gmail message ID or IMAP UID."""
    try:
        from familiar.skills.email.skill import (
            _gmail_trash_by_id,
            _imap_trash_by_id,
        )

        body = request.get_json(silent=True) or {}
        id_source = body.get("source", request.args.get("source", "gmail"))
        if id_source == "gmail":
            result = _gmail_trash_by_id(msg_id)
        else:
            result = _imap_trash_by_id(msg_id)
        if result == "ok":
            return jsonify({"status": "ok"})
        return jsonify({"error": result}), 500
    except Exception as e:
        logger.warning("Email delete error: %s", e)
        return jsonify({"error": f"Failed to delete email: {e}"}), 500


@app.route("/api/email/reply", methods=["POST"])
@require_auth
def api_email_reply():
    """Send a reply to an email via Gmail API or SMTP."""
    try:
        from familiar.skills.email.skill import send_email

        payload = request.get_json() or {}
        # Build send_email payload with reply context
        send_data = {
            "to": payload.get("to", ""),
            "subject": payload.get("subject", ""),
            "body": payload.get("body", ""),
            # Dashboard reply UI is the confirmation step — user reviewed the draft
            # before clicking Send, so we bypass the agent confirmation gate.
            "_confirmed": True,
        }
        # Include thread_id for Gmail threading
        if payload.get("thread_id"):
            send_data["_gmail_thread_id"] = payload["thread_id"]
        if payload.get("in_reply_to"):
            send_data["in_reply_to"] = payload["in_reply_to"]

        result = send_email(send_data)
        # send_email returns a string — treat anything not starting with ✅ as an error
        if not result.startswith("✅"):
            return jsonify({"error": result}), 500
        return jsonify({"result": result})
    except Exception as e:
        logger.warning("Email reply error: %s", e)
        return jsonify({"error": "Failed to send reply"}), 500


@app.route("/api/email/sent")
@require_auth
def api_email_sent():
    """Fetch sent mail — Gmail API (in:sent) or IMAP Sent folder."""
    limit = int(request.args.get("limit", 25))
    days_back = int(request.args.get("days_back", 30))

    try:
        from familiar.skills.email.skill import (
            _get_gmail_service,
            _gmail_available,
            _gmail_body,
            _gmail_header,
            _imap_config,
            _imap_connect,
            decode_mime_header,
            get_email_body,
        )
        from datetime import datetime as _dt, timedelta as _td

        emails = []

        if _gmail_available():
            svc = _get_gmail_service()
            since = (_dt.now() - _td(days=days_back)).strftime("%Y/%m/%d")
            q = f"in:sent after:{since}"
            res = svc.users().messages().list(userId="me", q=q, maxResults=limit).execute()
            for m in res.get("messages", []):
                full = (
                    svc.users()
                    .messages()
                    .get(
                        userId="me",
                        id=m["id"],
                        format="metadata",
                        metadataHeaders=["To", "Subject", "Date"],
                    )
                    .execute()
                )
                payload = full.get("payload", {})
                to_hdr = _gmail_header(payload, "To")
                to_name = to_hdr.split("<")[0].strip().strip('"') if "<" in to_hdr else to_hdr.split("@")[0]
                subject = _gmail_header(payload, "Subject") or "(no subject)"
                date_str = _gmail_header(payload, "Date") or ""
                emails.append({
                    "id": m["id"],
                    "thread_id": full.get("threadId", ""),
                    "id_source": "gmail",
                    "to_name": to_name,
                    "to_email": to_hdr,
                    "subject": subject,
                    "date": date_str,
                })

        else:
            # IMAP fallback — try common Sent folder names
            cfg = _imap_config()
            if cfg.get("address"):
                import email as _email_lib
                conn = _imap_connect(cfg)
                sent_folders = ["[Gmail]/Sent Mail", "Sent", "Sent Messages", "INBOX.Sent"]
                selected = False
                for folder in sent_folders:
                    try:
                        status, _ = conn.select(folder, readonly=True)
                        if status == "OK":
                            selected = True
                            break
                    except Exception:
                        continue
                if selected:
                    since_imap = (_dt.now() - _td(days=days_back)).strftime("%d-%b-%Y")
                    _, data = conn.search(None, f'SINCE {since_imap}')
                    ids = data[0].split() if data and data[0] else []
                    for uid in reversed(ids[-limit:]):
                        _, msg_data = conn.fetch(uid, "(RFC822.HEADER)")
                        if not msg_data or not msg_data[0]:
                            continue
                        raw = msg_data[0][1] if isinstance(msg_data[0], tuple) else msg_data[0]
                        msg = _email_lib.message_from_bytes(raw)
                        to_hdr = decode_mime_header(msg.get("To", ""))
                        to_name = to_hdr.split("<")[0].strip().strip('"') if "<" in to_hdr else to_hdr.split("@")[0]
                        subject = decode_mime_header(msg.get("Subject", "(no subject)"))
                        date_str = msg.get("Date", "")
                        emails.append({
                            "id": uid.decode() if isinstance(uid, bytes) else str(uid),
                            "id_source": "imap",
                            "to_name": to_name,
                            "to_email": to_hdr,
                            "subject": subject,
                            "date": date_str,
                        })
                conn.logout()

        return jsonify({"emails": emails, "total": len(emails)})

    except Exception as e:
        logger.warning("Email sent fetch error: %s", e)
        return jsonify({"error": str(e)}), 500


# ── Email topic flags, VIPs, and weekly digest ───────────────────────────────


@app.route("/api/email/topic-flags", methods=["GET", "POST", "DELETE"])
@require_auth
def api_email_topic_flags():
    """Manage topic flags that auto-escalate emails to urgent."""
    try:
        from familiar.skills.triage.skill import manage_topic_flags

        if request.method == "GET":
            result = manage_topic_flags({"action": "list"})
            from familiar.skills.triage.skill import _load_triage_data

            data = _load_triage_data()
            return jsonify({"flags": data.get("topic_flags", [])})
        elif request.method == "POST":
            body = request.get_json(silent=True) or {}
            result = manage_topic_flags(
                {"action": "add", "keyword": body.get("keyword", "")}
            )
            return jsonify({"result": result})
        else:  # DELETE
            body = request.get_json(silent=True) or {}
            result = manage_topic_flags(
                {"action": "remove", "keyword": body.get("keyword", "")}
            )
            return jsonify({"result": result})
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/email/vip-senders", methods=["GET", "POST", "DELETE"])
@require_auth
def api_email_vip_senders():
    """Manage VIP senders whose emails are always prioritized."""
    try:
        from familiar.skills.triage.skill import manage_vip_senders

        if request.method == "GET":
            from familiar.skills.triage.skill import _load_triage_data

            data = _load_triage_data()
            return jsonify({"vips": data.get("vip_senders", [])})
        elif request.method == "POST":
            body = request.get_json(silent=True) or {}
            result = manage_vip_senders(
                {"action": "add", "sender": body.get("sender", "")}
            )
            return jsonify({"result": result})
        else:  # DELETE
            body = request.get_json(silent=True) or {}
            result = manage_vip_senders(
                {"action": "remove", "sender": body.get("sender", "")}
            )
            return jsonify({"result": result})
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/email/weekly-digest")
@require_auth
def api_email_weekly_digest():
    """Generate weekly email digest for review."""
    try:
        from familiar.skills.triage.skill import get_weekly_digest

        days = int(request.args.get("days", 7))
        result = get_weekly_digest({"days_back": days})
        return jsonify(result)
    except Exception as e:
        return jsonify({"error": str(e)}), 500


# ── FileStore API ──

_file_store_instance = None
_file_store_init_lock = threading.Lock()


def _get_file_store():
    global _file_store_instance
    if _file_store_instance is None:
        with _file_store_init_lock:
            if _file_store_instance is None:
                from familiar.core.files import get_file_store
                _file_store_instance = get_file_store()
    return _file_store_instance


@app.route("/api/files/upload", methods=["POST"])
@require_auth
def api_files_upload():
    """Upload a file and attach it to a record."""
    if "file" not in request.files:
        return jsonify({"error": "No file provided"}), 400

    f = request.files["file"]
    if not f.filename:
        return jsonify({"error": "No filename"}), 400

    record_type = request.form.get("record_type", "general")
    record_id = request.form.get("record_id", "unlinked")
    job_class = request.form.get("job_class", "general")
    description = request.form.get("description", "")

    try:
        data = f.read()
        store = _get_file_store()
        record = store.store(
            data=data,
            filename=f.filename,
            job_class=job_class,
            record_type=record_type,
            record_id=record_id,
            description=description,
        )
        return jsonify({"status": "ok", "file": record.to_dict()}), 201
    except ValueError as e:
        return jsonify({"error": str(e)}), 400
    except Exception as e:
        logger.error("File upload error: %s", e)
        return jsonify({"error": f"Upload failed: {e}"}), 500


@app.route("/api/files/<file_id>")
@require_auth
def api_files_get(file_id):
    """Serve a stored file."""
    store = _get_file_store()
    path = store.get_path(file_id)
    if not path:
        return jsonify({"error": "File not found"}), 404
    record = store.get(file_id)
    return send_file(
        path,
        mimetype=record.mime_type if record else None,
        as_attachment=True,
        download_name=record.filename if record else None,
    )


@app.route("/api/files")
@require_auth
def api_files_list():
    """List files, optionally filtered by record or job class."""
    store = _get_file_store()
    record_type = request.args.get("record_type")
    record_id = request.args.get("record_id")
    job_class = request.args.get("job_class")
    query = request.args.get("q")

    if record_type and record_id:
        files = store.list_for_record(record_type, record_id, job_class)
    elif job_class:
        files = store.list_for_job_class(job_class)
    elif query:
        files = store.search(query=query, job_class=job_class)
    else:
        files = store.search(limit=100)

    return jsonify({"files": [f.to_dict() for f in files]})


@app.route("/api/files/<file_id>", methods=["DELETE"])
@require_auth
def api_files_delete(file_id):
    """Delete a stored file."""
    store = _get_file_store()
    if store.delete(file_id):
        return jsonify({"status": "ok"})
    return jsonify({"error": "File not found"}), 404


@app.route("/api/files/stats")
@require_auth
def api_files_stats():
    """Get file storage statistics."""
    store = _get_file_store()
    return jsonify(store.stats())


# ── Macro API ──


@app.route("/api/macros")
@require_auth
def api_macros_list():
    """List available macros, optionally filtered by job class."""
    from familiar.core.macros import get_macro_registry

    reg = get_macro_registry()
    job_class = request.args.get("job_class")
    if job_class:
        macros = reg.list_for_job_class(job_class)
    else:
        macros = reg.list_all()
    return jsonify({"macros": [m.to_dict() for m in macros]})


@app.route("/api/macros/<macro_name>", methods=["POST"])
@require_auth
def api_macros_execute(macro_name):
    """Execute a macro by name."""
    from familiar.core.macros import get_macro_registry

    reg = get_macro_registry()
    params = request.get_json(silent=True) or {}
    result = reg.execute(macro_name, params, context={"macro_name": macro_name})
    status_code = 200 if result.success else 400
    return jsonify(result.to_dict()), status_code


_briefing_cache: dict = {}  # In-memory structured briefing cache
_briefing_cache_lock = threading.Lock()


def _generate_briefing_full() -> dict:
    """Generate a full structured briefing with LLM synthesis. Heavy — call sparingly."""
    import sys

    sys.path.insert(0, str(Path(__file__).parent.parent / "skills" / "proactive"))
    from skill import generate_briefing, generate_briefing_data

    raw_data = generate_briefing_data()
    raw_text = generate_briefing({})

    summary = ""
    try:
        job_class = raw_data.get("job_class", "")
        role_hint = f" The user's role is {job_class.replace('_', ' ')}." if job_class else ""

        # Truncate raw_text to ~1500 chars to keep prompt small for CPU inference
        _raw_trimmed = raw_text[:1500] if len(raw_text) > 1500 else raw_text

        synthesis_prompt = (
            "You are generating the opening card of a daily briefing dashboard.{role_hint}\n\n"
            "Read the briefing data below and write a sharp 2-4 sentence executive summary that:\n"
            "- Calls out the single most urgent item that needs action today\n"
            "- Mentions any hard deadlines or time-sensitive events\n"
            "- Flags anything that could go wrong if ignored\n"
            "- Skips fluff — every sentence should be actionable\n\n"
            "No headers, no bullet points, no greeting. Just the summary:\n\n"
            f"{_raw_trimmed}"
        ).format(role_hint=role_hint)

        # Direct Ollama call — no tools, no agent overhead.
        # chat_with_results() sends full tool schemas which bloats context
        # and causes APITimeoutError on CPU-only machines.
        import urllib.request
        import urllib.error

        # Use agent config as single source of truth for model selection.
        # The agent already resolves config.yaml + env var precedence.
        _ollama_url = os.environ.get("OLLAMA_HOST", "http://localhost:11434")
        _ollama_model = "qwen2.5:0.5b"  # safe default
        try:
            agent = get_agent()
            if agent and hasattr(agent, "config") and agent.config.llm.ollama_model:
                _ollama_model = agent.config.llm.ollama_model
                _ollama_url = agent.config.llm.ollama_base_url
        except Exception:
            pass

        _payload = json.dumps({
            "model": _ollama_model,
            "messages": [
                {"role": "system", "content": "You are a concise briefing assistant."},
                {"role": "user", "content": synthesis_prompt},
            ],
            "stream": False,
            "options": {"num_predict": 256},
        }).encode()

        _req = urllib.request.Request(
            f"{_ollama_url}/api/chat",
            data=_payload,
            headers={"Content-Type": "application/json"},
            method="POST",
        )
        with urllib.request.urlopen(_req, timeout=180) as _resp:
            _body = json.loads(_resp.read())
            summary = _body.get("message", {}).get("content", "")

        # Record the call in analytics
        try:
            from familiar.core.analytics import get_analytics_store
            store = get_analytics_store()
            store.record_message(
                user_id="dashboard-briefing",
                channel="web",
                tokens_in=len(synthesis_prompt) // 4,
                tokens_out=len(summary) // 4,
                cost_usd=0.0,
                latency_ms=0,
                model=_ollama_model,
                skill="briefing",
            )
        except Exception:
            pass
    except Exception as e:
        logger.warning("LLM briefing synthesis failed: %s", e)

    result = {
        "summary": summary,
        "sections": raw_data["sections"],
        "timestamp": raw_data["timestamp"],
        "raw": raw_text,
    }

    with _briefing_cache_lock:
        _briefing_cache.update(result)

    # Also persist to disk — include sections so cache survives restarts
    _save_briefing(summary or raw_text, source="briefing_panel", sections=raw_data["sections"])

    return result


@app.route("/api/briefing")
@require_auth
def api_briefing():
    """Get daily briefing — serves from cache if available, generates only on first call."""
    # Serve from memory cache if fresh (< 30 min old)
    with _briefing_cache_lock:
        if _briefing_cache and _briefing_cache.get("timestamp"):
            try:
                from datetime import timezone

                cached_ts = datetime.fromisoformat(_briefing_cache["timestamp"])
                if cached_ts.tzinfo is None:
                    cached_ts = cached_ts.replace(tzinfo=timezone.utc)
                age_min = (datetime.now(timezone.utc) - cached_ts).total_seconds() / 60
                if age_min < 30:
                    return jsonify(_briefing_cache)
            except Exception:
                pass  # Parse error — regenerate

    # Serve from disk if available AND has structured sections (survives restarts).
    # Plain-text-only briefings (from scheduler agent.chat) are treated as stale —
    # they lack calendar, weather, and email data that the dashboard needs.
    disk = _load_briefing()
    if disk and disk.get("sections"):
        # Disk has structured data — check freshness (2 hours)
        _disk_fresh = False
        try:
            from datetime import timezone

            _disk_ts = datetime.fromisoformat(disk["timestamp"])
            if _disk_ts.tzinfo is None:
                _disk_ts = _disk_ts.replace(tzinfo=timezone.utc)
            _disk_age = (datetime.now(timezone.utc) - _disk_ts).total_seconds() / 60
            _disk_fresh = _disk_age < 120
        except Exception:
            pass
        if _disk_fresh:
            with _briefing_cache_lock:
                _briefing_cache.update(disk)
            return jsonify(disk)

    # Generate fresh structured briefing (first call, expired, or plain-text only on disk)
    try:
        return jsonify(_generate_briefing_full())
    except Exception:
        # Last resort: serve plain-text disk briefing if structured gen fails
        if disk and disk.get("message"):
            fallback = {
                "summary": disk["message"],
                "sections": {},
                "timestamp": disk.get("timestamp", ""),
                "raw": disk["message"],
            }
            return jsonify(fallback)
        return jsonify({"error": "Failed to generate briefing"}), 500


@app.route("/api/briefing/refresh", methods=["POST"])
@require_auth
def api_briefing_refresh():
    """Force-regenerate the structured briefing, bypassing cache."""
    try:
        with _briefing_cache_lock:
            _briefing_cache.clear()
        _generate_briefing_full()
        return jsonify({"ok": True})
    except Exception as e:
        logger.error("Briefing refresh failed: %s", e)
        return jsonify({"ok": False, "error": str(e)}), 500


@app.route("/api/briefing/action", methods=["POST"])
@require_auth
def api_briefing_action():
    """Handle calendar event quick actions (running late, cancel)."""
    data = request.json or {}
    action = data.get("action")
    event_summary = data.get("event_summary", "")
    attendees = data.get("attendees", [])
    event_time = data.get("event_time", "")

    if not action or not attendees:
        return jsonify({"error": "Missing action or attendees"}), 400

    prompt = f'I need to tell the attendees of my "{event_summary}" meeting at {event_time} '
    if action == "running_late":
        prompt += "that I'm running late. Draft and send a brief, polite email to: "
    elif action == "cancel":
        prompt += "that I need to cancel. Draft and send a brief, polite email to: "
    else:
        return jsonify({"error": f"Unknown action: {action}"}), 400
    prompt += ", ".join(attendees)

    try:
        agent = get_agent()
        if not agent:
            return jsonify({"error": "Agent not available"}), 503
        result = agent.chat_with_results(prompt, user_id="dashboard", channel="web")
        return jsonify({"response": result.text})
    except Exception as e:
        logger.warning("Briefing action error: %s", e)
        return jsonify({"error": "Failed to process action"}), 500


@app.route("/api/connect/discover", methods=["POST"])
@require_auth
def api_connect_discover():
    """Scan LAN for self-hosted services."""
    from familiar.channels.connect_wizard._discovery import discover_lan_services

    services = (request.json or {}).get("services") if request.is_json else None
    results = discover_lan_services(services=services)
    return jsonify(results)


@app.route("/api/connect/status")
@require_auth
def api_connect_status():
    """Get connection status of all supported services."""
    agent = get_agent()
    current_provider = agent.provider.name if agent else "unknown"

    services = []

    def _add(name, display_name, category, connected, detail=None, cmd=None):
        services.append(
            {
                "name": name,
                "display_name": display_name,
                "category": category,
                "connected": connected,
                "detail": detail,
                "connect_command": cmd or f"/connect {name}",
            }
        )

    # ── LLM Providers ────────────────────────────────────────
    anthropic_key = os.environ.get("ANTHROPIC_API_KEY", "")
    _add(
        "anthropic",
        "Anthropic (Claude)",
        "llm",
        bool(anthropic_key),
        f"...{anthropic_key[-4:]}" if len(anthropic_key) >= 4 else None,
    )
    openai_key = os.environ.get("OPENAI_API_KEY", "")
    _add(
        "openai",
        "OpenAI (GPT)",
        "llm",
        bool(openai_key),
        f"...{openai_key[-4:]}" if len(openai_key) >= 4 else None,
    )
    gemini_key = os.environ.get("GEMINI_API_KEY", "")
    _add(
        "gemini",
        "Gemini (Google AI)",
        "llm",
        bool(gemini_key),
        f"...{gemini_key[-4:]}" if len(gemini_key) >= 4 else None,
    )
    # Ollama — reachable if the API responds on localhost
    has_ollama = False
    ollama_detail = None
    try:
        import socket as _sock

        with _sock.create_connection(("127.0.0.1", 11434), timeout=2.0):
            has_ollama = True
        ollama_detail = current_provider if "Ollama" in current_provider else "Running"
    except (OSError, TimeoutError) as e:
        logger.debug("I/O error suppressed: %s", e)
    _add("ollama", "Ollama (local)", "llm", has_ollama, ollama_detail)

    # ── Integrations ─────────────────────────────────────────
    data_dir = DATA_DIR

    has_google = (data_dir / "google_credentials.json").exists()
    _add("google", "Google Workspace", "integrations", has_google, cmd="/connect google")

    has_gmail = (data_dir / "google_token_gmail.json").exists()
    _add("gmail", "Gmail API", "integrations", has_gmail, cmd="/connect google auth gmail")

    has_calendar_token = (data_dir / "google_token.json").exists()
    has_caldav = bool(os.environ.get("CALDAV_URL"))
    has_calendar = has_calendar_token or has_caldav
    _add(
        "calendar",
        "Calendar",
        "integrations",
        has_calendar,
        "CalDAV" if has_caldav else ("Google OAuth" if has_calendar_token else None),
        "/connect calendar",
    )

    has_drive = (data_dir / "google_token_drive.json").exists()
    _add("drive", "Google Drive", "integrations", has_drive, cmd="/connect google auth drive")

    try:
        from familiar.skills.email.accounts import get_all_accounts

        email_accounts = get_all_accounts()
        has_email = bool(email_accounts)
        email_detail = ", ".join(a["address"] for a in email_accounts) if email_accounts else None
    except Exception:
        has_email = bool(os.environ.get("EMAIL_ADDRESS"))
        email_detail = os.environ.get("EMAIL_ADDRESS") if has_email else None
    _add("email", "Email IMAP/SMTP", "integrations", has_email, email_detail, "/connect email")

    has_twilio = bool(os.environ.get("TWILIO_ACCOUNT_SID"))
    _add("sms", "SMS (Twilio)", "integrations", has_twilio, cmd="/connect sms")

    has_browser = False
    try:
        import playwright  # noqa: F401

        has_browser = True
    except Exception as e:
        logger.debug("Error suppressed: %s", e)
    _add("browser", "Browser (Playwright)", "integrations", has_browser, cmd="/connect browser")

    has_voice = False
    try:
        import faster_whisper  # noqa: F401

        has_voice = True
    except Exception:
        try:
            import whisper  # noqa: F401

            has_voice = True
        except Exception as e:
            logger.debug("Error suppressed: %s", e)
    _add("voice", "Voice (Whisper)", "integrations", has_voice, cmd="/connect voice")

    _add("websearch", "WebSearch (DuckDuckGo)", "integrations", True, "Built-in")

    _add(
        "github",
        "GitHub (issues & PRs)",
        "integrations",
        bool(os.environ.get("GITHUB_TOKEN")),
        cmd="/connect github",
    )
    _add(
        "slack",
        "Slack (Socket Mode)",
        "integrations",
        bool(os.environ.get("SLACK_BOT_TOKEN") and os.environ.get("SLACK_APP_TOKEN")),
        cmd="/connect slack",
    )

    # ── Self-Hosted ──────────────────────────────────────────
    for env_name, svc_name, display in [
        ("JELLYFIN_URL", "jellyfin", "Jellyfin (media)"),
        ("GITEA_URL", "gitea", "Gitea (code)"),
        ("HOMEASSISTANT_URL", "homeassistant", "Home Assistant"),
        ("JOPLIN_TOKEN", "joplin", "Joplin (notes)"),
        ("PIHOLE_URL", "pihole", "Pi-hole / AdGuard"),
        ("NEXTCLOUD_URL", "nextcloud", "Nextcloud"),
    ]:
        val = os.environ.get(env_name, "")
        _add(svc_name, display, "selfhosted", bool(val), val[:40] if val else None)

    # Stalwart Mail — TCP check on admin port or env var
    import socket

    _has_stalwart = bool(os.environ.get("EMAIL_SERVER_DOMAIN"))
    if not _has_stalwart:
        try:
            with socket.create_connection(("127.0.0.1", 8010), timeout=2.0):
                _has_stalwart = True
        except (OSError, TimeoutError):
            pass
    _stalwart_url = os.environ.get("EMAIL_SERVER_DOMAIN", "localhost:8010")
    _add(
        "email_server",
        "Stalwart Mail",
        "selfhosted",
        _has_stalwart,
        _stalwart_url if _has_stalwart else None,
    )

    # Proton Bridge — TCP check
    try:
        with socket.create_connection(("127.0.0.1", 1143), timeout=2.0):
            has_proton = True
    except (OSError, TimeoutError):
        has_proton = False
    _add("proton", "Proton Bridge", "selfhosted", has_proton, cmd="/connect proton")

    # ── Security ─────────────────────────────────────────────
    has_vaultwarden = bool(os.environ.get("VAULTWARDEN_URL"))
    _add(
        "vaultwarden",
        "Vaultwarden (passwords)",
        "security",
        has_vaultwarden,
        cmd="/connect security",
    )

    key_dir = AGENT_DIR / "keys"
    has_encryption = any(key_dir.glob("*.key")) if key_dir.exists() else False
    _add("encryption", "Encryption (at rest)", "security", has_encryption, cmd="/connect security")

    categories = ["llm", "integrations", "selfhosted", "security"]

    return jsonify(
        {
            "services": services,
            "categories": categories,
            "active_provider": current_provider,
        }
    )


@app.route("/api/connect/browse/browser/navigate", methods=["POST"])
@limiter.limit("10 per minute")
@require_auth
def api_browser_navigate():
    """Navigate the browser to a URL and return a page preview."""
    data = request.json or {}
    url = data.get("url", "").strip()
    if not url:
        return jsonify({"success": False, "error": "No URL provided"}), 400

    try:
        from familiar.skills.browser.skill import get_page_content, navigate

        nav_result = navigate({"url": url})
        if nav_result.startswith("❌"):
            return jsonify({"success": False, "error": nav_result})

        # Check for video URL before fetching page content
        video_detected = is_video_url(url)
        video_id = _extract_video_id(url) if video_detected else ""

        content_result = get_page_content({})
        if content_result.startswith("❌"):
            # Navigation succeeded but content fetch failed — return what we have
            result = {
                "success": True,
                "title": nav_result.split("\n")[0].replace("✓ Navigated to: ", ""),
                "url": url,
                "content": "",
            }
            if video_detected:
                result["is_video"] = True
                result["video_id"] = video_id
            return jsonify(result)

        # Parse: "📄 Title\nURL: ...\n\nContent..."
        lines = content_result.split("\n", 3)
        title = lines[0].lstrip("📄 ").strip() if lines else ""
        page_url = lines[1].replace("URL: ", "").strip() if len(lines) > 1 else url
        content = lines[3].strip() if len(lines) > 3 else ""
        content = content[:2000]

        result = {
            "success": True,
            "title": title,
            "url": page_url,
            "content": content,
        }
        if video_detected:
            result["is_video"] = True
            result["video_id"] = video_id
        return jsonify(result)
    except Exception:
        logger.exception("Browser navigate error")
        return jsonify({"success": False, "error": "Navigation failed"}), 500


@app.route("/api/connect/browse/browser/video/embed", methods=["POST"])
@require_auth
def api_browser_video_embed():
    """Get privacy-friendly embed HTML for a video URL."""
    url = (request.json or {}).get("url", "").strip()
    if not url:
        return jsonify({"success": False, "error": "No URL provided"}), 400
    try:
        video_id = _extract_video_id(url)
        if video_id and is_video_url(url):
            from urllib.parse import urlparse

            host = (urlparse(url).hostname or "").lstrip("www.").lstrip("m.")
            if host in ("youtube.com", "youtu.be"):
                # youtube-nocookie.com = YouTube's privacy-enhanced mode
                # (no cookies, no tracking, always available)
                html = (
                    f'<iframe width="640" height="360"'
                    f' src="https://www.youtube-nocookie.com/embed/{video_id}"'
                    f' frameborder="0"'
                    f' allow="accelerometer; autoplay; encrypted-media;'
                    f' gyroscope; picture-in-picture"'
                    f" allowfullscreen></iframe>"
                )
                return jsonify({"success": True, "html": html})
        # Fallback: try PrivacyPlayer for non-YouTube
        import asyncio

        from familiar.skills.video.player import PrivacyPlayer

        class _Cfg:
            invidious_instance = "https://yewtu.be"

        player = PrivacyPlayer(_Cfg())

        async def _get():
            try:
                return await player.get_embed_html(url, width=640, height=360, autoplay=False)
            finally:
                await player.close()

        html = asyncio.run(_get())
        return jsonify({"success": True, "html": html})
    except Exception:
        logger.exception("Video embed error")
        return jsonify({"success": False, "error": "Video embed failed"}), 500


@app.route("/api/connect/browse/browser/video/download", methods=["POST"])
@require_auth
def api_browser_video_download():
    """Download a video using yt-dlp."""
    data = request.json or {}
    url = data.get("url", "").strip()
    quality = data.get("quality", "720p")
    if not url:
        return jsonify({"success": False, "error": "No URL provided"}), 400
    try:
        import asyncio

        from familiar.skills.video import VideoSkill

        config = {"video": {"default_quality": quality}}
        skill = VideoSkill(None, config)

        async def _download():
            await skill.start()
            try:
                meta = await skill.download(url, quality=quality, transcribe=False)
                return {
                    "success": True,
                    "title": meta.title,
                    "path": meta.local_path or "",
                    "size": f"{(meta.file_size_bytes or 0) / 1048576:.1f} MB",
                    "duration": meta.duration_formatted,
                }
            finally:
                await skill.stop()

        result = asyncio.run(_download())
        return jsonify(result)
    except Exception:
        logger.exception("Video download error")
        return jsonify({"success": False, "error": "Video download failed"}), 500


def _parse_search_results(response: str) -> dict:
    """Extract structured JSON from an agent response containing a SEARCH_RESULTS_JSON: marker.

    The agent's tool returns human-readable text followed by a
    ``SEARCH_RESULTS_JSON:`` line and a JSON object.  This helper finds
    the marker and parses the JSON using brace-depth counting so that
    any trailing text the agent appends after the JSON is ignored.

    Falls back to ``{"items": [], "summary": <truncated response>}``
    when no marker is present (agent responded conversationally).
    """
    marker = "SEARCH_RESULTS_JSON:"
    idx = response.find(marker)
    if idx == -1:
        return {"items": [], "summary": response[:200]}

    json_start = response.find("{", idx + len(marker))
    if json_start == -1:
        return {"items": [], "summary": response[:200]}

    # Brace-depth counting to find the matching closing brace
    depth = 0
    in_string = False
    escape = False
    for i in range(json_start, len(response)):
        ch = response[i]
        if escape:
            escape = False
            continue
        if ch == "\\":
            if in_string:
                escape = True
            continue
        if ch == '"':
            in_string = not in_string
            continue
        if in_string:
            continue
        if ch == "{":
            depth += 1
        elif ch == "}":
            depth -= 1
            if depth == 0:
                try:
                    return json.loads(response[json_start : i + 1])
                except (json.JSONDecodeError, ValueError):
                    break

    return {"items": [], "summary": response[:200]}


@app.route("/api/connect/browse/video/search", methods=["POST"])
@limiter.limit("10 per minute")
@require_auth
def api_video_search_online():
    """Search videos online — direct DuckDuckGo search with agent fallback."""
    query = (request.json or {}).get("query", "").strip()
    if not query:
        return jsonify({"items": [], "error": "No query provided"}), 400
    platforms = (request.json or {}).get("platforms") or None

    # Direct search first (fast, reliable)
    from familiar.channels.connect_wizard._browse import search_video_online

    result = search_video_online(query, platforms=platforms)

    # Fall back to agent if direct search returned nothing
    if not result.get("items") and not result.get("error"):
        try:
            agent = get_agent()
            prompt = f"Search for videos: {query}"
            agent_result = agent.chat_with_results(prompt, user_id="dashboard", channel="web")
            agent_parsed = _parse_search_results(agent_result.text)
            if agent_parsed.get("items"):
                result = agent_parsed
        except Exception:
            logger.debug("Agent video search fallback also failed")

    if result.get("error"):
        return jsonify(result), 500
    return jsonify(result)


# ── Media Video Aliases ──────────────────────────────────────────────
# Thin aliases so the Media panel's Video tab can call cleaner paths.


@app.route("/api/media/video/search", methods=["POST"])
@require_auth
def api_media_video_search():
    return api_video_search_online()


@app.route("/api/media/video/embed", methods=["POST"])
@require_auth
def api_media_video_embed():
    return api_browser_video_embed()


@app.route("/api/media/video/download", methods=["POST"])
@require_auth
def api_media_video_download():
    return api_browser_video_download()


@app.route("/api/connect/browse/search", methods=["POST"])
@limiter.limit("10 per minute")
@require_auth
def api_connect_browse_search():
    """Search a connected service, routed through the agent for intelligent results."""
    data = request.json or {}
    service = data.get("service", "").strip().lower()
    query = data.get("query", "").strip()
    if not service or not query:
        return jsonify({"items": [], "error": "Missing service or query"}), 400
    if service not in SEARCHABLE_SERVICES:
        return jsonify({"items": [], "error": f"Unknown service: {service}"}), 400

    # Direct search first (fast, reliable)
    result = search_service(service, query)

    # Fall back to agent if direct search returned nothing
    if not result.get("items") and not result.get("error"):
        try:
            agent = get_agent()
            prompt = f"Search {service} for: {query}"
            agent_result = agent.chat_with_results(prompt, user_id="dashboard", channel="web")
            agent_parsed = _parse_search_results(agent_result.text)
            if agent_parsed.get("items"):
                result = agent_parsed
        except Exception:
            logger.debug("Agent search fallback also failed")

    if result.get("error"):
        return jsonify(result), 400 if "Unknown service" in result["error"] else 500
    return jsonify(result)


@app.route("/api/connect/browse")
@limiter.limit("10 per minute")
@require_auth
def api_connect_browse():
    """Browse data from connected services."""
    try:
        return jsonify({"services": browse_all()})
    except Exception:
        logger.exception("Browse all services failed")
        return jsonify({"services": {}})


# ============================================================
# Connect Wizard Routes
# ============================================================


@app.route("/api/connect/wizard/start", methods=["POST"])
@require_auth
def api_wizard_start():
    """Start (or restart) the connect wizard."""
    data = request.json or {}
    args = data.get("args", [])
    host = get_wizard_host()
    messages = host.start_wizard(args)
    return jsonify({"wizard_messages": messages})


@app.route("/api/connect/wizard/select", methods=["POST"])
@require_auth
def api_wizard_select():
    """Handle a wizard button click."""
    data = request.json or {}
    key = data.get("key", "")
    if not key:
        return jsonify({"error": "No key provided"}), 400
    # Google OAuth keys are handled by the dedicated panel — bypass the wizard
    if key.startswith("google_auth_") or key == "google_upload_creds":
        return jsonify({"redirect": "google_oauth"})
    host = get_wizard_host()
    messages = host.select_menu(key)
    return jsonify({"wizard_messages": messages})


@app.route("/api/connect/wizard/tab", methods=["POST"])
@require_auth
def api_wizard_tab():
    """Switch to a wizard tab."""
    data = request.json or {}
    tab = data.get("tab", "")
    if not tab:
        return jsonify({"error": "No tab provided"}), 400
    host = get_wizard_host()
    messages = host.switch_tab(tab)
    return jsonify({"wizard_messages": messages})


@app.route("/api/connect/wizard/input", methods=["POST"])
@require_auth
def api_wizard_input():
    """Send text input to the wizard (e.g. API key, URL)."""
    data = request.json or {}
    text = data.get("text", "")
    if not text:
        return jsonify({"error": "No text provided"}), 400
    host = get_wizard_host()
    messages = host.send_text(text)
    return jsonify({"wizard_messages": messages})


@app.route("/api/connect/upload/google-credentials", methods=["POST"])
@require_auth
def api_upload_google_credentials():
    """Upload Google OAuth credentials.json file."""
    if "file" not in request.files:
        return jsonify({"error": "No file uploaded"}), 400
    f = request.files["file"]
    if not f.filename:
        return jsonify({"error": "Empty filename"}), 400
    import json as _json

    try:
        content = f.read()
        data = _json.loads(content)
        # Validate it looks like a Google credentials file
        if "installed" not in data and "web" not in data:
            return jsonify({"error": "Not a valid Google OAuth credentials file"}), 400
    except _json.JSONDecodeError:
        return jsonify({"error": "Invalid JSON file"}), 400
    dest = DATA_DIR / "google_credentials.json"
    dest.parent.mkdir(parents=True, exist_ok=True)
    dest.write_bytes(content)
    cred_type = "Desktop app" if "installed" in data else "Web app"
    return jsonify({"success": True, "type": cred_type, "path": str(dest)})


# ── Google OAuth (dashboard-native, no wizard) ────────────────────────────────
# Keyed by service name ("gmail", "calendar", "drive", "contacts").
# Each entry: {"done": bool, "success": bool, "error": str|None, "auth_url": str|None}
_google_oauth_state: dict = {}
_google_oauth_lock = threading.Lock()

_GOOGLE_SERVICES = {
    "calendar": {
        "token_file": "google_token.json",
        "scopes": ["https://www.googleapis.com/auth/calendar"],
    },
    "gmail": {
        "token_file": "google_token_gmail.json",
        "scopes": [
            "https://www.googleapis.com/auth/gmail.readonly",
            "https://www.googleapis.com/auth/gmail.send",
            "https://www.googleapis.com/auth/gmail.modify",
        ],
    },
    "drive": {
        "token_file": "google_token_drive.json",
        "scopes": [
            "https://www.googleapis.com/auth/drive.readonly",
            "https://www.googleapis.com/auth/drive.file",
        ],
    },
    "contacts": {
        "token_file": "google_token_contacts.json",
        "scopes": ["https://www.googleapis.com/auth/contacts.readonly"],
    },
}


def _run_google_oauth_thread(service: str, creds_path: Path, token_path: Path, scopes: list):
    """Background thread: bind port → build auth URL → wait for redirect → save token."""
    import http.server
    import os as _os
    import urllib.parse

    # Allow Google to return broader scopes than requested (it includes all
    # previously-granted scopes in the token response; without this flag
    # requests_oauthlib raises a scope mismatch error).
    _os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1"

    with _google_oauth_lock:
        state = _google_oauth_state[service]
    try:
        from google_auth_oauthlib.flow import Flow

        flow = Flow.from_client_secrets_file(
            str(creds_path),
            scopes=scopes,
            redirect_uri=None,  # set after port bind
        )

        # Bind a free port
        server_address = ("127.0.0.1", 0)
        httpd = http.server.HTTPServer(server_address, http.server.BaseHTTPRequestHandler)
        port = httpd.server_address[1]
        redirect_uri = f"http://127.0.0.1:{port}/"
        flow.redirect_uri = redirect_uri

        auth_url, _ = flow.authorization_url(
            access_type="offline",
            prompt="consent",
        )
        with _google_oauth_lock:
            state["auth_url"] = auth_url

        # Capture the redirect request
        received: dict = {}

        class _Handler(http.server.BaseHTTPRequestHandler):
            def log_message(self, fmt, *args):  # silence server logs
                pass

            def do_GET(self):
                received["path"] = self.path
                self.send_response(200)
                self.send_header("Content-Type", "text/html")
                self.end_headers()
                self.wfile.write(
                    b"<html><body><h2>Authorization complete!</h2>"
                    b"<p>You can close this tab and return to Familiar.</p>"
                    b"<script>window.close();</script></body></html>"
                )

        httpd.RequestHandlerClass = _Handler
        httpd.handle_request()  # blocks until browser redirects

        # Exchange code for token
        parsed = urllib.parse.urlparse(received.get("path", "/"))
        params = dict(urllib.parse.parse_qsl(parsed.query))
        if "error" in params:
            with _google_oauth_lock:
                state["error"] = params["error"]
                state["done"] = True
            return

        flow.fetch_token(code=params["code"])
        from familiar.core.paths import save_secure_file
        save_secure_file(token_path, flow.credentials.to_json())
        with _google_oauth_lock:
            state["success"] = True
            state["done"] = True
    except Exception as exc:
        with _google_oauth_lock:
            state["error"] = str(exc)
            state["done"] = True
    finally:
        try:
            httpd.server_close()
        except Exception:
            pass


@app.route("/api/connect/google/start/<service>", methods=["POST"])
@require_auth
def api_google_oauth_start(service: str):
    """Start Google OAuth for the given service. Returns auth_url for the browser."""
    if service not in _GOOGLE_SERVICES:
        return jsonify({"error": f"Unknown service: {service}"}), 400

    creds_path = DATA_DIR / "google_credentials.json"
    if not creds_path.exists():
        return jsonify({"error": "google_credentials.json not found — upload it first"}), 400

    cfg = _GOOGLE_SERVICES[service]
    token_path = DATA_DIR / cfg["token_file"]

    # Reset state for this service
    with _google_oauth_lock:
        _google_oauth_state[service] = {
            "done": False, "success": False, "error": None, "auth_url": None,
        }

    t = threading.Thread(
        target=_run_google_oauth_thread,
        args=(service, creds_path, token_path, cfg["scopes"]),
        daemon=False,
        name=f"google-oauth-{service}",
    )
    t.start()

    # Wait up to 3 s for the auth URL to appear (the port bind is nearly instant)
    deadline = time.time() + 3.0
    while time.time() < deadline:
        with _google_oauth_lock:
            st = _google_oauth_state[service]
            if st["auth_url"] or st["done"]:
                break
        time.sleep(0.05)

    with _google_oauth_lock:
        st = dict(_google_oauth_state[service])
    if st.get("error"):
        return jsonify({"error": st["error"]}), 500
    return jsonify({"auth_url": st.get("auth_url"), "service": service})


@app.route("/api/connect/google/status/<service>")
@require_auth
def api_google_oauth_status(service: str):
    """Poll whether the OAuth flow for a service has completed."""
    with _google_oauth_lock:
        st = _google_oauth_state.get(service)
        if st is None:
            return jsonify({"done": False, "success": False, "error": "No active flow"})
        return jsonify({
            "done": st["done"],
            "success": st["success"],
            "error": st["error"],
        })


@app.route("/api/connect/google/reset", methods=["POST"])
@require_auth
def api_google_reset():
    """Delete Google credential and/or token files."""
    data = request.json or {}
    target = data.get("target", "tokens")  # "tokens" | "all"
    data_dir = DATA_DIR
    removed = []
    files = [cfg["token_file"] for cfg in _GOOGLE_SERVICES.values()]
    if target == "all":
        files.append("google_credentials.json")
    for fname in files:
        p = data_dir / fname
        if p.exists():
            p.unlink()
            removed.append(fname)
    return jsonify({"removed": removed})


@app.route("/api/connect/google/token-status")
@require_auth
def api_google_token_status():
    """Return which Google service tokens exist on disk."""
    data_dir = DATA_DIR
    result = {}
    for svc, cfg in _GOOGLE_SERVICES.items():
        result[svc] = (data_dir / cfg["token_file"]).exists()
    result["credentials"] = (data_dir / "google_credentials.json").exists()
    return jsonify(result)


@app.route("/api/connect/wizard/cancel", methods=["POST"])
@require_auth
def api_wizard_cancel():
    """Cancel any active wizard session so chat routes to the agent."""
    host = get_wizard_host()
    host.cancel()
    return jsonify({"success": True})


@app.route("/api/connect/wizard/state")
@require_auth
def api_wizard_state():
    """Check whether a wizard session is active."""
    host = get_wizard_host()
    return jsonify({"active": host.is_active()})


@app.route("/api/connect/wizard/context")
@require_auth
def api_wizard_context():
    """Return context about the active wizard step (for Helper Hat handoff)."""
    host = get_wizard_host()
    return jsonify(host.get_current_context())


@app.route("/api/wizard/handoff", methods=["POST"])
@require_auth
def api_wizard_handoff():
    """Hand off an active wizard to the Helper agent for completion.

    The wizard is cancelled and a structured prompt is sent to the agent
    so it can finish the setup using its host_setup / shell tools.
    """
    data = request.json or {}
    service_name = data.get("service_name", "unknown service")
    step = data.get("step", "")
    context = data.get("context", "")
    source = data.get("source", "connect")

    # Cancel the active wizard so chat routes to the agent
    host = get_wizard_host()
    host.cancel()

    # Build a prompt that gives the agent full context
    prompt_parts = [
        f"The user needs help setting up {service_name}.",
    ]
    if step:
        prompt_parts.append(f"They were on step '{step}' of the {source} wizard.")
    if context:
        prompt_parts.append(f"Context: {context}")
    prompt_parts.append(
        "Help them complete the installation using your host_setup, "
        "shell, and file tools. Detect the system first, then install "
        "and configure what's needed. Ask for any credentials or values "
        "you don't have."
    )
    prompt = " ".join(prompt_parts)

    agent = get_agent()
    result = agent.chat_with_results(prompt, user_id="dashboard", channel="web")

    return jsonify({
        "response": result.text,
        "panel": "chat",
        "timestamp": datetime.now().isoformat(),
    })


@app.route("/api/config")
@require_auth
def api_config():
    """Get current configuration."""
    agent = get_agent()
    config = {
        "agent_name": agent.config.agent.name,
        "provider": agent.provider.name,
        "memory_enabled": agent.config.agent.memory_enabled,
        "skills_enabled": agent.config.agent.skills_enabled,
        "scheduler_enabled": agent.config.agent.scheduler_enabled,
    }
    return jsonify(config)


@app.route("/api/config", methods=["POST"])
@require_auth
def api_config_update():
    """Update configuration."""
    data = request.json or {}
    # For now just switch provider
    if "provider" in data:
        agent = get_agent()
        result = agent.switch_provider(data["provider"])
        return jsonify({"result": result})

    return jsonify({"error": "No valid config option"}), 400


@app.route("/api/config/routing")
@require_auth
def api_config_routing():
    """Get current model routing configuration."""
    agent = get_agent()
    return jsonify(agent.get_routing_config())


@app.route("/api/config/routing", methods=["POST"])
@require_auth
def api_config_routing_update():
    """Update model routing mode and manual route assignments."""
    data = request.json or {}
    mode = data.get("mode", "auto")
    routes = data.get("routes")
    agent = get_agent()
    try:
        result = agent.set_routing_config(mode, routes)
        return jsonify(result)
    except ValueError:
        return jsonify({"error": "Invalid routing configuration"}), 400


@app.route("/api/config/routing/presets")
@require_auth
def api_routing_presets():
    """Get all routing preset slots."""
    agent = get_agent()
    return jsonify(agent.get_presets())


@app.route("/api/config/routing/presets/save", methods=["POST"])
@require_auth
def api_routing_presets_save():
    """Save current routing config to a preset slot."""
    data = request.json or {}
    slot = data.get("slot")
    name = data.get("name")
    agent = get_agent()
    try:
        result = agent.save_preset(slot, name)
        return jsonify(result)
    except (ValueError, TypeError) as e:
        return jsonify({"error": str(e)}), 400


@app.route("/api/config/routing/presets/load", methods=["POST"])
@require_auth
def api_routing_presets_load():
    """Load a preset into the active routing config."""
    data = request.json or {}
    slot = data.get("slot")
    agent = get_agent()
    try:
        result = agent.load_preset(slot)
        return jsonify(result)
    except (ValueError, TypeError) as e:
        return jsonify({"error": str(e)}), 400


@app.route("/api/config/routing/presets/default", methods=["POST"])
@require_auth
def api_routing_presets_default():
    """Set which preset auto-loads on startup."""
    data = request.json or {}
    slot = data.get("slot")
    agent = get_agent()
    try:
        result = agent.set_active_preset(slot)
        return jsonify(result)
    except (ValueError, TypeError) as e:
        return jsonify({"error": str(e)}), 400


@app.route("/api/config/routing/presets/rename", methods=["POST"])
@require_auth
def api_routing_presets_rename():
    """Rename a preset slot."""
    data = request.json or {}
    slot = data.get("slot")
    name = data.get("name", "")
    agent = get_agent()
    try:
        result = agent.rename_preset(slot, name)
        return jsonify(result)
    except (ValueError, TypeError) as e:
        return jsonify({"error": str(e)}), 400


@app.route("/api/config/routing/presets/delete", methods=["POST"])
@require_auth
def api_routing_presets_delete():
    """Clear a preset slot."""
    data = request.json or {}
    slot = data.get("slot")
    agent = get_agent()
    try:
        result = agent.delete_preset(slot)
        return jsonify(result)
    except (ValueError, TypeError) as e:
        return jsonify({"error": str(e)}), 400


@app.route("/api/config/job-class")
@require_auth
def api_job_class():
    """Get available job classes and the currently active one."""
    from familiar.creature.state import load_creature
    from familiar.jobs import list_job_classes

    creature = load_creature()
    active = creature.job_class if creature else ""
    classes = [
        {"key": jc.key, "name": jc.name, "icon": jc.icon, "description": jc.description}
        for jc in list_job_classes()
    ]
    return jsonify({"active": active, "classes": classes})


@app.route("/api/config/job-class", methods=["POST"])
@require_auth
def api_job_class_update():
    """Switch the active job class."""
    data = request.json or {}
    key = data.get("job_class", "")
    agent = get_agent()
    result = agent.switch_job_class(key)
    return jsonify({"result": result, "active": key})


# ============================================================
# Workspace Panel
# ============================================================


@app.route("/api/workspace")
@require_auth
def api_workspace():
    """Get workspace config + computed overview metrics for the active job class."""
    try:
        from familiar.jobs import get_active_job_class

        jc = get_active_job_class()
        if not jc or not jc.workspace:
            return jsonify({"workspace": None})

        from familiar.dashboard.workspace import (
            get_nonprofit_alerts,
            resolve_overview_cards,
        )

        ws = dict(jc.workspace)
        overview = ws.get("overview", {})
        if overview:
            ws["overview"]["computed_cards"] = resolve_overview_cards(overview)

        # Add smart alerts for nonprofit workspace
        if jc.key == "nonprofit_director":
            try:
                ws["alerts"] = get_nonprofit_alerts()
            except Exception:
                ws["alerts"] = []

        return jsonify({"workspace": ws})
    except Exception as e:
        logger.warning("Workspace load error: %s", e)
        return jsonify({"error": str(e)}), 500


@app.route("/api/workspace/section/<key>")
@require_auth
def api_workspace_section(key):
    """Get data for a specific workspace section."""
    try:
        from familiar.jobs import get_active_job_class

        jc = get_active_job_class()
        if not jc or not jc.workspace:
            return jsonify({"error": "No active workspace"}), 404

        sections = jc.workspace.get("sections", [])
        section_config = next((s for s in sections if s.get("key") == key), None)
        if not section_config:
            return jsonify({"error": f"Section '{key}' not found"}), 404

        from familiar.dashboard.workspace import get_section_data

        sort = request.args.get("sort")
        sort_dir = request.args.get("dir", "asc")
        page = int(request.args.get("page", 1))
        limit = int(request.args.get("limit", 50))

        data = get_section_data(section_config, sort, sort_dir, page, limit)
        return jsonify(data)
    except Exception as e:
        logger.warning("Workspace section error: %s", e)
        return jsonify({"error": str(e)}), 500


@app.route("/api/workspace/action", methods=["POST"])
@require_auth
def api_workspace_action():
    """Execute a workspace action (tool call or agent chat)."""
    try:
        payload = request.get_json() or {}
        action_config = {
            "mode": payload.get("mode", "agent"),
            "tool": payload.get("tool", ""),
            "tool_args": payload.get("tool_args", {}),
            "prompt": payload.get("prompt", ""),
            "prompt_template": payload.get("prompt_template", ""),
        }
        row_data = payload.get("row_data", {})

        from familiar.dashboard.workspace import execute_action

        agent = get_agent()
        result = execute_action(action_config, row_data, agent)
        return jsonify(result)
    except Exception as e:
        logger.warning("Workspace action error: %s", e)
        return jsonify({"result": str(e), "success": False}), 500


# ============================================================
# Workspace Data CRUD (generic JSON file records)
# ============================================================

# Allowed data files and their keys — prevents arbitrary file access
_WS_DATA_REGISTRY = {
    "studio": {"file": "studio.json", "keys": {"pieces", "commissions", "shows", "clients"}},
    "kitchen": {"file": "kitchen.json", "keys": {"recipes", "menus", "vendors"}},
    "security": {"file": "security.json", "keys": {"incidents"}},
    "osint": {"file": "osint.json", "keys": {"investigations"}},
    "network": {"file": "network.json", "keys": {"hosts"}},
}


def _ws_load_data(source: str) -> tuple:
    """Load a workspace JSON data file. Returns (data_dict, path)."""
    import json as _json
    from familiar.core.paths import DATA_DIR

    reg = _WS_DATA_REGISTRY.get(source)
    if not reg:
        return {}, None
    path = DATA_DIR / reg["file"]
    if not path.exists():
        return {}, path
    try:
        with open(path, encoding="utf-8") as f:
            return _json.load(f), path
    except (ValueError, IOError):
        return {}, path


def _ws_save_data(source: str, data: dict) -> bool:
    """Save workspace JSON data file (atomic: write to temp, then rename)."""
    import json as _json
    import tempfile
    from familiar.core.paths import DATA_DIR

    reg = _WS_DATA_REGISTRY.get(source)
    if not reg:
        return False
    path = DATA_DIR / reg["file"]
    path.parent.mkdir(parents=True, exist_ok=True)
    fd, tmp = tempfile.mkstemp(dir=path.parent, suffix=".tmp")
    try:
        with os.fdopen(fd, "w", encoding="utf-8") as f:
            _json.dump(data, f, indent=2, default=str)
        os.replace(tmp, path)
    except BaseException:
        try:
            os.unlink(tmp)
        except OSError:
            pass
        raise
    return True


@app.route("/api/workspace/data/<source>/<key>", methods=["POST"])
@require_auth
def api_workspace_data_create(source, key):
    """Create a record in a workspace data file."""
    import uuid

    reg = _WS_DATA_REGISTRY.get(source)
    if not reg or key not in reg["keys"]:
        return jsonify({"error": "Invalid source/key"}), 400
    payload = request.get_json(silent=True) or {}
    data, _ = _ws_load_data(source)
    records = data.get(key, [])
    record = {**payload, "id": str(uuid.uuid4())[:8]}
    records.append(record)
    data[key] = records
    _ws_save_data(source, data)
    return jsonify({"record": record, "id": record["id"]}), 201


@app.route("/api/workspace/data/<source>/<key>/<record_id>", methods=["PUT"])
@require_auth
def api_workspace_data_update(source, key, record_id):
    """Update a record in a workspace data file."""
    reg = _WS_DATA_REGISTRY.get(source)
    if not reg or key not in reg["keys"]:
        return jsonify({"error": "Invalid source/key"}), 400
    payload = request.get_json(silent=True) or {}
    data, _ = _ws_load_data(source)
    records = data.get(key, [])
    for i, r in enumerate(records):
        if r.get("id") == record_id:
            records[i] = {**r, **payload, "id": record_id}
            data[key] = records
            _ws_save_data(source, data)
            return jsonify({"record": records[i]})
    return jsonify({"error": "Record not found"}), 404


@app.route("/api/workspace/data/<source>/<key>/<record_id>", methods=["DELETE"])
@require_auth
def api_workspace_data_delete(source, key, record_id):
    """Delete a record from a workspace data file."""
    reg = _WS_DATA_REGISTRY.get(source)
    if not reg or key not in reg["keys"]:
        return jsonify({"error": "Invalid source/key"}), 400
    data, _ = _ws_load_data(source)
    records = data.get(key, [])
    before = len(records)
    records = [r for r in records if r.get("id") != record_id]
    if len(records) == before:
        return jsonify({"error": "Record not found"}), 404
    data[key] = records
    _ws_save_data(source, data)
    return jsonify({"status": "ok"})


# ============================================================
# Media Panel (Radio & Podcasts)
# ============================================================


@app.route("/api/media/radio/search")
@require_auth
def api_media_radio_search():
    """Search radio stations."""
    try:
        from familiar.skills.media.radio import search_stations

        name = request.args.get("name", "")
        tag = request.args.get("tag", "")
        country = request.args.get("country", "")
        limit = int(request.args.get("limit", 25))
        stations = search_stations(name=name, tag=tag, country=country, limit=limit)
        return jsonify({"stations": stations})
    except Exception as e:
        logger.debug("Radio search error: %s", e)
        return jsonify({"stations": [], "error": str(e)}), 502


@app.route("/api/media/radio/genres")
@require_auth
def api_media_radio_genres():
    """Get popular radio genres."""
    try:
        from familiar.skills.media.radio import get_genres

        limit = int(request.args.get("limit", 50))
        genres = get_genres(limit=limit)
        return jsonify({"genres": genres})
    except Exception as e:
        logger.debug("Radio genres error: %s", e)
        return jsonify({"genres": [], "error": str(e)}), 502


@app.route("/api/media/radio/popular")
@require_auth
def api_media_radio_popular():
    """Get popular radio stations."""
    try:
        from familiar.skills.media.radio import get_popular

        limit = int(request.args.get("limit", 25))
        stations = get_popular(limit=limit)
        return jsonify({"stations": stations})
    except Exception as e:
        logger.debug("Radio popular error: %s", e)
        return jsonify({"stations": [], "error": str(e)}), 502


@app.route("/api/media/podcasts/search")
@require_auth
def api_media_podcasts_search():
    """Search podcasts via iTunes."""
    try:
        from familiar.skills.media.podcasts import search_podcasts

        term = request.args.get("term", "")
        limit = int(request.args.get("limit", 25))
        podcasts = search_podcasts(term=term, limit=limit)
        return jsonify({"podcasts": podcasts})
    except Exception as e:
        logger.debug("Podcast search error: %s", e)
        return jsonify({"podcasts": [], "error": str(e)}), 502


@app.route("/api/media/podcasts/episodes")
@require_auth
def api_media_podcasts_episodes():
    """Get podcast episodes from RSS feed."""
    try:
        from familiar.skills.media.podcasts import get_episodes

        feed_url = request.args.get("feed_url", "")
        limit = int(request.args.get("limit", 20))
        episodes = get_episodes(feed_url=feed_url, limit=limit)
        return jsonify({"episodes": episodes})
    except Exception as e:
        logger.debug("Podcast episodes error: %s", e)
        return jsonify({"episodes": [], "error": str(e)}), 502


@app.route("/api/media/favorites")
@require_auth
def api_media_favorites():
    """Get saved media favorites."""
    try:
        from familiar.skills.media.skill import get_favorites

        return jsonify({"favorites": get_favorites()})
    except Exception as e:
        logger.debug("Media favorites error: %s", e)
        return jsonify({"favorites": [], "error": str(e)}), 502


@app.route("/api/media/favorites/add", methods=["POST"])
@require_auth
def api_media_favorites_add():
    """Add a media item to favorites."""
    try:
        from familiar.skills.media.skill import add_favorite

        item = request.get_json() or {}
        added = add_favorite(item)
        return jsonify({"success": added})
    except Exception as e:
        logger.debug("Media favorite add error: %s", e)
        return jsonify({"success": False, "error": str(e)}), 500


@app.route("/api/media/favorites/remove", methods=["POST"])
@require_auth
def api_media_favorites_remove():
    """Remove a media item from favorites."""
    try:
        from familiar.skills.media.skill import remove_favorite

        item = request.get_json() or {}
        removed = remove_favorite(item)
        return jsonify({"success": removed})
    except Exception as e:
        logger.debug("Media favorite remove error: %s", e)
        return jsonify({"success": False, "error": str(e)}), 500


@app.route("/api/media/history")
@require_auth
def api_media_history():
    """Get media play history."""
    try:
        from familiar.skills.media.skill import get_history

        return jsonify({"history": get_history()})
    except Exception as e:
        logger.debug("Media history error: %s", e)
        return jsonify({"history": [], "error": str(e)}), 502


# ── Music Vault Endpoints ────────────────────────────────────


MUSIC_EXTS = {".mp3", ".wav", ".ogg", ".flac", ".m4a", ".aac", ".wma"}
MUSIC_MIME = {
    ".mp3": "audio/mpeg",
    ".wav": "audio/wav",
    ".ogg": "audio/ogg",
    ".flac": "audio/flac",
    ".m4a": "audio/mp4",
    ".aac": "audio/aac",
    ".wma": "audio/x-ms-wma",
}


@app.route("/api/media/music/upload", methods=["POST"])
@require_auth
def api_music_upload():
    """Upload a music file to the local music vault."""
    import uuid

    from familiar.skills.media.skill import _get_music_dir, add_to_music_library

    MAX_SIZE = 50 * 1024 * 1024  # 50 MB

    if "file" not in request.files:
        return jsonify({"error": "No file provided", "success": False}), 400

    file = request.files["file"]
    if not file.filename:
        return jsonify({"error": "No filename", "success": False}), 400

    ext = Path(file.filename).suffix.lower()
    if ext not in MUSIC_EXTS:
        return jsonify({"error": f"File type {ext} not supported", "success": False}), 400

    music_dir = _get_music_dir()
    track_id = str(uuid.uuid4())[:12]
    safe_name = re.sub(r"[^\w\-.]", "_", Path(file.filename).stem)[:50]
    dest = music_dir / f"{track_id}_{safe_name}{ext}"
    file.save(str(dest))

    size = dest.stat().st_size
    if size > MAX_SIZE:
        dest.unlink()
        return jsonify({"error": "File exceeds 50 MB limit", "success": False}), 400

    track = {
        "id": track_id,
        "filename": file.filename,
        "path": str(dest),
        "size": size,
        "ext": ext,
        "added_at": datetime.now().isoformat(),
    }
    add_to_music_library(track)

    return jsonify({"success": True, "track_id": track_id, "filename": file.filename,
                     "path": str(dest), "size": size})


@app.route("/api/media/music/library")
@require_auth
def api_music_library():
    """List the music library, optionally filtering by search term."""
    from familiar.skills.media.skill import get_music_library

    search = request.args.get("search", "").strip() or None
    tracks = get_music_library(search=search)
    return jsonify({"tracks": tracks})


@app.route("/api/media/music/stream/<track_id>")
@require_auth
def api_music_stream(track_id):
    """Stream a music file by track ID."""
    if not re.match(r"^[a-f0-9\-]{8,36}$", track_id):
        return jsonify({"error": "Invalid track ID"}), 400

    from familiar.skills.media.skill import get_music_library

    tracks = get_music_library()
    track = next((t for t in tracks if t.get("id") == track_id), None)
    if not track:
        return jsonify({"error": "Track not found"}), 404

    file_path = Path(track["path"])
    if not file_path.is_file():
        return jsonify({"error": "File not found on disk"}), 404

    mime = MUSIC_MIME.get(track.get("ext", ""), "application/octet-stream")
    return send_file(str(file_path), mimetype=mime)


@app.route("/api/media/music/<track_id>", methods=["DELETE"])
@require_auth
def api_music_delete(track_id):
    """Delete a music track from the library and disk."""
    if not re.match(r"^[a-f0-9\-]{8,36}$", track_id):
        return jsonify({"error": "Invalid track ID"}), 400

    from familiar.skills.media.skill import remove_from_music_library

    removed = remove_from_music_library(track_id)
    if not removed:
        return jsonify({"error": "Track not found"}), 404
    return jsonify({"success": True})


# ── Gallery Vault Endpoints ───────────────────────────────────


@app.route("/api/gallery")
@require_auth
def api_gallery_list():
    """List all artworks in the gallery vault."""
    try:
        from familiar.skills.gallery.provenance import _load_gallery_data

        artworks = _load_gallery_data()
        items = []
        for a in artworks:
            items.append({
                "artwork_id": a["artwork_id"],
                "title": a["title"],
                "artist": a["artist"],
                "description": a.get("description", ""),
                "medium": a.get("medium", ""),
                "dimensions": a.get("dimensions", ""),
                "license": a.get("license", "All Rights Reserved"),
                "tags": a.get("tags", []),
                "watermark_types": a.get("watermark_types", []),
                "block_hash": a.get("block_hash", ""),
                "block_index": a.get("block_index", -1),
                "registered_at": a.get("registered_at", ""),
                "has_thumbnail": bool(a.get("thumbnail_path")),
            })
        return jsonify({"artworks": items, "count": len(items)})
    except Exception as e:
        logger.debug("Gallery list error: %s", e)
        return jsonify({"artworks": [], "count": 0, "error": str(e)}), 502


@app.route("/api/gallery/<artwork_id>")
@require_auth
def api_gallery_detail(artwork_id):
    """Get full artwork details including chain block info."""
    try:
        from familiar.skills.gallery.provenance import (
            _load_gallery_data,
            get_chain_blocks_for_artwork,
        )

        artworks = _load_gallery_data()
        artwork = None
        for a in artworks:
            if a["artwork_id"] == artwork_id:
                artwork = a
                break

        if artwork is None:
            return jsonify({"error": "Artwork not found"}), 404

        blocks = get_chain_blocks_for_artwork(artwork_id)
        artwork["chain_blocks"] = blocks
        return jsonify(artwork)
    except Exception as e:
        logger.debug("Gallery detail error: %s", e)
        return jsonify({"error": str(e)}), 502


@app.route("/api/gallery/upload", methods=["POST"])
@require_auth
def api_gallery_upload():
    """Upload an image to the gallery originals directory."""
    try:
        import uuid

        from familiar.skills.gallery.provenance import _get_gallery_dir
        from familiar.skills.gallery.watermark import generate_thumbnail

        if "file" not in request.files:
            return jsonify({"error": "No file provided"}), 400

        file = request.files["file"]
        if not file.filename:
            return jsonify({"error": "No filename"}), 400

        gallery_dir = _get_gallery_dir()
        temp_id = str(uuid.uuid4())[:12]
        ext = Path(file.filename).suffix or ".png"
        dest = gallery_dir / "originals" / f"{temp_id}{ext}"
        file.save(str(dest))

        # Generate thumbnail
        thumb_path = str(gallery_dir / "thumbnails" / f"{temp_id}.png")
        generate_thumbnail(str(dest), output_path=thumb_path)

        return jsonify({
            "success": True,
            "temp_id": temp_id,
            "filename": file.filename,
            "path": str(dest),
            "thumbnail_path": thumb_path,
        })
    except Exception as e:
        logger.warning("Gallery upload error: %s", e)
        return jsonify({"error": str(e), "success": False}), 500


@app.route("/api/gallery/watermark", methods=["POST"])
@require_auth
def api_gallery_watermark():
    """Apply watermark(s) to an uploaded artwork."""
    try:
        from familiar.skills.gallery.watermark import (
            apply_invisible_watermark,
            apply_visible_watermark,
            compute_image_hash,
        )

        data = request.get_json(force=True)
        image_path = data.get("image_path", "")
        if not image_path or not Path(image_path).exists():
            return jsonify({"error": "Image file not found"}), 400

        wm_type = data.get("type", "visible")
        text = data.get("text", "")
        output_path = data.get("output_path", "")

        if wm_type == "invisible":
            embed_data = data.get("data", text or "watermark")
            result_path = apply_invisible_watermark(
                image_path, embed_data, output_path=output_path
            )
        else:
            if not text:
                return jsonify({"error": "text is required for visible watermarks"}), 400
            result_path = apply_visible_watermark(
                image_path, text,
                position=data.get("position", "bottom-right"),
                opacity=data.get("opacity", 0.3),
                output_path=output_path,
            )

        return jsonify({
            "success": True,
            "output_path": result_path,
            "type": wm_type,
            "hash": compute_image_hash(result_path),
        })
    except Exception as e:
        logger.debug("Gallery watermark error: %s", e)
        return jsonify({"error": str(e), "success": False}), 500


@app.route("/api/gallery/register", methods=["POST"])
@require_auth
def api_gallery_register():
    """Register + chain-notarize an artwork."""
    try:
        from familiar.skills.gallery.provenance import register_artwork

        data = request.get_json(force=True)
        image_path = data.get("image_path", "")
        if not image_path or not Path(image_path).exists():
            return jsonify({"error": "Image file not found"}), 400

        artwork = register_artwork(
            image_path=image_path,
            title=data.get("title", "Untitled"),
            artist=data.get("artist", "Unknown"),
            description=data.get("description", ""),
            medium=data.get("medium", ""),
            dimensions=data.get("dimensions", ""),
            license_type=data.get("license", "All Rights Reserved"),
            tags=data.get("tags", []),
            watermark_types=data.get("watermark_types", ["visible", "invisible", "certificate"]),
        )

        return jsonify({
            "success": True,
            "artwork_id": artwork["artwork_id"],
            "title": artwork["title"],
            "artist": artwork["artist"],
            "block_hash": artwork.get("block_hash", ""),
            "watermark_types": artwork["watermark_types"],
            "certificate_path": artwork.get("certificate_path", ""),
        })
    except Exception as e:
        logger.warning("Gallery register error: %s", e)
        return jsonify({"error": str(e), "success": False}), 500


@app.route("/api/gallery/verify/<artwork_id>")
@require_auth
def api_gallery_verify(artwork_id):
    """Verify chain integrity for an artwork."""
    try:
        from familiar.skills.gallery.provenance import verify_artwork

        result = verify_artwork(artwork_id)
        return jsonify(result)
    except Exception as e:
        logger.debug("Gallery verify error: %s", e)
        return jsonify({"error": str(e), "verified": False}), 502


@app.route("/api/gallery/certificate/<artwork_id>")
@require_auth
def api_gallery_certificate(artwork_id):
    """Download provenance certificate markdown."""
    try:
        from familiar.skills.gallery.provenance import _load_gallery_data

        artworks = _load_gallery_data()
        artwork = None
        for a in artworks:
            if a["artwork_id"] == artwork_id:
                artwork = a
                break

        if artwork is None:
            return jsonify({"error": "Artwork not found"}), 404

        cert_path = artwork.get("certificate_path", "")
        if not cert_path or not Path(cert_path).exists():
            return jsonify({"error": "Certificate not found"}), 404

        return send_file(
            cert_path,
            as_attachment=True,
            download_name=f"certificate_{artwork_id}.md",
            mimetype="text/markdown",
        )
    except Exception as e:
        logger.debug("Gallery certificate error: %s", e)
        return jsonify({"error": str(e)}), 502


@app.route("/api/gallery/image/<artwork_id>")
@require_auth
def api_gallery_image(artwork_id):
    """Serve artwork image file. Use ?type=original or ?type=watermarked."""
    try:
        from familiar.skills.gallery.provenance import _load_gallery_data

        artworks = _load_gallery_data()
        artwork = None
        for a in artworks:
            if a["artwork_id"] == artwork_id:
                artwork = a
                break

        if artwork is None:
            return jsonify({"error": "Artwork not found"}), 404

        img_type = request.args.get("type", "watermarked")
        if img_type == "original":
            img_path = artwork.get("original_path", "")
        else:
            img_path = artwork.get("watermarked_path", "")

        if not img_path or not Path(img_path).exists():
            return jsonify({"error": "Image file not found"}), 404

        ext = Path(img_path).suffix.lower()
        mime_map = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
                    ".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp"}
        mimetype = mime_map.get(ext, "image/png")
        return send_file(img_path, mimetype=mimetype)
    except Exception as e:
        logger.debug("Gallery image error: %s", e)
        return jsonify({"error": str(e)}), 502


@app.route("/api/gallery/thumbnail/<artwork_id>")
@require_auth
def api_gallery_thumbnail(artwork_id):
    """Serve artwork thumbnail."""
    try:
        from familiar.skills.gallery.provenance import _load_gallery_data

        artworks = _load_gallery_data()
        artwork = None
        for a in artworks:
            if a["artwork_id"] == artwork_id:
                artwork = a
                break

        if artwork is None:
            return jsonify({"error": "Artwork not found"}), 404

        thumb_path = artwork.get("thumbnail_path", "")
        if not thumb_path or not Path(thumb_path).exists():
            return jsonify({"error": "Thumbnail not found"}), 404

        return send_file(thumb_path, mimetype="image/png")
    except Exception as e:
        logger.debug("Gallery thumbnail error: %s", e)
        return jsonify({"error": str(e)}), 502


@app.route("/api/upload", methods=["POST"])
@require_auth
def api_upload():
    """Upload a file for chat attachment."""
    import uuid

    ALLOWED_EXTS = {
        ".pdf", ".docx", ".doc", ".xlsx", ".xls", ".csv", ".tsv",
        ".txt", ".md", ".json", ".xml", ".html", ".htm", ".rtf",
        ".png", ".jpg", ".jpeg", ".tiff", ".tif", ".bmp", ".gif", ".webp",
    }
    IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".tiff", ".tif", ".bmp", ".gif", ".webp"}
    MAX_SIZE = 25 * 1024 * 1024  # 25 MB

    if "file" not in request.files:
        return jsonify({"error": "No file provided", "success": False}), 400

    file = request.files["file"]
    if not file.filename:
        return jsonify({"error": "No filename", "success": False}), 400

    ext = Path(file.filename).suffix.lower()
    if ext not in ALLOWED_EXTS:
        return jsonify({"error": f"File type {ext} not supported", "success": False}), 400

    uploads_dir = DATA_DIR / "uploads"
    uploads_dir.mkdir(parents=True, exist_ok=True)

    file_id = str(uuid.uuid4())[:12]
    safe_name = re.sub(r"[^\w\-.]", "_", Path(file.filename).stem)[:50]
    dest = uploads_dir / f"{file_id}_{safe_name}{ext}"
    file.save(str(dest))

    size = dest.stat().st_size
    if size > MAX_SIZE:
        dest.unlink()
        return jsonify({"error": "File exceeds 25 MB limit", "success": False}), 400

    return jsonify({
        "success": True,
        "file_id": file_id,
        "filename": file.filename,
        "path": str(dest),
        "size": size,
        "is_image": ext in IMAGE_EXTS,
    })


@app.route("/api/upload/preview/<file_id>")
@require_auth
def api_upload_preview(file_id):
    """Serve an uploaded image file for thumbnail preview."""
    import glob as _glob
    import mimetypes

    # Sanitize file_id to prevent path traversal
    if not re.match(r"^[a-f0-9\-]{8,36}$", file_id):
        return jsonify({"error": "Invalid file ID"}), 400

    uploads_dir = DATA_DIR / "uploads"
    matches = _glob.glob(str(uploads_dir / f"{file_id}_*"))
    if not matches:
        return jsonify({"error": "File not found"}), 404

    fpath = Path(matches[0])
    mime = mimetypes.guess_type(str(fpath))[0] or "application/octet-stream"
    if not mime.startswith("image/"):
        return jsonify({"error": "Not an image"}), 400

    return send_file(str(fpath), mimetype=mime)


@app.route("/api/history")
@require_auth
def api_history():
    """Get conversation history."""
    agent = get_agent()
    history = agent.history.get_history("dashboard", "web", limit=50)
    return jsonify({"history": history})


@app.route("/api/history", methods=["DELETE"])
@require_auth
def api_history_clear():
    """Clear conversation history."""
    agent = get_agent()
    agent.clear_history("dashboard", "web")
    return jsonify({"success": True})


# ============================================================
# Onboarding Routes (no auth — first-run, localhost only)
# ============================================================

# Session-scoped engine instance for the onboarding flow
_onboard_engine = None
_onboard_lock = threading.Lock()
_activation_lock = threading.Lock()


def _get_onboard_engine():
    """Get or create the onboarding engine for this session."""
    global _onboard_engine
    with _onboard_lock:
        if _onboard_engine is None:
            from familiar.onboard_engine import OnboardingEngine

            _onboard_engine = OnboardingEngine()
    return _onboard_engine


def _require_localhost(f):
    """Only allow onboard routes from localhost.

    Set the ``FAMILIAR_ONBOARD_LAN`` environment variable to ``1`` or
    ``true`` to allow non-localhost IPs to access ``/onboard`` routes,
    enabling setup from another device on the same network.
    """

    @wraps(f)
    def decorated(*args, **kwargs):
        # First-run bypass — allow onboarding from anywhere during setup
        if _first_run_mode:
            return f(*args, **kwargs)

        lan_allowed = os.environ.get("FAMILIAR_ONBOARD_LAN", "").lower() in ("1", "true")
        if not lan_allowed:
            remote = request.remote_addr
            if remote not in ("127.0.0.1", "::1", "localhost"):
                abort(403, description="Onboarding is only available from localhost")
        return f(*args, **kwargs)

    return decorated


@app.route("/api/tutorial/complete", methods=["POST"])
@require_auth
def api_tutorial_complete():
    """Mark the dashboard tutorial as completed (server-side flag)."""
    flag = DATA_DIR / ".dashboard_tutorial_done"
    flag.parent.mkdir(parents=True, exist_ok=True)
    flag.write_text("1")
    return jsonify({"ok": True})


@app.route("/api/tutorial/reset", methods=["POST"])
@require_auth
def api_tutorial_reset():
    """Reset the dashboard tutorial so it runs again on next load."""
    flag = DATA_DIR / ".dashboard_tutorial_done"
    try:
        flag.unlink(missing_ok=True)
    except OSError as e:
        logger.debug("I/O error suppressed: %s", e)
    return jsonify({"ok": True, "message": "Tutorial reset. Reload the dashboard to start it."})


@app.route("/onboard")
@_require_localhost
def onboard_page():
    """Serve the onboarding wizard UI."""
    return render_template("onboard.html")


@app.route("/api/onboard/detect")
@_require_localhost
def api_onboard_detect():
    """Detect available LLM providers."""
    engine = _get_onboard_engine()
    result = engine.detect_providers()
    return jsonify(result.data)


@app.route("/api/onboard/creature-data")
@_require_localhost
def api_onboard_creature_data():
    """Return species, palette, and constitution data for the onboarding wizard."""
    from familiar.onboard_engine import OnboardingEngine

    return jsonify(OnboardingEngine.get_creature_data())


@app.route("/api/onboard/creature-preview", methods=["POST"])
@_require_localhost
def api_onboard_creature_preview():
    """Render a creature sprite preview given species and colors."""
    data = request.json or {}
    species = data.get("species", "fox")
    color_body = data.get("color_body", "#4A90D9")
    color_eyes = data.get("color_eyes", "#FFD700")
    color_accent = data.get("color_accent", "#FF6B6B")

    from familiar.creature.palette import build_color_map
    from familiar.creature.renderer_html import render_frame_html
    from familiar.creature.sprites import EGG_FRAMES, get_sprite_frames

    if species == "egg":
        frames = EGG_FRAMES
        colors = build_color_map(color_body, color_eyes, color_accent)
    else:
        colors = build_color_map(color_body, color_eyes, color_accent)
        frames = get_sprite_frames(species, "baby", "idle")

    sprite_html = render_frame_html(frames[0], colors)
    return jsonify({"sprite_html": sprite_html})


@app.route("/api/onboard/validate/<step>", methods=["POST"])
@_require_localhost
def api_onboard_validate(step):
    """Validate a single onboarding step."""
    engine = _get_onboard_engine()
    data = request.json or {}

    validators = {
        "provider": engine.validate_provider,
        "channels": engine.validate_channels,
        "owner_pin": engine.validate_owner_pin,
        "identity": engine.validate_identity,
        "profile": engine.validate_profile,
        "credentials": engine.validate_credentials,
        "briefing": engine.validate_briefing,
        "encryption": engine.validate_encryption,
        "creature": engine.validate_creature,
    }

    validator = validators.get(step)
    if not validator:
        return jsonify({"success": False, "errors": [f"Unknown step: {step}"]}), 400

    result = validator(data)
    return jsonify(
        {
            "success": result.success,
            "errors": result.errors,
            "warnings": result.warnings,
            "data": result.data,
        }
    )


@app.route("/api/onboard/ollama-bundles")
@_require_localhost
def api_onboard_ollama_bundles():
    """Get RAM-based Ollama model bundle recommendations."""
    engine = _get_onboard_engine()
    bundles = engine.get_ollama_bundles()
    return jsonify(bundles)


@app.route("/api/onboard/test-message", methods=["POST"])
@_require_localhost
def api_onboard_test_message():
    """Send a test message after activation."""
    engine = _get_onboard_engine()
    result = engine.send_test_message()
    return jsonify(
        {
            "success": result.success,
            "warnings": result.warnings,
            "errors": result.errors,
        }
    )


@app.route("/api/onboard/checklist")
@_require_localhost
def api_onboard_checklist():
    """Get post-activation next-steps checklist."""
    engine = _get_onboard_engine()
    return jsonify(engine.get_post_activation_checklist())


@app.route("/api/onboard/save-state", methods=["POST"])
@_require_localhost
def api_onboard_save_state():
    """Persist wizard state server-side for cross-browser resume."""
    engine = _get_onboard_engine()
    data = request.json or {}
    step = data.get("step", 0)
    # Save JS UI state alongside engine state (don't overwrite engine._state)
    engine.save_state(step, ui_state=data.get("state"))
    return jsonify({"success": True})


@app.route("/api/onboard/load-state")
@_require_localhost
def api_onboard_load_state():
    """Load saved wizard state from server."""
    engine = _get_onboard_engine()
    saved = engine.load_saved_state()
    if saved:
        return jsonify(
            {"found": True, "step": saved.get("step", 0),
             "state": saved.get("ui_state", saved.get("state", {}))}
        )
    return jsonify({"found": False})


@app.route("/api/onboard/credentials.pdf")
@_require_localhost
def api_onboard_credentials_pdf():
    """Generate a PDF with dashboard URL and API key."""
    import io

    api_key = request.args.get("key", "")
    port = request.host.split(":")[-1] if ":" in request.host else "5000"
    dash_url = f"http://127.0.0.1:{port}?api_key={api_key}"

    # Minimal valid PDF with credentials
    lines = [
        "Familiar Dashboard Credentials",
        "",
        f"Dashboard URL:",
        dash_url,
        "",
        f"API Key:",
        api_key,
        "",
        "Keep this file safe. You can also find your key in:",
        "~/.familiar/.env (FAMILIAR_DASHBOARD_KEY)",
        "",
        "To start Familiar, run:",
        "  source ~/familiar-env/bin/activate",
        "  familiar",
    ]
    text = "\n".join(lines)

    # Build raw PDF (no dependencies needed)
    stream_content = f"BT\n/F1 18 Tf\n50 740 Td\n(Familiar Dashboard Credentials) Tj\n"
    stream_content += "/F1 11 Tf\n0 -36 Td\n"
    y_items = [
        ("Dashboard URL:", ""),
        (dash_url, ""),
        ("", ""),
        ("API Key:", ""),
        (api_key, ""),
        ("", ""),
        ("Keep this file safe.", ""),
        ("Your key is also saved in: ~/.familiar/.env", ""),
        ("", ""),
        ("To start Familiar, run:", ""),
        ("  source ~/familiar-env/bin/activate && familiar", ""),
    ]
    for line_text, _ in y_items:
        escaped = (line_text.replace("\\", "\\\\")
                   .replace("(", "\\(").replace(")", "\\)"))
        stream_content += f"0 -16 Td\n({escaped}) Tj\n"
    stream_content += "ET"

    obj1 = "1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
    obj2 = "2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"
    obj3 = ("3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
            "/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>\nendobj\n")
    obj4 = f"4 0 obj\n<< /Length {len(stream_content)} >>\nstream\n{stream_content}\nendstream\nendobj\n"
    obj5 = "5 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Courier >>\nendobj\n"

    body = "%PDF-1.4\n"
    offsets = []
    for obj in [obj1, obj2, obj3, obj4, obj5]:
        offsets.append(len(body))
        body += obj

    xref_offset = len(body)
    body += "xref\n"
    body += f"0 {len(offsets) + 1}\n"
    body += "0000000000 65535 f \n"
    for off in offsets:
        body += f"{off:010d} 00000 n \n"
    body += "trailer\n"
    body += f"<< /Size {len(offsets) + 1} /Root 1 0 R >>\n"
    body += "startxref\n"
    body += f"{xref_offset}\n"
    body += "%%EOF\n"

    buf = io.BytesIO(body.encode("latin-1"))
    return send_file(
        buf,
        mimetype="application/pdf",
        as_attachment=True,
        download_name="familiar-credentials.pdf",
    )


@app.route("/api/onboard/clear-state", methods=["POST"])
@_require_localhost
def api_onboard_clear_state():
    """Clear saved wizard state on the server."""
    engine = _get_onboard_engine()
    engine.clear_saved_state()
    return jsonify({"success": True})


@app.route("/api/onboard/activate", methods=["POST"])
@_require_localhost
def api_onboard_activate():
    """Activate the configuration (write config, install deps)."""
    global _first_run_mode
    engine = _get_onboard_engine()
    result = engine.activate()
    # Reload .env so the running process picks up new keys
    env_path = AGENT_DIR / ".env"
    if env_path.exists():
        try:
            from dotenv import load_dotenv

            load_dotenv(str(env_path), override=True)
        except ImportError:
            pass
    if result.success:
        _first_run_mode = False
        # Set up auth key for the now-active dashboard
        key = os.environ.get("FAMILIAR_DASHBOARD_KEY")
        if not key:
            import secrets as _secrets

            key = _secrets.token_urlsafe(32)
            try:
                with open(env_path, "a") as f:
                    f.write(f"\nFAMILIAR_DASHBOARD_KEY={key}\n")
            except OSError:
                pass
        os.environ["FAMILIAR_DASHBOARD_KEY"] = key
        require_auth._auto_key = key
    return jsonify(
        {
            "success": result.success,
            "errors": result.errors,
            "warnings": result.warnings,
            "api_key": os.environ.get("FAMILIAR_DASHBOARD_KEY", ""),
        }
    )


@app.route("/api/onboard/activate/stream")
@_require_localhost
def api_onboard_activate_stream():
    """SSE stream of activation progress events."""
    if not _activation_lock.acquire(blocking=False):
        return jsonify({"error": "Activation already in progress"}), 409

    engine = _get_onboard_engine()
    msg_queue = queue.Queue()

    def _notify(msg):
        msg_queue.put(msg)

    def _run_activation():
        global _first_run_mode
        try:
            result = engine.activate(notify=_notify)
            # Reload .env so the running process picks up new keys
            env_path = AGENT_DIR / ".env"
            if env_path.exists():
                try:
                    from dotenv import load_dotenv

                    load_dotenv(str(env_path), override=True)
                except ImportError:
                    pass
            if result.success:
                _first_run_mode = False
                # Set up auth key for the now-active dashboard
                key = os.environ.get("FAMILIAR_DASHBOARD_KEY")
                if not key:
                    import secrets as _secrets

                    key = _secrets.token_urlsafe(32)
                    try:
                        with open(env_path, "a") as f:
                            f.write(f"\nFAMILIAR_DASHBOARD_KEY={key}\n")
                    except OSError:
                        pass
                os.environ["FAMILIAR_DASHBOARD_KEY"] = key
                require_auth._auto_key = key
            msg_queue.put(("__DONE__", result.success, result.errors, result.data))
        finally:
            _activation_lock.release()

    # Run activation in background thread
    t = threading.Thread(target=_run_activation, daemon=True)
    t.start()

    def generate():
        progress = 10
        step_increment = 12  # ~8 steps, 12% each gets to ~100%
        while True:
            try:
                item = msg_queue.get(timeout=60)
            except queue.Empty:
                yield f"data: {json.dumps({'message': 'Waiting...', 'progress': progress})}\n\n"
                continue

            if isinstance(item, tuple) and item[0] == "__DONE__":
                _, success, errors, data = item
                payload = {
                    "done": True,
                    "success": success,
                    "progress": 100,
                    "message": "Complete!" if success else "; ".join(errors),
                    "type": "ok" if success else "error",
                }
                if data and data.get("dashboard_key"):
                    payload["dashboard_key"] = data["dashboard_key"]
                yield f"data: {json.dumps(payload)}\n\n"
                break
            else:
                progress = min(progress + step_increment, 95)
                payload = {"message": str(item), "progress": progress, "type": "ok"}
                yield f"data: {json.dumps(payload)}\n\n"

    resp = Response(generate(), mimetype="text/event-stream")
    resp.headers["Cache-Control"] = "no-cache"
    resp.headers["X-Accel-Buffering"] = "no"
    return resp


# ============================================================
# Model Library, Tarot Spread & Downloads
# ============================================================


@app.route("/api/models/ollama")
@require_auth
def api_models_ollama():
    """Return Ollama model catalog with install/active status."""
    import socket

    from familiar.channels.connect_wizard._llm import OLLAMA_MODEL_CATALOG

    # Check Ollama is running
    ollama_running = False
    try:
        with socket.create_connection(("127.0.0.1", 11434), timeout=2.0):
            ollama_running = True
    except (OSError, TimeoutError) as e:
        logger.debug("I/O error suppressed: %s", e)
    installed_names: set[str] = set()
    if ollama_running:
        try:
            import httpx

            resp = httpx.get("http://localhost:11434/api/tags", timeout=5)
            data = resp.json()
            for m in data.get("models", []):
                name = m.get("name", "")
                installed_names.add(name)
                if ":" not in name:
                    installed_names.add(f"{name}:latest")
        except Exception:
            logger.debug("Failed to query Ollama /api/tags", exc_info=True)

    # Detect active model
    active_model = ""
    try:
        agent = get_agent()
        llm_cfg = getattr(getattr(agent, "config", None), "llm", None)
        active_model = getattr(llm_cfg, "ollama_model", "") if llm_cfg else ""
    except Exception as e:
        logger.debug("Error suppressed: %s", e)
    models = []
    catalog_tags: set[str] = set()
    for tag, display_name, size, description, _url in OLLAMA_MODEL_CATALOG:
        installed = tag in installed_names or f"{tag}:latest" in installed_names
        # Active match: exact or base name match (e.g. qwen2.5:7b matches qwen2.5:14b)
        is_active = (
            tag == active_model
            or f"{tag}:latest" == active_model
            or (active_model and tag.split(":")[0] == active_model.split(":")[0]
                and active_model in installed_names)
        )
        models.append(
            {
                "tag": tag,
                "display_name": display_name,
                "size": size,
                "description": description,
                "installed": installed,
                "active": is_active and tag == active_model,
            }
        )
        catalog_tags.add(tag)
        catalog_tags.add(f"{tag}:latest")

    # Add installed models not in the catalog
    for name in sorted(installed_names):
        if name not in catalog_tags:
            base = name.split(":")[0]
            suffix = name.split(":")[-1] if ":" in name else ""
            display = f"{base} ({suffix})" if suffix and suffix != "latest" else base
            models.append(
                {
                    "tag": name,
                    "display_name": display.title(),
                    "size": "",
                    "description": "Locally installed model",
                    "installed": True,
                    "active": name == active_model,
                }
            )

    return jsonify(
        {
            "ollama_running": ollama_running,
            "models": models,
            "active_model": active_model,
        }
    )


@app.route("/api/models/ollama/loaded")
@require_auth
def api_models_ollama_loaded():
    """Return currently loaded Ollama models with memory usage."""
    try:
        import httpx

        resp = httpx.get("http://localhost:11434/api/ps", timeout=5)
        ps_data = resp.json()
    except Exception:
        return jsonify({"loaded": [], "total_ram_bytes": 0})

    loaded = []
    total_ram = 0
    for m in ps_data.get("models", []):
        size = m.get("size", 0)
        vram = m.get("size_vram", 0)
        ram = size - vram  # what's in system RAM vs GPU VRAM
        total_ram += size
        loaded.append({
            "name": m.get("name", ""),
            "size_bytes": size,
            "size_mb": round(size / 1024 / 1024),
            "vram_bytes": vram,
            "ram_bytes": ram,
            "context_length": m.get("context_length", 0),
            "expires_at": m.get("expires_at", ""),
            "family": m.get("details", {}).get("family", ""),
            "parameter_size": m.get("details", {}).get("parameter_size", ""),
            "quantization": m.get("details", {}).get("quantization_level", ""),
        })

    return jsonify({
        "loaded": loaded,
        "total_ram_bytes": total_ram,
        "total_ram_mb": round(total_ram / 1024 / 1024),
        "model_count": len(loaded),
    })


@app.route("/api/models/ollama/pull", methods=["POST"])
@require_auth
def api_models_ollama_pull():
    """Stream an Ollama model download via SSE."""
    from familiar.channels.connect_wizard._llm import OLLAMA_MODEL_CATALOG

    data = request.json or {}
    model = data.get("model", "").strip()
    if not model:
        return jsonify({"error": "No model specified"}), 400

    valid_tags = {tag for tag, *_ in OLLAMA_MODEL_CATALOG}
    if model not in valid_tags:
        return jsonify({"error": f"Unknown model: {model}"}), 400

    msg_queue = queue.Queue()

    def _pull():
        try:
            proc = subprocess.Popen(
                ["ollama", "pull", model],
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                text=True,
                bufsize=1,
            )
            for line in proc.stdout:
                msg_queue.put({"message": line.strip(), "done": False, "success": False})
            proc.wait()
            msg_queue.put(
                {
                    "message": "Complete" if proc.returncode == 0 else "Failed",
                    "done": True,
                    "success": proc.returncode == 0,
                }
            )
        except FileNotFoundError:
            msg_queue.put({"message": "ollama not found in PATH", "done": True, "success": False})
        except Exception as e:
            msg_queue.put({"message": str(e), "done": True, "success": False})
        finally:
            msg_queue.put(None)

    t = threading.Thread(target=_pull, daemon=True)
    t.start()

    def generate():
        while True:
            try:
                item = msg_queue.get(timeout=600)
            except queue.Empty:
                yield f"data: {json.dumps({'message': 'Timeout', 'done': True, 'success': False})}\n\n"
                break
            if item is None:
                break
            yield f"data: {json.dumps(item)}\n\n"

    resp = Response(generate(), mimetype="text/event-stream")
    resp.headers["Cache-Control"] = "no-cache"
    resp.headers["X-Accel-Buffering"] = "no"
    return resp


@app.route("/api/models/ollama/activate", methods=["POST"])
@require_auth
def api_models_ollama_activate():
    """Switch the active Ollama model."""
    data = request.json or {}
    model = data.get("model", "").strip()
    if not model:
        return jsonify({"error": "No model specified"}), 400

    try:
        agent = get_agent()
        # Update the config BEFORE switching so the provider picks up the new model
        agent.config.llm.ollama_model = model
        agent.switch_provider("ollama")

        # Persist to config file so it survives restarts
        try:
            config_path = CONFIG_FILE
            if config_path.exists():
                import yaml

                cfg_data = yaml.safe_load(config_path.read_text()) or {}
                cfg_data.setdefault("llm", {})["ollama_model"] = model
                config_path.write_text(yaml.dump(cfg_data, default_flow_style=False))
        except Exception as e:
            logger.debug("Failed to persist ollama_model to config: %s", e)

        return jsonify({"success": True, "active_model": model})
    except Exception as e:
        logger.exception("Failed to activate Ollama model")
        return jsonify({"error": str(e)}), 500


@app.route("/api/models/ollama/remove", methods=["POST"])
@require_auth
def api_models_ollama_remove():
    """Delete an installed Ollama model."""
    data = request.json or {}
    model = data.get("model", "").strip()
    if not model:
        return jsonify({"error": "No model specified"}), 400

    try:
        result = subprocess.run(
            ["ollama", "rm", model],
            capture_output=True,
            text=True,
            timeout=30,
        )
        if result.returncode == 0:
            return jsonify({"success": True})
        return jsonify({"error": result.stderr.strip() or "Failed to remove model"}), 500
    except FileNotFoundError:
        return jsonify({"error": "ollama not found in PATH"}), 500
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/tarot/spread")
@require_auth
def api_tarot_spread():
    """Return the tarot spread with live service status."""
    from familiar.channels.connect_wizard._helpers import SERVICE_DISPLAY
    from familiar.channels.connect_wizard._tarot import (
        ARCANA_ORDER,
        TAROT_ARCANA,
        _card_reveal,
    )
    from familiar.creature.state import load_creature

    # Get live service status
    host = get_wizard_host()
    service_status = host._get_service_status()

    # Load creature species
    species = "fox"
    try:
        creature = load_creature()
        if creature:
            species = creature.species
    except Exception as e:
        logger.debug("Error suppressed: %s", e)
    cards = []
    for key in ARCANA_ORDER:
        arcana = TAROT_ARCANA[key]
        services = arcana["services"]
        configured = sum(1 for s in services if service_status.get(s, False))
        total = len(services)
        pending = total - configured

        svc_list = []
        for svc in services:
            svc_list.append(
                {
                    "name": svc,
                    "display": SERVICE_DISPLAY.get(svc, svc),
                    "connected": service_status.get(svc, False),
                }
            )

        cards.append(
            {
                "key": key,
                "name": arcana["name"],
                "icon": arcana["icon"],
                "numeral": arcana["numeral"],
                "configured": configured,
                "total": total,
                "pending": pending,
                "description": arcana["description"],
                "services": svc_list,
                "species_flavor": _card_reveal(species, key),
            }
        )

    return jsonify({"cards": cards, "species": species})


@app.route("/api/downloads/tome")
@require_auth
def api_downloads_tome():
    """Download the user guide as PDF (or HTML fallback)."""
    try:
        from familiar.dashboard.guide import generate_guide_pdf

        pdf_path = generate_guide_pdf()
        return send_file(
            str(pdf_path),
            as_attachment=True,
            download_name="familiar_user_guide.pdf",
            mimetype="application/pdf",
        )
    except ImportError:
        # WeasyPrint not installed — serve HTML fallback
        from familiar.dashboard.guide import get_guide_html

        return Response(get_guide_html(), mimetype="text/html")
    except Exception:
        logger.exception("Failed to generate guide PDF")
        # Fallback to HTML
        try:
            from familiar.dashboard.guide import get_guide_html

            return Response(get_guide_html(), mimetype="text/html")
        except Exception:
            return jsonify({"error": "Guide generation failed"}), 500


@app.route("/api/downloads/bundle")
@require_auth
def api_downloads_bundle():
    """Download a config-only backup archive."""
    import shutil
    import tempfile

    try:
        from familiar.core.backup import BackupManager, BackupType

        manager = BackupManager()
        manifest = manager.create_backup(backup_type=BackupType.CONFIG_ONLY)

        # The backup file is in the manager's backup dir
        backup_path = manager.config.path / f"{manifest.backup_id}.tar.gz"
        if not backup_path.exists():
            return jsonify({"error": "Backup file not found"}), 500

        # Copy to temp file for serving (so we don't hold a lock)
        fd, tmp_path = tempfile.mkstemp(suffix=".tar.gz")
        os.close(fd)
        shutil.copy2(str(backup_path), tmp_path)

        return send_file(
            tmp_path,
            as_attachment=True,
            download_name=f"familiar_config_{manifest.backup_id}.tar.gz",
            mimetype="application/gzip",
        )
    except Exception:
        logger.exception("Failed to create config backup")
        return jsonify({"error": "Backup creation failed"}), 500


# ============================================================
# Marketplace
# ============================================================


@app.route("/api/marketplace/catalog/browse")
@require_auth
def api_marketplace_catalog_browse():
    """Browse/search the unified marketplace catalog."""
    catalog = _get_marketplace_catalog()
    entity_type = request.args.get("entity_type")
    search = request.args.get("search", "").strip()
    limit = int(request.args.get("limit", 50))
    offset = int(request.args.get("offset", 0))

    entries = catalog.browse(
        entity_type=entity_type,
        limit=limit,
        offset=offset,
    )
    # Apply search filter on name/description/tags
    if search:
        lower = search.lower()
        entries = [
            e for e in entries
            if lower in e.name.lower()
            or lower in (e.description or "").lower()
            or any(lower in t.lower() for t in (e.tags or []))
        ]
    total = len(catalog.browse(entity_type=entity_type, limit=9999))
    results = []
    for e in entries:
        results.append({
            "id": e.id,
            "entity_type": e.entity_type,
            "name": e.name,
            "description": e.description,
            "version": e.version,
            "author": e.author,
            "source": e.source,
            "tags": e.tags or [],
            "entity_data": e.entity_data or {},
        })
    return jsonify({"entries": results, "total": total})


@app.route("/api/marketplace/catalog/stats")
@require_auth
def api_marketplace_catalog_stats():
    """Return counts per entity type."""
    catalog = _get_marketplace_catalog()
    return jsonify({"stats": catalog.stats()})


@app.route("/api/marketplace/catalog/entry/<entity_type>/<entry_id>")
@require_auth
def api_marketplace_catalog_entry(entity_type, entry_id):
    """Get a single catalog entry with full detail."""
    catalog = _get_marketplace_catalog()
    entry = catalog.get_entry(entity_type, entry_id)
    if not entry:
        return jsonify({"error": "Entry not found"}), 404
    return jsonify({
        "id": entry.id,
        "entity_type": entry.entity_type,
        "name": entry.name,
        "description": entry.description,
        "version": entry.version,
        "author": entry.author,
        "source": entry.source,
        "tags": entry.tags or [],
        "entity_data": entry.entity_data or {},
    })


@app.route("/api/marketplace/catalog/install", methods=["POST"])
@require_auth
def api_marketplace_catalog_install():
    """Validate and install a JSON package."""
    data = request.json or {}
    pkg_type = data.get("type", "")

    try:
        if pkg_type in ("job_class", "bundle", "familiar"):
            from familiar.jobs._marketplace import install_package

            result = install_package(data)
        elif pkg_type == "skill":
            from familiar.skills._marketplace import install_skill_package

            result = install_skill_package(data)
        elif pkg_type == "species":
            catalog = _get_marketplace_catalog()
            from familiar.marketplace.catalog import CatalogEntry

            entry = CatalogEntry(
                id=data.get("metadata", {}).get("id", "custom"),
                entity_type="species",
                name=data.get("metadata", {}).get("name", "Unknown"),
                description=data.get("metadata", {}).get("description", ""),
                version=data.get("metadata", {}).get("version", "1.0.0"),
                author=data.get("metadata", {}).get("author", ""),
                source="custom",
                tags=data.get("metadata", {}).get("tags", []),
                entity_data=data,
            )
            catalog.add(entry)
            result = f"Added species '{entry.name}' to catalog"
        else:
            return jsonify({"error": f"Unknown package type: {pkg_type}"}), 400

        if result and ("failed" in result.lower() or "error" in result.lower()):
            return jsonify({"error": result}), 400
        return jsonify({"result": result})
    except Exception as e:
        logger.warning("Marketplace install error: %s", e)
        return jsonify({"error": str(e)}), 500


@app.route("/api/marketplace/plugins")
@require_auth
def api_marketplace_plugins():
    """List available plugins with optional category/search filter."""
    from familiar.skills.marketplace.skill import get_marketplace

    mp = get_marketplace()
    category = request.args.get("category")
    search = request.args.get("search", "").strip()
    plugins = mp.list_plugins(category=category, search=search or None)

    # Collect unique categories
    all_plugins = mp.list_plugins()
    categories = sorted({p.get("category", "") for p in all_plugins if p.get("category")})

    return jsonify({"plugins": plugins, "categories": categories})


@app.route("/api/marketplace/plugins/<plugin_id>/install", methods=["POST"])
@require_auth
def api_marketplace_plugin_install(plugin_id):
    """Install a plugin by ID."""
    from familiar.skills.marketplace.skill import get_marketplace

    mp = get_marketplace()
    result = mp.install(plugin_id)
    return jsonify({"result": result})


@app.route("/api/marketplace/plugins/<plugin_id>/uninstall", methods=["POST"])
@require_auth
def api_marketplace_plugin_uninstall(plugin_id):
    """Uninstall a plugin by ID."""
    from familiar.skills.marketplace.skill import get_marketplace

    mp = get_marketplace()
    result = mp.uninstall(plugin_id)
    return jsonify({"result": result})


@app.route("/api/marketplace/plugins/<plugin_id>/update", methods=["POST"])
@require_auth
def api_marketplace_plugin_update(plugin_id):
    """Update a plugin by ID."""
    from familiar.skills.marketplace.skill import get_marketplace

    mp = get_marketplace()
    result = mp.update(plugin_id)
    return jsonify({"result": result})


# ============================================================
# Server
# ============================================================


@app.route("/api/server/runtime")
@require_auth
def api_server_runtime():
    """Get detected container runtime info.

    Query params:
        refresh: If "true", invalidates the cached runtime and re-detects.
                 Use after installing a new runtime (e.g. Podman).
    """
    mgr = _get_service_manager()
    if request.args.get("refresh", "").lower() == "true":
        mgr.invalidate_runtime()
    return jsonify(mgr.get_runtime_info())


@app.route("/api/server/security")
@require_auth
def api_server_security():
    """Get server security posture summary."""
    mgr = _get_service_manager()
    return jsonify(mgr.get_security_posture())


@app.route("/api/apps")
@require_auth
def api_apps():
    """Get running services with web UI URLs for the Apps panel."""
    mgr = _get_service_manager()
    apps = []
    for s in mgr.get_all_statuses():
        if s.get("status") != "running":
            continue
        if not s.get("web_ui", True):
            continue
        url = s.get("url", "")
        if not url:
            port = s.get("health_check_port", 0)
            if port:
                url = f"http://localhost:{port}"
        if not url:
            continue
        apps.append({
            "key": s.get("key", ""),
            "display_name": s.get("display_name", s.get("key", "")),
            "icon": s.get("icon", "fa-cube"),
            "url": url,
            "description": s.get("description", ""),
            "doc_url": s.get("doc_url", ""),
        })
    return jsonify({"apps": apps})


@app.route("/api/server/status")
@require_auth
def api_server_status():
    """Get status of all managed services."""
    mgr = _get_service_manager()
    return jsonify(mgr.get_all_statuses())


@app.route("/api/server/provision/<service>", methods=["POST"])
@require_auth
def api_server_provision(service):
    """Provision (pull + create) a service."""
    try:
        mgr = _get_service_manager()
        result = mgr.provision(service)
        # Include setup notes if available
        try:
            from familiar.services.specs import SERVICE_SPECS
            spec = SERVICE_SPECS.get(service)
            if spec and spec.setup_notes:
                result["setup_notes"] = spec.setup_notes
        except Exception:
            pass
        return jsonify(result)
    except Exception as exc:
        return jsonify({"success": False, "error": str(exc)}), 500


@app.route("/api/server/start/<service>", methods=["POST"])
@require_auth
def api_server_start(service):
    """Start a provisioned service."""
    try:
        mgr = _get_service_manager()
        result = mgr.start(service)
        return jsonify(result)
    except Exception as exc:
        return jsonify({"success": False, "error": str(exc)}), 500


@app.route("/api/server/stop/<service>", methods=["POST"])
@require_auth
def api_server_stop(service):
    """Stop a running service."""
    mgr = _get_service_manager()
    result = mgr.stop(service)
    return jsonify(result)


@app.route("/api/server/restart/<service>", methods=["POST"])
@require_auth
def api_server_restart(service):
    """Restart a service."""
    mgr = _get_service_manager()
    result = mgr.restart(service)
    return jsonify(result)


@app.route("/api/server/toggle/<service>", methods=["POST"])
@require_auth
def api_server_toggle(service):
    """Toggle a service: stop if running, start if stopped."""
    mgr = _get_service_manager()
    status = mgr.get_status(service)
    if status.get("status") == "running":
        result = mgr.stop(service)
        result["action"] = "stopped"
    else:
        result = mgr.start(service)
        result["action"] = "started"
    return jsonify(result)


@app.route("/api/server/remove/<service>", methods=["POST"])
@require_auth
def api_server_remove(service):
    """Remove a service container, optionally with data."""
    mgr = _get_service_manager()
    remove_data = False
    if request.is_json:
        remove_data = request.json.get("remove_data", False)
    result = mgr.remove(service, remove_data=remove_data)
    return jsonify(result)


@app.route("/api/server/logs/<service>")
@require_auth
def api_server_logs(service):
    """Get recent logs for a service."""
    mgr = _get_service_manager()
    tail = request.args.get("tail", 100, type=int)
    result = mgr.get_logs(service, tail=tail)
    return jsonify(result)


@app.route("/api/server/credentials")
@require_auth
def api_server_credentials():
    """Return all service credentials aggregated from three sources.

    1. Auto-generated secrets (secrets.json)
    2. Default upstream credentials (baked into container images)
    3. Service URLs for context

    Localhost-only by design — never expose credentials over the network.
    """
    import json as _json

    from familiar.services.specs import SERVICE_SPECS

    mgr = _get_service_manager()
    statuses = mgr.get_all_statuses()
    running_keys = {s["key"] for s in statuses if s.get("status") == "running"}

    # Load auto-generated secrets (decrypted via credentials module)
    try:
        from familiar.services.credentials import _load_all as _load_all_creds
        generated: dict = _load_all_creds()
    except Exception:
        generated = {}

    services = []
    for key in sorted(running_keys):
        spec = SERVICE_SPECS.get(key)
        if not spec:
            continue

        entry: dict = {
            "key": key,
            "display_name": spec.display_name,
            "icon": spec.icon,
            "url": "",
        }

        # Find URL from status
        for s in statuses:
            if s.get("key") == key:
                entry["url"] = s.get("url", "")
                break

        # Default upstream credentials
        if spec.default_credentials:
            entry["default_credentials"] = spec.default_credentials

        # Auto-generated credentials
        if key in generated and generated[key]:
            entry["generated_credentials"] = {
                k: v for k, v in generated[key].items()
            }

        # Setup notes
        if spec.setup_notes:
            entry["setup_notes"] = spec.setup_notes

        # Only include services that have some credential info
        if any(k in entry for k in ("default_credentials", "generated_credentials", "setup_notes")):
            services.append(entry)

    return jsonify({"services": services})


# ============================================================
# Phase 8: Peer Messaging API
# ============================================================


@app.route("/api/messages/contacts")
@require_auth
def api_messages_contacts():
    """Get conversations with online status and unread counts."""
    bridge = _get_message_bridge()
    if not bridge:
        return jsonify({"contacts": [], "error": "Messaging not available"})
    return jsonify({"contacts": bridge.get_contacts()})


@app.route("/api/messages/unread")
@require_auth
def api_messages_unread():
    """Get total unread message count (for nav badge)."""
    bridge = _get_message_bridge()
    if not bridge:
        return jsonify({"unread": 0})
    return jsonify({"unread": bridge.get_unread_count()})


@app.route("/api/messages/poll")
@require_auth
def api_messages_poll():
    """Combined poll: contacts + unread + typing + peers + mesh_state."""
    bridge = _get_message_bridge()
    if not bridge:
        # Build mesh_state even without bridge
        agent = get_agent()
        gw = getattr(agent, "_gateway", None) if agent else None
        active = gw is not None
        peer_count = len(gw.peer_gateways) if gw and hasattr(gw, "peer_gateways") else 0
        return jsonify({
            "contacts": [], "unread": 0, "typing": [], "peers": [],
            "mesh_state": {
                "active": active, "peer_count": peer_count,
                "trusted_count": 0, "messageable_count": 0,
            },
        })
    # Build mesh_state from trust manager
    tm = bridge._trust_manager
    trusted_count = 0
    messageable_count = 0
    if tm:
        for rec in tm._trust_store.get_all():
            if rec.trust_level == "trusted":
                trusted_count += 1
                if rec.permissions.send_messages:
                    messageable_count += 1
    gw = bridge._gateway
    peer_count = len(gw.peer_gateways) if gw and hasattr(gw, "peer_gateways") else 0
    peers = bridge.get_messageable_peers()
    return jsonify({
        "contacts": bridge.get_contacts(),
        "unread": bridge.get_unread_count(),
        "typing": bridge.get_typing_peers(),
        "peers": peers,
        "mesh_state": {
            "active": True,
            "peer_count": peer_count,
            "trusted_count": trusted_count,
            "messageable_count": messageable_count,
        },
    })


@app.route("/api/messages/<peer_id>")
@require_auth
def api_messages_history(peer_id):
    """Get message history for a conversation."""
    bridge = _get_message_bridge()
    if not bridge:
        return jsonify({"messages": []})
    limit = request.args.get("limit", 50, type=int)
    before = request.args.get("before", None, type=str)
    messages = bridge.get_messages(peer_id, limit=limit, before=before)
    return jsonify({"messages": messages})


@app.route("/api/messages/<peer_id>/send", methods=["POST"])
@require_auth
def api_messages_send(peer_id):
    """Send a message to a peer."""
    bridge = _get_message_bridge()
    if not bridge:
        return jsonify({"error": "Messaging not available"}), 503
    data = request.json or {}
    body = data.get("body", "").strip()
    if not body:
        return jsonify({"error": "Empty message"}), 400
    reply_to = data.get("reply_to")
    result = bridge.send_message(peer_id, body, reply_to=reply_to)
    if result.get("status") == "error":
        return jsonify(result), 403
    return jsonify(result)


@app.route("/api/messages/<peer_id>/read", methods=["POST"])
@require_auth
def api_messages_mark_read(peer_id):
    """Mark a conversation as read."""
    bridge = _get_message_bridge()
    if not bridge:
        return jsonify({"ok": False})
    bridge.mark_as_read(peer_id)
    return jsonify({"ok": True})


@app.route("/api/messages/<peer_id>/typing", methods=["POST"])
@require_auth
def api_messages_typing(peer_id):
    """Send typing indicator to a peer."""
    bridge = _get_message_bridge()
    if not bridge:
        return jsonify({"ok": False})
    bridge.send_typing(peer_id)
    return jsonify({"ok": True})


@app.route("/api/messages/peers")
@require_auth
def api_messages_peers():
    """Get trusted peers available for new conversations."""
    bridge = _get_message_bridge()
    if not bridge:
        return jsonify({"peers": []})
    return jsonify({"peers": bridge.get_messageable_peers()})


# ============================================================
# Corpus / Training Dashboard
# ============================================================


@app.route("/api/corpus/status")
@require_auth
def api_corpus_status():
    """Overall corpus status: consent, latest run, entry count."""
    from familiar.skills.training.corpus import (
        CURRENT_CONSENT_VERSION,
        check_corpus_consent,
        get_corpus_status,
        load_consent_registry,
    )

    status = get_corpus_status()
    registry = load_consent_registry()
    user_id = "default"
    rec = registry.get(user_id, {})
    status["consent"] = {
        "active": check_corpus_consent(user_id),
        "consent_version": rec.get("consent_version", ""),
        "granted_at": rec.get("granted_at", ""),
        "chain_block_hash": rec.get("chain_block_hash", ""),
        "revoked": rec.get("revoked", False),
    }
    status["consent_version"] = CURRENT_CONSENT_VERSION
    return jsonify(status)


@app.route("/api/corpus/consent", methods=["POST"])
@require_auth
def api_corpus_consent():
    """Grant or revoke corpus consent."""
    from familiar.skills.training.corpus import grant_corpus_consent, revoke_corpus_consent

    data = request.get_json(silent=True) or {}
    action = data.get("action", "grant")
    user_id = data.get("user_id", "default")
    if action == "grant":
        block_hash = grant_corpus_consent(user_id)
        return jsonify({"ok": True, "chain_block_hash": block_hash})
    elif action == "revoke":
        revoke_corpus_consent(user_id)
        return jsonify({"ok": True})
    else:
        return jsonify({"ok": False, "error": f"Unknown action: {action}"}), 400


@app.route("/api/corpus/memories")
@require_auth
def api_corpus_memories():
    """List markdown memory files with training eligibility metadata."""
    from familiar.skills.training.corpus import list_memories

    memories = list_memories()
    return jsonify({"memories": memories, "total": len(memories)})


@app.route("/api/corpus/memories/toggle", methods=["POST"])
@require_auth
def api_corpus_memories_toggle():
    """Toggle training_eligible on a memory file."""
    from familiar.skills.training.corpus import toggle_memory_eligible

    data = request.get_json(silent=True) or {}
    filename = data.get("filename", "")
    eligible = bool(data.get("eligible", False))
    if not filename:
        return jsonify({"ok": False, "error": "filename required"}), 400
    found = toggle_memory_eligible(filename, eligible)
    if found:
        return jsonify({"ok": True, "filename": filename, "eligible": eligible})
    return jsonify({"ok": False, "error": f"File not found: {filename}"}), 404


@app.route("/api/corpus/entries")
@require_auth
def api_corpus_entries():
    """Return paginated corpus entries."""
    from familiar.skills.training.corpus import list_entries

    try:
        limit = int(request.args.get("limit", 20))
        offset = int(request.args.get("offset", 0))
    except (TypeError, ValueError):
        limit, offset = 20, 0
    limit = min(limit, 50)
    return jsonify(list_entries(limit=limit, offset=offset))


@app.route("/api/corpus/entries/<entry_id>", methods=["DELETE"])
@require_auth
def api_corpus_entry_delete(entry_id: str):
    """Delete a specific corpus entry by id."""
    from familiar.skills.training.corpus import delete_entry

    found = delete_entry(entry_id)
    if found:
        return jsonify({"ok": True})
    return jsonify({"ok": False, "error": "Entry not found"}), 404


@app.route("/api/corpus/run", methods=["POST"])
@require_auth
def api_corpus_run():
    """Trigger a corpus pipeline run."""
    from familiar.skills.training.corpus import (
        CorpusPipeline,
        check_corpus_consent,
        record_corpus_run,
    )

    if not check_corpus_consent("default"):
        return jsonify({"ok": False, "error": "No active corpus consent. Grant consent first."})
    try:
        pipeline = CorpusPipeline()
        summary = pipeline.run()
        record_corpus_run(summary)
        return jsonify({"ok": True, "summary": summary})
    except Exception as e:
        logger.exception("Corpus pipeline run failed")
        return jsonify({"ok": False, "error": str(e)}), 500


# ============================================================
# Social Dashboard
# ============================================================

_SOCIAL_TIMEOUT = 15


def _strip_html(text: str) -> str:
    """Strip HTML tags from text (double-pass to catch entity-encoded tags)."""
    import html as _html_mod

    text = re.sub(r"<[^>]+>", "", text)
    text = _html_mod.unescape(text)
    text = re.sub(r"<[^>]+>", "", text)
    return text.strip()


def _social_fetch_bluesky(endpoint: str, params: dict | None = None) -> dict | None:
    """Fetch from Bluesky XRPC API using skill-layer auth."""
    try:
        from familiar.skills.bluesky.skill import _base_url, _configured, _headers

        if not _configured():
            return None
        import httpx

        resp = httpx.get(
            f"{_base_url()}/xrpc/{endpoint}",
            headers=_headers(),
            params=params,
            timeout=_SOCIAL_TIMEOUT,
        )
        if resp.status_code != 200:
            logger.warning("Bluesky %s returned %s", endpoint, resp.status_code)
            return None
        return resp.json()
    except Exception as e:
        logger.error("Bluesky fetch error (%s): %s", endpoint, e)
        return None


def _social_fetch_mastodon(endpoint: str, params: dict | None = None) -> dict | None:
    """Fetch from Mastodon REST API using env var Bearer token."""
    try:
        from familiar.skills.mastodon.skill import _base_url, _configured, _headers

        if not _configured():
            return None
        import httpx

        resp = httpx.get(
            f"{_base_url()}{endpoint}",
            headers=_headers(),
            params=params,
            timeout=_SOCIAL_TIMEOUT,
        )
        if resp.status_code != 200:
            logger.warning("Mastodon %s returned %s", endpoint, resp.status_code)
            return None
        return resp.json()
    except Exception as e:
        logger.error("Mastodon fetch error (%s): %s", endpoint, e)
        return None


@app.route("/api/social/accounts")
@limiter.limit("30 per minute")
@require_auth
def api_social_accounts():
    """Return which social platforms are configured."""
    bluesky = False
    mastodon = False
    try:
        from familiar.skills.bluesky.skill import _configured as bs_conf

        bluesky = bs_conf()
    except Exception:
        pass
    try:
        from familiar.skills.mastodon.skill import _configured as ma_conf

        mastodon = ma_conf()
    except Exception:
        pass
    return jsonify({"bluesky": bluesky, "mastodon": mastodon})


@app.route("/api/social/config")
@limiter.limit("30 per minute")
@require_auth
def api_social_config():
    """Return display names, handles, avatars for configured accounts."""
    accounts = {}
    try:
        from familiar.skills.bluesky.skill import _configured as bs_conf

        if bs_conf():
            from familiar.skills.bluesky.skill import _get_session

            session = _get_session()
            if session:
                handle = session.get("handle", "")
                profile = _social_fetch_bluesky(
                    "app.bsky.actor.getProfile", {"actor": handle}
                )
                accounts["bluesky"] = {
                    "handle": handle,
                    "display_name": (profile or {}).get("displayName", handle),
                    "avatar": (profile or {}).get("avatar", ""),
                }
    except Exception:
        pass
    try:
        from familiar.skills.mastodon.skill import _configured as ma_conf

        if ma_conf():
            profile = _social_fetch_mastodon("/api/v1/accounts/verify_credentials")
            if profile:
                accounts["mastodon"] = {
                    "handle": f"@{profile.get('acct', '?')}",
                    "display_name": profile.get("display_name", ""),
                    "avatar": profile.get("avatar", ""),
                }
    except Exception:
        pass
    return jsonify({"accounts": accounts})


@app.route("/api/social/feed")
@limiter.limit("30 per minute")
@require_auth
def api_social_feed():
    """Combined timeline from configured platforms, sorted by timestamp."""
    platform_filter = request.args.get("platform", "")
    cursor = request.args.get("cursor", "")
    posts = []

    # Bluesky feed
    if platform_filter in ("", "bluesky"):
        params = {"limit": "30"}
        if cursor and platform_filter == "bluesky":
            params["cursor"] = cursor
        data = _social_fetch_bluesky("app.bsky.feed.getTimeline", params)
        if data:
            for item in data.get("feed", []):
                post = item.get("post", {})
                author = post.get("author", {})
                record = post.get("record", {})
                posts.append({
                    "platform": "bluesky",
                    "author": author.get("handle", "?"),
                    "display_name": author.get("displayName", ""),
                    "avatar": author.get("avatar", ""),
                    "text": record.get("text", ""),
                    "timestamp": record.get("createdAt", ""),
                    "likes": post.get("likeCount", 0),
                    "reposts": post.get("repostCount", 0),
                    "replies": post.get("replyCount", 0),
                    "uri": post.get("uri", ""),
                })

    # Mastodon feed
    if platform_filter in ("", "mastodon"):
        params = {"limit": "30"}
        if cursor and platform_filter == "mastodon":
            params["max_id"] = cursor
        data = _social_fetch_mastodon("/api/v1/timelines/home", params)
        if data and isinstance(data, list):
            for status in data:
                acct = status.get("account", {})
                posts.append({
                    "platform": "mastodon",
                    "author": f"@{acct.get('acct', '?')}",
                    "display_name": acct.get("display_name", ""),
                    "avatar": acct.get("avatar", ""),
                    "text": _strip_html(status.get("content", "")),
                    "timestamp": status.get("created_at", ""),
                    "likes": status.get("favourites_count", 0),
                    "reposts": status.get("reblogs_count", 0),
                    "replies": status.get("replies_count", 0),
                    "uri": status.get("url", status.get("uri", "")),
                })

    # Sort combined by timestamp descending
    posts.sort(key=lambda p: p.get("timestamp", ""), reverse=True)

    return jsonify({"posts": posts, "cursor": ""})


@app.route("/api/social/notifications")
@limiter.limit("30 per minute")
@require_auth
def api_social_notifications():
    """Merged notifications from configured platforms."""
    notifications = []

    # Bluesky notifications
    data = _social_fetch_bluesky(
        "app.bsky.notification.listNotifications", {"limit": "30"}
    )
    if data:
        for n in data.get("notifications", []):
            author = n.get("author", {})
            record = n.get("record", {})
            text = record.get("text", "")
            notifications.append({
                "platform": "bluesky",
                "type": n.get("reason", "unknown"),
                "author": author.get("handle", "?"),
                "avatar": author.get("avatar", ""),
                "text": text[:200] if text else "",
                "timestamp": n.get("indexedAt", ""),
                "is_read": n.get("isRead", False),
            })

    # Mastodon notifications
    data = _social_fetch_mastodon("/api/v1/notifications", {"limit": "30"})
    if data and isinstance(data, list):
        for n in data:
            acct = n.get("account", {})
            status = n.get("status", {})
            text = _strip_html(status.get("content", "")) if status else ""
            notifications.append({
                "platform": "mastodon",
                "type": n.get("type", "unknown"),
                "author": f"@{acct.get('acct', '?')}",
                "avatar": acct.get("avatar", ""),
                "text": text[:200] if text else "",
                "timestamp": n.get("created_at", ""),
                "is_read": False,
            })

    notifications.sort(key=lambda n: n.get("timestamp", ""), reverse=True)
    return jsonify({"notifications": notifications})


@app.route("/api/social/post", methods=["POST"])
@limiter.limit("30 per minute")
@require_auth
def api_social_post():
    """Create a post on selected platforms."""
    data = request.json or {}
    text = data.get("text", "").strip()
    platforms = data.get("platforms", [])

    if not text:
        return jsonify({"error": "Post text is required"}), 400
    if not platforms:
        return jsonify({"error": "Select at least one platform"}), 400

    results = {}

    if "bluesky" in platforms:
        try:
            from familiar.skills.bluesky.skill import bluesky_post

            results["bluesky"] = bluesky_post({"text": text})
        except Exception as e:
            results["bluesky"] = f"Error: {e}"

    if "mastodon" in platforms:
        try:
            from familiar.skills.mastodon.skill import mastodon_post

            results["mastodon"] = mastodon_post({"text": text})
        except Exception as e:
            results["mastodon"] = f"Error: {e}"

    return jsonify({"results": results})


@app.route("/api/social/poll")
@limiter.limit("30 per minute")
@require_auth
def api_social_poll():
    """Lightweight poll — returns unread notification count only."""
    count = 0

    data = _social_fetch_bluesky(
        "app.bsky.notification.getUnreadCount"
    )
    if data:
        count += data.get("count", 0)

    data = _social_fetch_mastodon("/api/v1/notifications", {"limit": "1"})
    if data and isinstance(data, list):
        # Mastodon doesn't have an unread-count endpoint natively;
        # approximate by checking if first notification is recent
        count += len(data)

    return jsonify({"unread_count": count})


@app.route("/api/mesh/config/name", methods=["POST"])
@require_auth
def api_mesh_config_name():
    """Set the node display name."""
    data = request.json or {}
    name = data.get("name", "").strip()
    if not name or len(name) < 2 or len(name) > 32:
        return jsonify({"error": "Name must be 2-32 characters"}), 400
    try:
        agent = get_agent()
        gw = getattr(agent, "_gateway", None)
        if not gw:
            return jsonify({"error": "Mesh gateway not active"}), 503
        if hasattr(gw, "config") and gw.config:
            gw.config.set("node_name", name)
        if hasattr(gw, "node_name"):
            gw.node_name = name
        # Update message bridge node_name too
        if hasattr(gw, "message_bridge") and gw.message_bridge:
            gw.message_bridge.node_name = name
        return jsonify({"success": True, "name": name})
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/onboard/server-recommendations")
@_require_localhost
def api_onboard_server_recommendations():
    """Get service recommendations based on job class."""
    try:
        engine = _get_onboard_engine()
        result = engine.get_server_recommendations()
        return jsonify(result.data if result.success else {"recommendations": []})
    except Exception:
        return jsonify({"recommendations": []})


@app.route("/api/server/full-setup", methods=["POST"])
@require_auth
def api_server_full_setup():
    """One-button server setup: detect runtime → install → provision → start.

    No LLM involved. Reads the user's job class, maps it to services,
    and does everything from installing Podman to starting containers.

    Body (optional):
        job_class: Override job class (default: read from creature state)
        services: Explicit list of service keys (overrides job class mapping)
    """
    mgr = _get_service_manager()
    data = request.get_json(silent=True) or {}

    # Determine which services to set up
    service_keys = data.get("services")
    if not service_keys:
        try:
            engine = _get_onboard_engine()
            result = engine.get_server_recommendations()
            service_keys = [r["key"] for r in result.data.get("recommendations", [])] if result.success else []
        except Exception:
            service_keys = []

    if not service_keys:
        return jsonify({"error": "No services to set up. Set a job class first."}), 400

    # Run the full setup pipeline
    try:
        result = mgr.full_server_setup(service_keys)
        return jsonify(result)
    except Exception as exc:
        return jsonify({"error": str(exc)}), 500


@app.route("/api/server/bootstrap", methods=["POST"])
@require_auth
def api_server_bootstrap():
    """Bootstrap container runtime from scratch.

    If no container runtime (Podman/Docker) is detected, installs Podman via
    the system package manager. Optionally chains to auto-provision afterwards.

    Query params:
        auto_provision: If "true", runs auto-provision after bootstrap succeeds.
    """
    mgr = _get_service_manager()

    # Bootstrap the runtime
    result = mgr.bootstrap_runtime()
    if not result.get("success"):
        return jsonify(result), 500

    # Optionally chain to auto-provision
    if request.args.get("auto_provision", "").lower() == "true" and not result.get("already_installed"):
        # Runtime just installed — re-init manager runtime cache
        mgr.invalidate_runtime()

    return jsonify(result)


@app.route("/api/server/auto-provision", methods=["POST"])
@require_auth
def api_server_auto_provision():
    """Auto-provision all recommended services for the current job class.

    Iterates through the job class service map, skips already-running services,
    and provisions + starts each one. Returns per-service results.
    """
    mgr = _get_service_manager()

    # Auto-bootstrap container runtime if missing
    from familiar.services.manager import ContainerRuntime
    if mgr.detect_runtime() == ContainerRuntime.NONE:
        bootstrap = mgr.bootstrap_runtime()
        if not bootstrap.get("success"):
            return jsonify({
                "error": "No container runtime available and auto-install failed: "
                + bootstrap.get("error", "unknown"),
                "bootstrap_attempted": True,
            }), 500

    try:
        engine = _get_onboard_engine()
        result = engine.get_server_recommendations()
        recs = [r["key"] for r in result.data.get("recommendations", [])] if result.success else []
    except Exception:
        recs = []

    if not recs:
        return jsonify({"error": "No services recommended for current job class."}), 400

    # Check current state
    results = []
    to_provision = []

    for key in recs:
        st = mgr.get_status(key)
        state = st.get("status", "unknown")
        if state == "running":
            results.append({"service": key, "status": "already_running", "ok": True})
        else:
            to_provision.append(key)

    # Provision and start in order (respecting depends_on)
    from familiar.services.specs import SERVICE_SPECS
    for key in to_provision:
        spec = SERVICE_SPECS.get(key)
        if not spec:
            results.append({"service": key, "status": "unknown_spec", "ok": False})
            continue
        if spec.service_type.value == "native":
            # Native services (cockpit) — just check if systemd unit exists
            if spec.systemctl_unit:
                results.append({"service": key, "status": "native_skipped", "ok": True,
                                "hint": f"Install via: sudo dnf install {key} or sudo apt install {key}"})
            else:
                results.append({"service": key, "status": "native_skipped", "ok": True})
            continue
        try:
            prov = mgr.provision(key)
            if prov.get("error"):
                results.append({"service": key, "status": "provision_failed",
                                "error": prov["error"], "ok": False})
                continue
            start = mgr.start(key)
            if start.get("error"):
                results.append({"service": key, "status": "start_failed",
                                "error": start["error"], "ok": False})
            else:
                entry = {"service": key, "status": "started", "ok": True}
                if spec.setup_notes:
                    entry["setup_notes"] = spec.setup_notes
                results.append(entry)
        except Exception as e:
            results.append({"service": key, "status": "error", "error": str(e), "ok": False})

    ok_count = sum(1 for r in results if r["ok"])
    return jsonify({
        "results": results,
        "total": len(recs),
        "ok": ok_count,
        "failed": len(results) - ok_count,
        "job_class": result.data.get("job_class", "") if result.success else "",
    })


# ============================================================
# Nonprofit CRUD API
# ============================================================


@app.route("/api/nonprofit/donors")
@require_auth
def api_nonprofit_donors():
    """List all donors."""
    from familiar.skills.nonprofit.skill import get_all_donors
    return jsonify({"donors": get_all_donors()})


@app.route("/api/nonprofit/donors", methods=["POST"])
@require_auth
def api_nonprofit_donor_add():
    """Create a new donor."""
    data = request.json or {}
    if not data.get("name", "").strip():
        return jsonify({"error": "Name is required."}), 400

    from familiar.skills.nonprofit.skill import log_donor
    log_donor(data)

    # Return the newly created donor
    from familiar.skills.nonprofit.skill import get_all_donors
    donors = get_all_donors()
    # Find by name match (just created)
    created = next(
        (d for d in donors if d["name"].lower() == data["name"].strip().lower()),
        donors[-1] if donors else None,
    )
    return jsonify({"donor": created}), 201


@app.route("/api/nonprofit/donors/<donor_id>", methods=["PUT"])
@require_auth
def api_nonprofit_donor_update(donor_id):
    """Update a donor."""
    data = request.json or {}
    data["id"] = donor_id

    from familiar.skills.nonprofit.skill import update_donor
    result = update_donor(data)
    if "error" in result:
        return jsonify(result), 404
    return jsonify({"donor": result})


@app.route("/api/nonprofit/donors/<donor_id>", methods=["DELETE"])
@require_auth
def api_nonprofit_donor_delete(donor_id):
    """Delete a donor."""
    from familiar.skills.nonprofit.skill import delete_donor
    result = delete_donor(donor_id)
    if "error" in result:
        return jsonify(result), 404
    return jsonify(result)


@app.route("/api/nonprofit/donors/<donor_id>/interaction", methods=["POST"])
@require_auth
def api_nonprofit_donor_interaction(donor_id):
    """Log an interaction for a donor."""
    data = request.json or {}
    data["donor_id"] = donor_id

    from familiar.skills.nonprofit.skill import add_interaction
    result = add_interaction(data)
    if "error" in result:
        return jsonify(result), 404
    return jsonify({"donor": result})


@app.route("/api/nonprofit/grants")
@require_auth
def api_nonprofit_grants():
    """List all grants."""
    from familiar.skills.nonprofit.skill import get_all_grants
    return jsonify({"grants": get_all_grants()})


@app.route("/api/nonprofit/grants", methods=["POST"])
@require_auth
def api_nonprofit_grant_add():
    """Create a new grant."""
    data = request.json or {}
    if not data.get("name", "").strip():
        return jsonify({"error": "Grant name is required."}), 400

    from familiar.skills.nonprofit.skill import add_grant
    add_grant(data)

    from familiar.skills.nonprofit.skill import get_all_grants
    grants = get_all_grants()
    return jsonify({"grant": grants[-1] if grants else None}), 201


@app.route("/api/nonprofit/grants/<grant_id>", methods=["PUT"])
@require_auth
def api_nonprofit_grant_update(grant_id):
    """Update a grant."""
    data = request.json or {}
    data["id"] = grant_id

    from familiar.skills.nonprofit.skill import update_grant
    result = update_grant(data)
    if "error" in result:
        return jsonify(result), 404
    return jsonify({"grant": result})


@app.route("/api/nonprofit/grants/<grant_id>", methods=["DELETE"])
@require_auth
def api_nonprofit_grant_delete(grant_id):
    """Delete a grant."""
    from familiar.skills.nonprofit.skill import delete_grant
    result = delete_grant(grant_id)
    if "error" in result:
        return jsonify(result), 404
    return jsonify(result)


@app.route("/api/nonprofit/notes")
@require_auth
def api_nonprofit_notes():
    """List all notes."""
    from familiar.skills.nonprofit.skill import get_all_notes
    return jsonify({"notes": get_all_notes()})


@app.route("/api/nonprofit/notes", methods=["POST"])
@require_auth
def api_nonprofit_note_add():
    """Create a new note."""
    data = request.json or {}
    if not data.get("content", "").strip():
        return jsonify({"error": "Note content is required."}), 400

    from familiar.skills.nonprofit.skill import add_note
    add_note(data)

    from familiar.skills.nonprofit.skill import get_all_notes
    notes = get_all_notes()
    return jsonify({"note": notes[-1] if notes else None}), 201


@app.route("/api/nonprofit/notes/<note_id>", methods=["DELETE"])
@require_auth
def api_nonprofit_note_delete(note_id):
    """Delete a note."""
    from familiar.skills.nonprofit.skill import delete_note
    result = delete_note(note_id)
    if "error" in result:
        return jsonify(result), 404
    return jsonify(result)


# ============================================================
# Bookkeeping CRUD API
# ============================================================


@app.route("/api/bookkeeping/transactions")
@require_auth
def api_bookkeeping_transactions():
    """List transactions with optional filters."""
    from familiar.skills.bookkeeping.skill import get_all_transactions
    filters = {
        "start_date": request.args.get("start_date"),
        "end_date": request.args.get("end_date"),
        "category": request.args.get("category"),
        "fund": request.args.get("fund"),
        "type": request.args.get("type"),
    }
    # Remove None values
    filters = {k: v for k, v in filters.items() if v}
    txns = get_all_transactions(filters if filters else None)
    limit = request.args.get("limit", 200, type=int)
    return jsonify({"transactions": txns[:limit]})


@app.route("/api/bookkeeping/transactions", methods=["POST"])
@require_auth
def api_bookkeeping_transaction_add():
    """Create a new transaction (income or expense)."""
    data = request.json or {}
    if not data.get("amount"):
        return jsonify({"error": "Amount is required."}), 400

    tx_type = data.get("type", "income")
    if tx_type == "expense":
        from familiar.skills.bookkeeping.skill import log_expense
        log_expense(data)
    else:
        from familiar.skills.bookkeeping.skill import log_income
        log_income(data)

    from familiar.skills.bookkeeping.skill import get_all_transactions
    txns = get_all_transactions()
    return jsonify({"transaction": txns[0] if txns else None}), 201


@app.route("/api/bookkeeping/transactions/<tx_id>", methods=["PUT"])
@require_auth
def api_bookkeeping_transaction_update(tx_id):
    """Update a transaction."""
    data = request.json or {}
    data["id"] = tx_id

    from familiar.skills.bookkeeping.skill import update_transaction
    result = update_transaction(data)
    if "error" in result:
        return jsonify(result), 404
    return jsonify({"transaction": result})


@app.route("/api/bookkeeping/transactions/<tx_id>", methods=["DELETE"])
@require_auth
def api_bookkeeping_transaction_delete(tx_id):
    """Delete a transaction."""
    from familiar.skills.bookkeeping.skill import delete_transaction
    result = delete_transaction(tx_id)
    if "error" in result:
        return jsonify(result), 404
    return jsonify(result)


@app.route("/api/bookkeeping/categories")
@require_auth
def api_bookkeeping_categories():
    """Get income/expense category lists."""
    from familiar.skills.bookkeeping.skill import get_categories
    return jsonify({"categories": get_categories()})


@app.route("/api/bookkeeping/funds")
@require_auth
def api_bookkeeping_funds():
    """Get fund list."""
    from familiar.skills.bookkeeping.skill import get_funds
    return jsonify({"funds": get_funds()})


@app.route("/api/nonprofit/import", methods=["POST"])
@require_auth
def api_nonprofit_import():
    """Import donors from CSV data.

    Expects JSON body: {"rows": [{"name": "...", "email": "...", ...}, ...]}
    or a CSV file upload via multipart form.
    """
    from familiar.skills.nonprofit.skill import log_donor

    rows = []
    if request.is_json:
        rows = (request.json or {}).get("rows", [])
    elif request.files.get("file"):
        import csv
        import io
        f = request.files["file"]
        text = f.read().decode("utf-8-sig")
        reader = csv.DictReader(io.StringIO(text))
        rows = list(reader)

    if not rows:
        return jsonify({"error": "No data provided."}), 400

    imported = 0
    errors = []
    for i, row in enumerate(rows):
        name = (row.get("name") or row.get("Name") or "").strip()
        if not name:
            errors.append(f"Row {i + 1}: missing name")
            continue

        donor_data = {
            "name": name,
            "email": row.get("email") or row.get("Email") or "",
            "phone": row.get("phone") or row.get("Phone") or "",
            "notes": row.get("notes") or row.get("Notes") or "",
        }

        # Handle tags (comma-separated string or list)
        tags = row.get("tags") or row.get("Tags") or ""
        if isinstance(tags, str) and tags:
            donor_data["tags"] = [t.strip() for t in tags.split(",") if t.strip()]

        # If there's an amount, log as gift
        amount = row.get("amount") or row.get("Amount") or row.get("total_given") or ""
        if amount:
            try:
                # Strip currency symbols
                amount_clean = str(amount).replace("$", "").replace(",", "").strip()
                donor_data["amount"] = float(amount_clean)
                donor_data["type"] = "gift"
                donor_data["description"] = row.get("description") or "Imported"
            except (ValueError, TypeError):
                pass

        log_donor(donor_data)
        imported += 1

    return jsonify({
        "imported": imported,
        "errors": errors,
        "total_rows": len(rows),
    })


# ============================================================
# Financial Charts API
# ============================================================


@app.route("/api/bookkeeping/chart/<chart_type>")
@require_auth
def api_bookkeeping_chart(chart_type):
    """Generate a financial chart as base64 PNG.

    Chart types:
      - monthly: income vs expense bar chart by month (current year)
      - category_income: pie chart of income by category
      - category_expense: pie chart of expense by category
      - fund: bar chart of fund balances
    """
    import base64
    from datetime import datetime as _dt

    from familiar.skills.bookkeeping.skill import get_all_transactions

    year = request.args.get("year", str(_dt.now().year))
    txns = get_all_transactions({"start_date": f"{year}-01-01", "end_date": f"{year}-12-31"})

    try:
        from familiar.skills.charts.skill import render_chart_bytes
    except ImportError:
        return jsonify({"error": "Charts skill not available (matplotlib not installed)."}), 501

    chart_bytes = None

    if chart_type == "monthly":
        # Monthly income vs expenses
        months = {}
        for t in txns:
            m = t.get("date", "")[:7]  # "2026-01"
            if m not in months:
                months[m] = {"income": 0, "expense": 0}
            months[m][t["type"]] += t["amount"]
        sorted_months = sorted(months.keys())
        if not sorted_months:
            return jsonify({"error": "No transactions for this year."})
        labels = [m[5:] for m in sorted_months]  # "01", "02", etc.
        month_names = {"01": "Jan", "02": "Feb", "03": "Mar", "04": "Apr",
                       "05": "May", "06": "Jun", "07": "Jul", "08": "Aug",
                       "09": "Sep", "10": "Oct", "11": "Nov", "12": "Dec"}
        labels = [month_names.get(l, l) for l in labels]
        chart_bytes = render_chart_bytes(
            chart_type="bar",
            title=f"Income vs Expenses — {year}",
            labels=labels,
            series=[
                {"name": "Income", "values": [months[m]["income"] for m in sorted_months]},
                {"name": "Expenses", "values": [months[m]["expense"] for m in sorted_months]},
            ],
            y_label="Amount ($)",
            currency=True,
        )

    elif chart_type == "category_income":
        cats = {}
        for t in txns:
            if t["type"] == "income":
                cat = t.get("category", "other").replace("_", " ").title()
                cats[cat] = cats.get(cat, 0) + t["amount"]
        if not cats:
            return jsonify({"error": "No income transactions."})
        labels = list(cats.keys())
        values = list(cats.values())
        chart_bytes = render_chart_bytes(
            chart_type="pie",
            title=f"Income by Category — {year}",
            labels=labels,
            values=values,
            currency=True,
        )

    elif chart_type == "category_expense":
        cats = {}
        for t in txns:
            if t["type"] == "expense":
                cat = t.get("category", "other").replace("_", " ").title()
                cats[cat] = cats.get(cat, 0) + t["amount"]
        if not cats:
            return jsonify({"error": "No expense transactions."})
        labels = list(cats.keys())
        values = list(cats.values())
        chart_bytes = render_chart_bytes(
            chart_type="pie",
            title=f"Expenses by Category — {year}",
            labels=labels,
            values=values,
            currency=True,
        )

    elif chart_type == "fund":
        funds = {}
        for t in txns:
            fund = t.get("fund", "General")
            if fund not in funds:
                funds[fund] = {"income": 0, "expense": 0}
            funds[fund][t["type"]] += t["amount"]
        if not funds:
            return jsonify({"error": "No transactions."})
        labels = list(funds.keys())
        chart_bytes = render_chart_bytes(
            chart_type="bar",
            title=f"Fund Balances — {year}",
            labels=labels,
            series=[
                {"name": "Income", "values": [funds[f]["income"] for f in labels]},
                {"name": "Expenses", "values": [funds[f]["expense"] for f in labels]},
            ],
            y_label="Amount ($)",
            currency=True,
        )

    else:
        return jsonify({"error": f"Unknown chart type: {chart_type}"}), 400

    if not chart_bytes:
        return jsonify({"error": "Chart rendering failed — matplotlib may not be installed."}), 501

    b64 = base64.b64encode(chart_bytes).decode("ascii")
    return jsonify({"image": f"data:image/png;base64,{b64}", "chart_type": chart_type})


# ============================================================
# Grants.gov Search API
# ============================================================


@app.route("/api/grants-gov/search")
@require_auth
def api_grants_gov_search():
    """Search Grants.gov for federal funding opportunities."""
    keyword = request.args.get("keyword", "") or request.args.get("q", "")
    agency = request.args.get("agency", "")
    category = request.args.get("category", "")
    date_range = request.args.get("dateRange", "")  # posted within last N days
    sort_by = request.args.get("sortBy", "")  # closeDateAsc, closeDateDesc, openDate, etc.
    limit = min(int(request.args.get("limit", "25")), 100)
    offset = int(request.args.get("offset", "0"))

    if not keyword and not agency and not category:
        return jsonify({"error": "Provide keyword, agency, or category"}), 400

    try:
        from familiar.skills.grants_gov.skill import search_grants as _search_grants  # noqa: F401
        from familiar.skills._api_utils import api_post
        body = {
            "paging": {"num": limit, "offset": offset},
            "criteria": {"oppStatuses": "forecasted|posted"},
        }
        if keyword:
            body["criteria"]["keyword"] = keyword
        if agency:
            body["criteria"]["agency"] = agency
        if category:
            body["criteria"]["fundingCategory"] = category
        if date_range:
            body["criteria"]["dateRange"] = date_range
        if sort_by:
            body["criteria"]["sortBy"] = sort_by

        result = api_post(
            "https://api.grants.gov/v1/api/search2",
            json=body,
            headers={"Content-Type": "application/json"},
            service_name="Grants.gov",
        )
        search_data = result.get("data", result)
        opportunities = search_data.get("oppHits", [])
        grants = []
        for opp in opportunities:
            grants.append({
                "id": opp.get("id", opp.get("number", "")),
                "title": opp.get("title", "Untitled"),
                "agency": opp.get("agency", ""),
                "open_date": opp.get("openDate", ""),
                "close_date": opp.get("closeDate", ""),
                "award_floor": opp.get("awardFloor", 0),
                "award_ceiling": opp.get("awardCeiling", 0),
                "number": opp.get("number", ""),
                "status": opp.get("oppStatus", ""),
            })
        total = search_data.get("hitCount", search_data.get("totalCount", len(grants)))
        return jsonify({
            "grants": grants,
            "total": total,
            "offset": offset,
            "limit": limit,
            "has_more": (offset + limit) < total,
        })
    except Exception as e:
        return jsonify({"error": str(e)}), 502


# ============================================================
# Legislation Search + Tracking API
# ============================================================


@app.route("/api/legislation/search")
@require_auth
def api_legislation_search():
    """Search Congress.gov for federal bills."""
    query = request.args.get("query", "")
    congress = request.args.get("congress", "119")  # Current congress
    sort = request.args.get("sort", "updateDate+desc")
    limit = min(int(request.args.get("limit", "25")), 100)
    offset = int(request.args.get("offset", "0"))

    if not query:
        return jsonify({"error": "Provide a search query"}), 400

    try:
        import urllib.request
        import urllib.error

        api_key = os.environ.get("CONGRESS_API_KEY", "DEMO_KEY")
        params = (
            f"query={urllib.parse.quote(query)}"
            f"&limit={limit}&offset={offset}"
            f"&sort={urllib.parse.quote(sort)}"
            f"&format=json&api_key={api_key}"
        )
        url = f"https://api.congress.gov/v3/bill/{congress}?{params}"

        req = urllib.request.Request(url, headers={
            "Accept": "application/json",
            "User-Agent": "Familiar-Agent/1.15 (nonprofit-tool)",
        })
        with urllib.request.urlopen(req, timeout=15) as resp:
            data = json.loads(resp.read())

        bills = []
        for b in data.get("bills", []):
            latest = b.get("latestAction", {})
            bills.append({
                "bill_id": f"{b.get('type', '')}{b.get('number', '')}",
                "title": b.get("title", ""),
                "chamber": b.get("originChamber", ""),
                "congress": str(b.get("congress", congress)),
                "latest_action": latest.get("text", ""),
                "latest_action_date": latest.get("actionDate", ""),
                "url": b.get("url", "").replace("format=json", "").replace(
                    "api_key=" + api_key, ""
                ).rstrip("?&"),
                "type": b.get("type", ""),
                "number": str(b.get("number", "")),
                "update_date": b.get("updateDate", ""),
            })

        pagination = data.get("pagination", {})
        total = pagination.get("count", len(bills))

        return jsonify({
            "bills": bills,
            "total": total,
            "offset": offset,
            "limit": limit,
            "has_more": (offset + limit) < total,
        })
    except Exception as e:
        return jsonify({"error": str(e)}), 502


@app.route("/api/legislation/search/state")
@require_auth
def api_legislation_search_state():
    """Search state bills via OpenStates or LegiScan.

    Tries OpenStates first (OPENSTATES_API_KEY), falls back to LegiScan
    (LEGISCAN_API_KEY). Returns 501 if neither key is configured.
    """
    query = request.args.get("query", "")
    state = request.args.get("state", "")
    limit = min(int(request.args.get("limit", "25")), 50)
    page = int(request.args.get("page", "1"))

    if not query:
        return jsonify({"error": "Provide a search query"}), 400
    if not state:
        return jsonify({"error": "Provide a state (e.g. WA, CA, NY)"}), 400

    import urllib.request
    import urllib.error

    # --- Try OpenStates first ---
    os_key = os.environ.get("OPENSTATES_API_KEY", "")
    if os_key:
        try:
            params = (
                f"jurisdiction={urllib.parse.quote(state)}"
                f"&q={urllib.parse.quote(query)}"
                f"&page={page}&per_page={limit}"
                f"&sort=updated_desc"
                f"&apikey={os_key}"
            )
            url = f"https://v3.openstates.org/bills?{params}"
            req = urllib.request.Request(url, headers={
                "Accept": "application/json",
                "User-Agent": "Familiar-Agent/1.15 (nonprofit-tool)",
            })
            with urllib.request.urlopen(req, timeout=15) as resp:
                data = json.loads(resp.read())

            bills = []
            for b in data.get("results", []):
                latest = (b.get("latest_action_description") or "")
                latest_date = (b.get("latest_action_date") or "")[:10]
                bills.append({
                    "bill_id": b.get("identifier", ""),
                    "title": b.get("title", ""),
                    "chamber": b.get("from_organization", {}).get("name", ""),
                    "congress": b.get("session", ""),
                    "state": state.upper(),
                    "level": "state",
                    "latest_action": latest,
                    "latest_action_date": latest_date,
                    "url": b.get("openstates_url", ""),
                    "sponsor": (
                        b["sponsorships"][0]["name"]
                        if b.get("sponsorships") else ""
                    ),
                    "update_date": (b.get("updated_at") or "")[:10],
                })
            pagination = data.get("pagination", {})
            total = pagination.get("total_items", len(bills))
            max_page = pagination.get("max_page", 1)
            return jsonify({
                "bills": bills,
                "total": total,
                "page": page,
                "limit": limit,
                "has_more": page < max_page,
                "source": "openstates",
            })
        except Exception as e:
            logger.warning("OpenStates search failed: %s", e)
            # Fall through to LegiScan

    # --- Try LegiScan ---
    ls_key = os.environ.get("LEGISCAN_API_KEY", "")
    if ls_key:
        try:
            params = (
                f"key={ls_key}&op=searchRaw"
                f"&state={urllib.parse.quote(state.upper())}"
                f"&query={urllib.parse.quote(query)}"
                f"&page={page}"
            )
            url = f"https://api.legiscan.com/?{params}"
            req = urllib.request.Request(url, headers={
                "User-Agent": "Familiar-Agent/1.15 (nonprofit-tool)",
            })
            with urllib.request.urlopen(req, timeout=15) as resp:
                data = json.loads(resp.read())

            if data.get("status") == "ERROR":
                return jsonify({"error": data.get("alert", {}).get("message", "LegiScan error")}), 502

            search_result = data.get("searchresult", {})
            # LegiScan returns {summary: {}, 0: {}, 1: {}, ...}
            summary = search_result.pop("summary", {})
            total = summary.get("count", 0)

            bills = []
            for _key, b in sorted(search_result.items()):
                if not isinstance(b, dict) or "bill_number" not in b:
                    continue
                bills.append({
                    "bill_id": b.get("bill_number", ""),
                    "title": b.get("title", ""),
                    "chamber": "",
                    "congress": str(b.get("session_id", "")),
                    "state": b.get("state", state.upper()),
                    "level": "state",
                    "latest_action": b.get("last_action", ""),
                    "latest_action_date": b.get("last_action_date", ""),
                    "url": b.get("url", ""),
                    "sponsor": "",
                    "update_date": b.get("last_action_date", ""),
                })

            return jsonify({
                "bills": bills,
                "total": total,
                "page": page,
                "limit": limit,
                "has_more": page * 50 < total,  # LegiScan pages are 50 items
                "source": "legiscan",
            })
        except Exception as e:
            logger.warning("LegiScan search failed: %s", e)

    # --- Fallback: WA state has a free API (no key needed) ---
    if state.upper() == "WA":
        try:
            import xml.etree.ElementTree as _ET

            _wa_url = (
                "https://wslwebservices.leg.wa.gov/LegislationService.asmx/"
                "GetLegislativeStatusChangesByDateRange"
                f"?biennium=2025-26"
                f"&beginDate=2025-01-01&endDate=2026-12-31"
            )
            req = urllib.request.Request(_wa_url, headers={
                "User-Agent": "Familiar-Agent/1.15 (nonprofit-tool)",
            })
            with urllib.request.urlopen(req, timeout=15) as resp:
                xml_data = resp.read()
            root = _ET.fromstring(xml_data)
            ns = {"ws": "http://WSLWebServices.leg.wa.gov/"}
            changes = root.findall(".//ws:LegislativeStatus", ns)

            # Filter by keyword, deduplicate by bill ID, take most recent action
            q_lower = query.lower()
            bill_map = {}
            for c in changes:
                bid_el = c.find("ws:BillId", ns)
                action_el = c.find("ws:HistoryLine", ns)
                date_el = c.find("ws:ActionDate", ns)
                bid = bid_el.text if bid_el is not None else ""
                action = action_el.text if action_el is not None else ""
                adate = (date_el.text or "")[:10] if date_el is not None else ""

                # Match keyword against bill ID or action text
                if q_lower not in bid.lower() and q_lower not in action.lower():
                    continue

                if bid not in bill_map or adate > bill_map[bid]["latest_action_date"]:
                    bill_map[bid] = {
                        "bill_id": bid,
                        "title": bid,  # WA API doesn't include title in status changes
                        "chamber": "House" if bid.startswith("H") else "Senate",
                        "congress": "2025-26",
                        "state": "WA",
                        "level": "state",
                        "latest_action": action,
                        "latest_action_date": adate,
                        "url": f"https://app.leg.wa.gov/billsummary?BillNumber="
                               f"{bid.replace('HB ', '').replace('SB ', '').replace('SHB ', '').replace('SSB ', '').split()[0]}"
                               f"&Year=2025",
                        "sponsor": "",
                        "update_date": adate,
                    }

            bills = sorted(bill_map.values(), key=lambda b: b["latest_action_date"], reverse=True)
            total = len(bills)
            start = (page - 1) * limit
            page_bills = bills[start:start + limit]

            return jsonify({
                "bills": page_bills,
                "total": total,
                "page": page,
                "limit": limit,
                "has_more": start + limit < total,
                "source": "WA Legislature (leg.wa.gov)",
            })
        except Exception as e:
            logger.warning("WA Legislature search failed: %s", e)

    # Neither configured and no built-in fallback for this state
    return jsonify({
        "error": (
            "State legislation search requires an API key for most states. "
            "Set OPENSTATES_API_KEY (free at openstates.org/accounts/signup) "
            "or LEGISCAN_API_KEY (free at legiscan.com/legiscan) "
            "in Settings > Credentials. "
            "Washington State (WA) and Seattle work without keys."
        ),
        "setup_hint": True,
    }), 501


@app.route("/api/legislation/search/city")
@require_auth
def api_legislation_search_city():
    """Search city council legislation via Legistar API.

    Currently supports Seattle. Extensible to any city using Legistar
    (NYC, Chicago, LA, Portland, etc.) by adding to LEGISTAR_CITIES.
    """
    query = request.args.get("query", "")
    city = request.args.get("city", "seattle").lower()
    limit = min(int(request.args.get("limit", "25")), 50)
    page = int(request.args.get("page", "1"))

    if not query:
        return jsonify({"error": "Provide a search query"}), 400

    # Cities that use Legistar (free, no key needed)
    LEGISTAR_CITIES = {
        "seattle": {"client": "seattle", "label": "Seattle City Council"},
        "nyc": {"client": "nyc", "label": "New York City Council"},
        "chicago": {"client": "chicago", "label": "Chicago City Council"},
        "la": {"client": "lacity", "label": "Los Angeles City Council"},
        "portland": {"client": "portland", "label": "Portland City Council"},
        "denver": {"client": "denver", "label": "Denver City Council"},
        "philadelphia": {"client": "phlapi", "label": "Philadelphia City Council"},
        "san_francisco": {"client": "sfgov", "label": "San Francisco Board of Supervisors"},
    }

    city_cfg = LEGISTAR_CITIES.get(city)
    if not city_cfg:
        return jsonify({
            "error": f"City '{city}' not supported yet. Available: {', '.join(LEGISTAR_CITIES.keys())}",
            "available_cities": list(LEGISTAR_CITIES.keys()),
        }), 400

    try:
        import urllib.request
        import urllib.parse

        skip = (page - 1) * limit
        # Legistar OData filter — search title text
        odata_filter = urllib.parse.quote(
            f"substringof('{query}', MatterTitle) eq true"
        )
        url = (
            f"https://webapi.legistar.com/v1/{city_cfg['client']}/matters"
            f"?$top={limit}&$skip={skip}"
            f"&$filter={odata_filter}"
            f"&$orderby=MatterLastModifiedUtc+desc"
        )
        req = urllib.request.Request(url, headers={
            "Accept": "application/json",
            "User-Agent": "Familiar-Agent/1.15 (nonprofit-tool)",
        })
        with urllib.request.urlopen(req, timeout=15) as resp:
            matters = json.loads(resp.read())

        bills = []
        for m in (matters or []):
            title = m.get("MatterTitle", "") or m.get("MatterName", "")
            bills.append({
                "bill_id": m.get("MatterFile", ""),
                "title": title[:200],
                "chamber": m.get("MatterBodyName", ""),
                "congress": "",
                "state": "",
                "level": "city",
                "city": city,
                "latest_action": m.get("MatterStatusName", ""),
                "latest_action_date": (m.get("MatterIntroDate") or "")[:10],
                "url": (
                    f"https://{city_cfg['client']}.legistar.com/"
                    f"LegislationDetail.aspx?ID={m.get('MatterId', '')}"
                    f"&GUID={m.get('MatterGuid', '')}"
                ),
                "sponsor": "",
                "type": m.get("MatterTypeName", ""),
                "update_date": (m.get("MatterLastModifiedUtc") or "")[:10],
            })

        # Legistar doesn't return total count easily — estimate from results
        has_more = len(bills) == limit
        total = skip + len(bills) + (1 if has_more else 0)

        return jsonify({
            "bills": bills,
            "total": total,
            "page": page,
            "limit": limit,
            "has_more": has_more,
            "source": city_cfg["label"] + " (Legistar)",
        })
    except Exception as e:
        return jsonify({"error": str(e)}), 502


@app.route("/api/nonprofit/legislation")
@require_auth
def api_legislation_list():
    """List all tracked legislation (uses advocacy BillStore)."""
    try:
        from familiar.skills.advocacy.skill import get_bill_store
        store = get_bill_store()
        return jsonify({"legislation": store.list_bills()})
    except Exception as e:
        return jsonify({"legislation": [], "error": str(e)})


@app.route("/api/nonprofit/legislation", methods=["POST"])
@require_auth
def api_legislation_add():
    """Track a bill via advocacy BillStore."""
    data = request.json or {}
    if not data.get("title", "").strip():
        return jsonify({"error": "Bill title is required."}), 400
    try:
        from familiar.skills.advocacy.skill import get_bill_store
        store = get_bill_store()
        bill_id = store.track_bill(
            bill_number=data.get("bill_id", ""),
            title=data.get("title", ""),
            jurisdiction="federal" if data.get("level") == "federal" else data.get("state", ""),
            session=data.get("congress", ""),
            status="introduced",
            our_position="watch",
            priority=data.get("priority", "medium"),
            sponsor=data.get("sponsor", ""),
            summary=data.get("latest_action", ""),
        )
        if bill_id is None:
            return jsonify({"error": "Bill already tracked."}), 409
        bill = store.get_bill(bill_id)
        return jsonify({"bill": bill}), 201
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/nonprofit/legislation/<bill_id>", methods=["DELETE"])
@require_auth
def api_legislation_delete(bill_id):
    """Stop tracking a bill."""
    try:
        from familiar.skills.advocacy.skill import get_bill_store
        store = get_bill_store()
        ok = store.delete_bill(bill_id)
        return jsonify({"deleted": ok}), 200 if ok else 404
    except Exception as e:
        return jsonify({"error": str(e)}), 500


@app.route("/api/nonprofit/watchlist")
@require_auth
def api_watchlist_list():
    """Get bill keyword watchlist."""
    from familiar.skills.nonprofit.skill import get_bill_watchlist
    return jsonify({"watchlist": get_bill_watchlist()})


@app.route("/api/nonprofit/watchlist", methods=["POST"])
@require_auth
def api_watchlist_add():
    """Add keyword to watchlist."""
    data = request.json or {}
    if not data.get("keyword", "").strip():
        return jsonify({"error": "Keyword is required."}), 400
    from familiar.skills.nonprofit.skill import add_watchlist_keyword
    msg = add_watchlist_keyword(data)
    from familiar.skills.nonprofit.skill import get_bill_watchlist
    return jsonify({"message": msg, "watchlist": get_bill_watchlist()}), 201


@app.route("/api/nonprofit/watchlist/<keyword_id>", methods=["DELETE"])
@require_auth
def api_watchlist_delete(keyword_id):
    """Remove keyword from watchlist."""
    from familiar.skills.nonprofit.skill import remove_watchlist_keyword
    ok = remove_watchlist_keyword(keyword_id)
    return jsonify({"deleted": ok}), 200 if ok else 404


# ============================================================
# Social Worker Cases CRUD API
# ============================================================


@app.route("/api/cases")
@require_auth
def api_cases_list():
    """List all cases."""
    from familiar.dashboard.cases_crud import get_all_cases
    return jsonify({"cases": get_all_cases()})


@app.route("/api/cases", methods=["POST"])
@require_auth
def api_cases_create():
    """Create a new case."""
    from familiar.dashboard.cases_crud import add_case
    data = request.get_json(force=True)
    case = add_case(data)
    return jsonify({"case": case}), 201


@app.route("/api/cases/<case_id>", methods=["PUT"])
@require_auth
def api_cases_update(case_id):
    """Update a case."""
    from familiar.dashboard.cases_crud import update_case
    data = request.get_json(force=True)
    data["id"] = case_id
    result = update_case(data)
    if "error" in result:
        return jsonify(result), 404
    return jsonify({"case": result})


@app.route("/api/cases/<case_id>", methods=["DELETE"])
@require_auth
def api_cases_delete(case_id):
    """Delete a case."""
    from familiar.dashboard.cases_crud import delete_case
    result = delete_case(case_id)
    if "error" in result:
        return jsonify(result), 404
    return jsonify(result)


@app.route("/api/cases/<case_id>/interaction", methods=["POST"])
@require_auth
def api_cases_interaction(case_id):
    """Add an interaction to a case."""
    from familiar.dashboard.cases_crud import add_case_interaction
    data = request.get_json(force=True)
    data["case_id"] = case_id
    result = add_case_interaction(data)
    if "error" in result:
        return jsonify(result), 404
    return jsonify({"case": result})


@app.route("/api/cases/resources")
@require_auth
def api_resources_list():
    """List all resources."""
    from familiar.dashboard.cases_crud import get_all_resources
    return jsonify({"resources": get_all_resources()})


@app.route("/api/cases/resources", methods=["POST"])
@require_auth
def api_resources_create():
    """Create a new resource."""
    from familiar.dashboard.cases_crud import add_resource
    data = request.get_json(force=True)
    resource = add_resource(data)
    return jsonify({"resource": resource}), 201


@app.route("/api/cases/resources/<res_id>", methods=["PUT"])
@require_auth
def api_resources_update(res_id):
    """Update a resource."""
    from familiar.dashboard.cases_crud import update_resource
    data = request.get_json(force=True)
    data["id"] = res_id
    result = update_resource(data)
    if "error" in result:
        return jsonify(result), 404
    return jsonify({"resource": result})


@app.route("/api/cases/resources/<res_id>", methods=["DELETE"])
@require_auth
def api_resources_delete(res_id):
    """Delete a resource."""
    from familiar.dashboard.cases_crud import delete_resource
    result = delete_resource(res_id)
    if "error" in result:
        return jsonify(result), 404
    return jsonify(result)


@app.route("/api/cases/notes")
@require_auth
def api_case_notes_list():
    """List all case notes."""
    from familiar.dashboard.cases_crud import get_all_case_notes
    return jsonify({"notes": get_all_case_notes()})


@app.route("/api/cases/notes", methods=["POST"])
@require_auth
def api_case_notes_create():
    """Add a case note."""
    from familiar.dashboard.cases_crud import add_case_note
    data = request.get_json(force=True)
    note = add_case_note(data)
    return jsonify({"note": note}), 201


@app.route("/api/cases/notes/<note_id>", methods=["DELETE"])
@require_auth
def api_case_notes_delete(note_id):
    """Delete a case note."""
    from familiar.dashboard.cases_crud import delete_case_note
    result = delete_case_note(note_id)
    if "error" in result:
        return jsonify(result), 404
    return jsonify(result)


@app.route("/api/cases/crisis")
@require_auth
def api_crisis_list():
    """List all crisis events."""
    from familiar.dashboard.cases_crud import get_all_crisis_events
    return jsonify({"crisis_log": get_all_crisis_events()})


@app.route("/api/cases/crisis", methods=["POST"])
@require_auth
def api_crisis_create():
    """Log a crisis event."""
    from familiar.dashboard.cases_crud import add_crisis_event
    data = request.get_json(force=True)
    event = add_crisis_event(data)
    return jsonify({"event": event}), 201


@app.route("/api/cases/crisis/<event_id>", methods=["DELETE"])
@require_auth
def api_crisis_delete(event_id):
    """Delete a crisis event."""
    from familiar.dashboard.cases_crud import delete_crisis_event
    result = delete_crisis_event(event_id)
    if "error" in result:
        return jsonify(result), 404
    return jsonify(result)


# ── Clinical Assessments & Safety Plans (HMIS / SAMHSA) ──


@app.route("/api/cases/<case_id>/assessments")
@require_auth
def api_case_assessments_list(case_id):
    """List assessments for a case."""
    from familiar.dashboard.cases_crud import get_assessments
    return jsonify({"assessments": get_assessments(case_id)})


@app.route("/api/cases/<case_id>/assessments", methods=["POST"])
@require_auth
def api_case_assessments_create(case_id):
    """Add a clinical assessment (PHQ-9, GAD-7, C-SSRS, AUDIT, DAST-10, ACE)."""
    from familiar.dashboard.cases_crud import add_assessment
    data = request.get_json(force=True)
    result = add_assessment(case_id, data)
    if "error" in result:
        return jsonify(result), 404
    return jsonify({"assessment": result}), 201


@app.route("/api/cases/<case_id>/safety-plan", methods=["GET"])
@require_auth
def api_case_safety_plan_get(case_id):
    """Get safety plan for a case."""
    from familiar.dashboard.cases_crud import get_all_cases
    for case in get_all_cases():
        if case["id"] == case_id:
            return jsonify({"safety_plan": case.get("safety_plan", {})})
    return jsonify({"error": f"Case {case_id} not found"}), 404


@app.route("/api/cases/<case_id>/safety-plan", methods=["POST"])
@require_auth
def api_case_safety_plan_save(case_id):
    """Save or update a Stanley-Brown safety plan."""
    from familiar.dashboard.cases_crud import save_safety_plan
    data = request.get_json(force=True)
    result = save_safety_plan(case_id, data)
    if "error" in result:
        return jsonify(result), 404
    return jsonify({"safety_plan": result})


@app.route("/api/cases/<case_id>/mandatory-reports", methods=["POST"])
@require_auth
def api_case_mandatory_report(case_id):
    """Log a mandatory report."""
    from familiar.dashboard.cases_crud import add_mandatory_report
    data = request.get_json(force=True)
    result = add_mandatory_report(case_id, data)
    if "error" in result:
        return jsonify(result), 404
    return jsonify({"report": result}), 201


# ============================================================
# CSV Export Endpoints
# ============================================================


@app.route("/api/nonprofit/donors/export")
@require_auth
def api_nonprofit_donors_export():
    """Export donors as CSV file download."""
    import csv
    import io

    from familiar.skills.nonprofit.skill import get_all_donors

    donors = get_all_donors()
    output = io.StringIO()
    writer = csv.writer(output)
    writer.writerow(["name", "email", "phone", "total_given", "first_gift",
                      "last_gift", "tags", "notes", "interactions_count"])
    for d in donors:
        writer.writerow([
            d.get("name", ""),
            d.get("email", ""),
            d.get("phone", ""),
            d.get("total_given", 0),
            (d.get("first_gift") or "")[:10],
            (d.get("last_gift") or "")[:10],
            ", ".join(d.get("tags", [])),
            d.get("notes", ""),
            len(d.get("interactions", [])),
        ])

    resp = make_response(output.getvalue())
    resp.headers["Content-Disposition"] = "attachment; filename=donors.csv"
    resp.headers["Content-Type"] = "text/csv; charset=utf-8"
    return resp


@app.route("/api/nonprofit/grants/export")
@require_auth
def api_nonprofit_grants_export():
    """Export grants as CSV file download."""
    import csv
    import io

    from familiar.skills.nonprofit.skill import get_all_grants

    grants = get_all_grants()
    output = io.StringIO()
    writer = csv.writer(output)
    writer.writerow(["name", "funder", "amount", "status", "deadline",
                      "report_due", "start_date", "end_date", "notes"])
    for g in grants:
        writer.writerow([
            g.get("name", ""),
            g.get("funder", ""),
            g.get("amount", ""),
            g.get("status", ""),
            g.get("deadline", ""),
            g.get("report_due", ""),
            g.get("start_date", ""),
            g.get("end_date", ""),
            g.get("notes", ""),
        ])

    resp = make_response(output.getvalue())
    resp.headers["Content-Disposition"] = "attachment; filename=grants.csv"
    resp.headers["Content-Type"] = "text/csv; charset=utf-8"
    return resp


@app.route("/api/bookkeeping/transactions/export")
@require_auth
def api_bookkeeping_transactions_export():
    """Export transactions as CSV file download."""
    import csv
    import io

    from familiar.skills.bookkeeping.skill import get_all_transactions

    filters = {
        "start_date": request.args.get("start_date"),
        "end_date": request.args.get("end_date"),
        "category": request.args.get("category"),
        "fund": request.args.get("fund"),
        "type": request.args.get("type"),
    }
    filters = {k: v for k, v in filters.items() if v}
    txns = get_all_transactions(filters if filters else None)

    output = io.StringIO()
    writer = csv.writer(output)
    writer.writerow(["date", "type", "amount", "category", "fund", "source",
                      "vendor", "description", "id"])
    for t in txns:
        writer.writerow([
            t.get("date", ""),
            t.get("type", ""),
            t.get("amount", ""),
            t.get("category", ""),
            t.get("fund", ""),
            t.get("source", ""),
            t.get("vendor", ""),
            t.get("description", ""),
            t.get("id", ""),
        ])

    resp = make_response(output.getvalue())
    resp.headers["Content-Disposition"] = "attachment; filename=transactions.csv"
    resp.headers["Content-Type"] = "text/csv; charset=utf-8"
    return resp


# ============================================================
# Document Generation Endpoints
# ============================================================


@app.route("/api/nonprofit/generate/receipt", methods=["POST"])
@require_auth
def api_nonprofit_generate_receipt():
    """Generate a tax-deductible donation receipt for a donor."""
    from familiar.skills.documents.skill import create_receipt

    data = request.get_json(force=True)
    donor_name = data.get("donor_name", "Donor")
    amount = data.get("amount", 0)
    date_of_gift = data.get("date", "")
    description = data.get("description", "Charitable donation")

    result = create_receipt({
        "donor_name": donor_name,
        "amount": amount,
        "date_of_gift": date_of_gift,
        "description": description,
        "format": "docx",
    })
    # result is a string like "Receipt saved to /path/to/file.docx"
    # Extract path and send as download
    return _send_generated_doc(result)


@app.route("/api/nonprofit/generate/thankyou", methods=["POST"])
@require_auth
def api_nonprofit_generate_thankyou():
    """Generate a thank-you letter for a donor using the bundled template."""
    from familiar.skills.documents.skill import fill_template

    data = request.get_json(force=True)
    fields = {
        "donor_name": data.get("donor_name", "Valued Donor"),
        "amount": str(data.get("amount", "")),
        "organization": data.get("organization", "Our Organization"),
        "gift_date": data.get("gift_date", ""),
        "impact_area": data.get("impact_area", "our mission"),
        "personal_note": data.get("personal_note", ""),
        "mission_statement": data.get("mission_statement", ""),
        "signer_name": data.get("signer_name", ""),
        "signer_title": data.get("signer_title", "Executive Director"),
    }

    result = fill_template({
        "template": "nonprofit_director_donor_thankyou",
        "fields": fields,
        "format": "docx",
    })
    return _send_generated_doc(result)


@app.route("/api/nonprofit/generate/board-report", methods=["POST"])
@require_auth
def api_nonprofit_generate_board_report():
    """Generate a board report with auto-populated financial data."""
    from datetime import datetime as _dt

    from familiar.skills.bookkeeping.skill import get_all_transactions
    from familiar.skills.documents.skill import fill_template
    from familiar.skills.nonprofit.skill import get_all_donors, get_all_grants

    now = _dt.now()
    year = str(now.year)
    txns = get_all_transactions(
        {"start_date": f"{year}-01-01", "end_date": f"{year}-12-31"}
    )

    # Compute financials
    total_income = sum(t["amount"] for t in txns if t["type"] == "income")
    total_expense = sum(t["amount"] for t in txns if t["type"] == "expense")
    net = total_income - total_expense

    donors = get_all_donors()
    grants = get_all_grants()
    active_grants = [g for g in grants if g.get("status") in ("applied", "awarded")]

    # Build sections text
    fin_overview = (
        f"Year-to-date income: ${total_income:,.2f}\n"
        f"Year-to-date expenses: ${total_expense:,.2f}\n"
        f"Net position: ${net:,.2f}\n"
        f"Total transactions: {len(txns)}"
    )

    fundraising = (
        f"Total donors: {len(donors)}\n"
        f"Total donated (all time): ${sum(d.get('total_given', 0) for d in donors):,.2f}"
    )

    grant_lines = []
    for g in active_grants:
        status = g.get("status", "unknown").title()
        amt = g.get("amount", 0)
        grant_lines.append(f"- {g.get('name', 'Unnamed')}: {status} (${amt:,.0f})")
    grant_text = "\n".join(grant_lines) if grant_lines else "No active grants."

    data = request.get_json(force=True) or {}
    report_period = data.get("report_period", now.strftime("%B %Y"))

    fields = {
        "report_period": report_period,
        "executive_summary": data.get(
            "executive_summary",
            f"Financial and program summary for {report_period}.",
        ),
        "financial_overview": fin_overview,
        "program_updates": data.get("program_updates", "See grant pipeline below."),
        "fundraising_report": fundraising + "\n\nGrant Pipeline:\n" + grant_text,
        "upcoming_events": data.get("upcoming_events", "None scheduled."),
        "action_items": data.get("action_items", "None."),
        "motions": data.get("motions", "None."),
    }

    result = fill_template({
        "template": "nonprofit_director_board_report",
        "fields": fields,
        "format": "docx",
    })
    return _send_generated_doc(result)


@app.route("/api/nonprofit/generate/990-data", methods=["POST"])
@require_auth
def api_nonprofit_generate_990_data():
    """Generate 990 filing data export as a structured report."""
    from datetime import datetime as _dt

    from familiar.skills.bookkeeping.skill import get_all_transactions
    from familiar.skills.documents.skill import create_report
    from familiar.skills.nonprofit.skill import get_all_donors, get_all_grants

    data = request.get_json(force=True) or {}
    year = data.get("year", str(_dt.now().year))
    txns = get_all_transactions(
        {"start_date": f"{year}-01-01", "end_date": f"{year}-12-31"}
    )

    income_txns = [t for t in txns if t["type"] == "income"]
    expense_txns = [t for t in txns if t["type"] == "expense"]
    total_income = sum(t["amount"] for t in income_txns)
    total_expense = sum(t["amount"] for t in expense_txns)

    # Income by category
    income_cats = {}
    for t in income_txns:
        cat = t.get("category", "other").replace("_", " ").title()
        income_cats[cat] = income_cats.get(cat, 0) + t["amount"]

    # Expense by category
    expense_cats = {}
    for t in expense_txns:
        cat = t.get("category", "other").replace("_", " ").title()
        expense_cats[cat] = expense_cats.get(cat, 0) + t["amount"]

    donors = get_all_donors()
    grants = get_all_grants()
    awarded = [g for g in grants if g.get("status") == "awarded"]

    sections = [
        {
            "heading": "Revenue Summary",
            "body": f"Total Revenue: ${total_income:,.2f}",
            "table": {
                "headers": ["Category", "Amount"],
                "rows": [[k, f"${v:,.2f}"] for k, v in sorted(
                    income_cats.items(), key=lambda x: x[1], reverse=True
                )],
            },
        },
        {
            "heading": "Expense Summary",
            "body": f"Total Expenses: ${total_expense:,.2f}",
            "table": {
                "headers": ["Category", "Amount"],
                "rows": [[k, f"${v:,.2f}"] for k, v in sorted(
                    expense_cats.items(), key=lambda x: x[1], reverse=True
                )],
            },
        },
        {
            "heading": "Net Assets",
            "body": (
                f"Net Income: ${total_income - total_expense:,.2f}\n"
                f"Total transactions: {len(txns)}"
            ),
        },
        {
            "heading": "Contributor Information",
            "body": (
                f"Total individual contributors: {len(donors)}\n"
                f"Total individual contributions: "
                f"${sum(d.get('total_given', 0) for d in donors):,.2f}\n"
                f"Active grants received: {len(awarded)}\n"
                f"Grant revenue: "
                f"${sum(g.get('amount', 0) for g in awarded):,.2f}"
            ),
        },
    ]

    result = create_report({
        "title": f"Form 990 Data — Fiscal Year {year}",
        "sections": sections,
        "format": "docx",
        "letterhead": True,
    })
    return _send_generated_doc(result)


# ============================================================
# Nonprofit Email Pipeline API — Pending Events + Action Queue
# ============================================================


@app.route("/api/nonprofit/pending-events")
@require_auth
def api_nonprofit_pending_events():
    """List all pending calendar events detected from emails."""
    from familiar.skills.email.pending_events import get_pending_event_store
    store = get_pending_event_store()
    return jsonify({"events": store.get_pending(), "count": store.count_pending()})


@app.route("/api/nonprofit/pending-events/<event_id>/confirm", methods=["POST"])
@require_auth
def api_nonprofit_pending_event_confirm(event_id):
    """Confirm a pending event (add to calendar)."""
    from familiar.skills.email.pending_events import get_pending_event_store
    store = get_pending_event_store()
    if store.mark_confirmed(event_id):
        return jsonify({"status": "confirmed", "id": event_id})
    return jsonify({"error": "Event not found."}), 404


@app.route("/api/nonprofit/pending-events/<event_id>/dismiss", methods=["POST"])
@require_auth
def api_nonprofit_pending_event_dismiss(event_id):
    """Dismiss a pending event (don't add to calendar)."""
    from familiar.skills.email.pending_events import get_pending_event_store
    store = get_pending_event_store()
    if store.mark_dismissed(event_id):
        return jsonify({"status": "dismissed", "id": event_id})
    return jsonify({"error": "Event not found."}), 404


@app.route("/api/nonprofit/action-queue")
@require_auth
def api_nonprofit_action_queue():
    """List all pending email follow-up actions."""
    from familiar.skills.email.action_queue import get_action_store
    store = get_action_store()
    return jsonify({
        "actions": store.get_pending(),
        "count": store.count_pending(),
        "by_type": store.summary_by_type(),
    })


@app.route("/api/nonprofit/action-queue/<action_id>/complete", methods=["POST"])
@require_auth
def api_nonprofit_action_complete(action_id):
    """Mark an action as completed."""
    from familiar.skills.email.action_queue import get_action_store
    store = get_action_store()
    if store.mark_complete(action_id):
        return jsonify({"status": "completed", "id": action_id})
    return jsonify({"error": "Action not found."}), 404


@app.route("/api/nonprofit/action-queue/<action_id>/delegate", methods=["POST"])
@require_auth
def api_nonprofit_action_delegate(action_id):
    """Delegate an action to a staff member."""
    data = request.json or {}
    delegate_to = data.get("delegate_to", "").strip()
    if not delegate_to:
        return jsonify({"error": "delegate_to is required."}), 400

    from familiar.skills.email.action_queue import get_action_store
    store = get_action_store()
    if store.mark_delegated(action_id, delegate_to):
        return jsonify({"status": "delegated", "id": action_id, "delegate_to": delegate_to})
    return jsonify({"error": "Action not found."}), 404


# ============================================================
# Event Management API
# ============================================================


@app.route("/api/nonprofit/events")
@require_auth
def api_nonprofit_events():
    """List all events."""
    from familiar.skills.events.skill import get_event_store
    store = get_event_store()
    status = request.args.get("status")
    return jsonify({"events": store.list_events(status=status)})


@app.route("/api/nonprofit/events", methods=["POST"])
@require_auth
def api_nonprofit_event_create():
    """Create a new event."""
    data = request.json or {}
    if not data.get("name", "").strip():
        return jsonify({"error": "Event name is required."}), 400
    from familiar.skills.events.skill import get_event_store
    store = get_event_store()
    eid = store.create_event(
        name=data["name"].strip(),
        event_type=data.get("event_type", "meeting"),
        start_date=data.get("start_date", ""),
        end_date=data.get("end_date", ""),
        budget_total=float(data.get("budget_total", 0)),
        description=data.get("description", ""),
        volunteer_slots_total=int(data.get("volunteer_slots_total", 0)),
        venue=data.get("venue"),
        partner_orgs=data.get("partner_orgs"),
    )
    return jsonify({"event": store.get_event(eid)}), 201


@app.route("/api/nonprofit/events/<event_id>", methods=["PUT"])
@require_auth
def api_nonprofit_event_update(event_id):
    """Update an event."""
    data = request.json or {}
    from familiar.skills.events.skill import get_event_store
    store = get_event_store()
    result = store.update_event(event_id, data)
    if result:
        return jsonify({"event": result})
    return jsonify({"error": "Event not found."}), 404


@app.route("/api/nonprofit/events/<event_id>", methods=["DELETE"])
@require_auth
def api_nonprofit_event_delete(event_id):
    """Delete an event."""
    from familiar.skills.events.skill import get_event_store
    store = get_event_store()
    if store.delete_event(event_id):
        return jsonify({"status": "deleted"})
    return jsonify({"error": "Event not found."}), 404


@app.route("/api/nonprofit/events/<event_id>/vendors")
@require_auth
def api_nonprofit_event_vendors(event_id):
    """List vendors for an event."""
    from familiar.skills.events.skill import get_vendor_store
    return jsonify({"vendors": get_vendor_store().list_vendors(event_id=event_id)})


@app.route("/api/nonprofit/events/<event_id>/vendors", methods=["POST"])
@require_auth
def api_nonprofit_event_vendor_add(event_id):
    """Add a vendor to an event."""
    data = request.json or {}
    if not data.get("name", "").strip():
        return jsonify({"error": "Vendor name is required."}), 400
    from familiar.skills.events.skill import get_vendor_store
    store = get_vendor_store()
    vid = store.add_vendor(
        event_id=event_id,
        name=data["name"].strip(),
        service_type=data.get("service_type", "other"),
        contact_name=data.get("contact_name", ""),
        contact_email=data.get("contact_email", ""),
        quote_amount=float(data.get("quote_amount", 0)),
    )
    return jsonify({"vendor_id": vid}), 201


@app.route("/api/nonprofit/events/<event_id>/volunteers")
@require_auth
def api_nonprofit_event_volunteers(event_id):
    """List volunteers for an event."""
    from familiar.skills.events.skill import get_volunteer_store
    return jsonify({"volunteers": get_volunteer_store().list_volunteers(event_id=event_id)})


@app.route("/api/nonprofit/events/<event_id>/volunteers", methods=["POST"])
@require_auth
def api_nonprofit_event_volunteer_add(event_id):
    """Add a volunteer to an event."""
    data = request.json or {}
    if not data.get("name", "").strip():
        return jsonify({"error": "Volunteer name is required."}), 400
    from familiar.skills.events.skill import get_volunteer_store
    store = get_volunteer_store()
    vid = store.add_volunteer(
        event_id=event_id,
        name=data["name"].strip(),
        email=data.get("email", ""),
        role=data.get("role", "other"),
        shift_start=data.get("shift_start", ""),
        shift_end=data.get("shift_end", ""),
    )
    return jsonify({"volunteer_id": vid}), 201


# ============================================================
# Advocacy & Legislative API
# ============================================================


@app.route("/api/nonprofit/bills")
@require_auth
def api_nonprofit_bills():
    """List tracked bills."""
    from familiar.skills.advocacy.skill import get_bill_store
    store = get_bill_store()
    return jsonify({"bills": store.list_bills(
        position=request.args.get("position"),
        priority=request.args.get("priority"),
    )})


@app.route("/api/nonprofit/bills", methods=["POST"])
@require_auth
def api_nonprofit_bill_add():
    """Track a new bill."""
    data = request.json or {}
    if not data.get("bill_number", "").strip() or not data.get("title", "").strip():
        return jsonify({"error": "bill_number and title are required."}), 400
    from familiar.skills.advocacy.skill import get_bill_store
    store = get_bill_store()
    bid = store.track_bill(
        bill_number=data["bill_number"].strip(),
        title=data["title"].strip(),
        jurisdiction=data.get("jurisdiction", "WA"),
        status=data.get("status", "introduced"),
        our_position=data.get("our_position", "watch"),
        priority=data.get("priority", "medium"),
        sponsor=data.get("sponsor", ""),
        committee=data.get("committee", ""),
        summary=data.get("summary", ""),
    )
    if bid:
        return jsonify({"bill": store.get_bill(bid)}), 201
    return jsonify({"error": "Bill already tracked."}), 409


@app.route("/api/nonprofit/bills/<bill_id>", methods=["PUT"])
@require_auth
def api_nonprofit_bill_update(bill_id):
    """Update a tracked bill."""
    data = request.json or {}
    from familiar.skills.advocacy.skill import get_bill_store
    store = get_bill_store()
    result = store.update_bill(bill_id, data)
    if result:
        return jsonify({"bill": result})
    return jsonify({"error": "Bill not found."}), 404


@app.route("/api/nonprofit/bills/<bill_id>", methods=["DELETE"])
@require_auth
def api_nonprofit_bill_delete(bill_id):
    """Stop tracking a bill."""
    from familiar.skills.advocacy.skill import get_bill_store
    if get_bill_store().remove_bill(bill_id):
        return jsonify({"status": "removed"})
    return jsonify({"error": "Bill not found."}), 404


@app.route("/api/nonprofit/hearings")
@require_auth
def api_nonprofit_hearings():
    """List upcoming hearings across all bills."""
    days = int(request.args.get("days", 30))
    from familiar.skills.advocacy.skill import get_bill_store
    return jsonify({"hearings": get_bill_store().list_upcoming_hearings(days=days)})


@app.route("/api/nonprofit/bills/<bill_id>/testimony", methods=["POST"])
@require_auth
def api_nonprofit_bill_testimony(bill_id):
    """Record testimony for a bill."""
    data = request.json or {}
    if not data.get("testifier", "").strip():
        return jsonify({"error": "testifier is required."}), 400
    from familiar.skills.advocacy.skill import get_bill_store
    store = get_bill_store()
    if store.add_testimony(
        bill_id, data["testifier"].strip(),
        position=data.get("position", ""),
        format_type=data.get("format", "written"),
    ):
        return jsonify({"status": "recorded"}), 201
    return jsonify({"error": "Bill not found."}), 404


# ============================================================
# Stewardship API
# ============================================================


@app.route("/api/nonprofit/stewardship")
@require_auth
def api_nonprofit_stewardship():
    """Get stewardship queue — overdue + pending."""
    from familiar.skills.nonprofit.stewardship import get_stewardship_store
    store = get_stewardship_store()
    return jsonify({
        "overdue": store.get_overdue(),
        "pending": store.get_pending(),
        "completion_rate": store.completion_rate(),
    })


@app.route("/api/nonprofit/stewardship/<entry_id>/complete", methods=["POST"])
@require_auth
def api_nonprofit_stewardship_complete(entry_id):
    """Complete a stewardship action."""
    data = request.json or {}
    index = int(data.get("action_index", 0))
    from familiar.skills.nonprofit.stewardship import get_stewardship_store
    store = get_stewardship_store()
    if store.complete_action(entry_id, index):
        return jsonify({"status": "completed"})
    return jsonify({"error": "Entry or action not found."}), 404


# ============================================================
# Batch Communications API
# ============================================================


@app.route("/api/nonprofit/batches")
@require_auth
def api_nonprofit_batches():
    """List batch communications."""
    from familiar.skills.nonprofit.batch_comms import get_batch_store
    return jsonify({"batches": get_batch_store().list_batches()})


@app.route("/api/nonprofit/batches", methods=["POST"])
@require_auth
def api_nonprofit_batch_create():
    """Create a new batch communication."""
    data = request.json or {}
    if not data.get("name", "").strip() and not data.get("segment_key"):
        return jsonify({"error": "name or segment_key is required."}), 400
    from familiar.skills.nonprofit.batch_comms import get_batch_store
    store = get_batch_store()
    bid = store.create_batch(
        name=data.get("name", "").strip(),
        segment_key=data.get("segment_key", ""),
        segment_criteria=data.get("segment_criteria"),
        template_name=data.get("template_name", ""),
        channel=data.get("channel", "email"),
    )
    return jsonify({"batch": store.get_batch(bid)}), 201


@app.route("/api/nonprofit/batches/<batch_id>")
@require_auth
def api_nonprofit_batch_details(batch_id):
    """Get batch details."""
    from familiar.skills.nonprofit.batch_comms import get_batch_store
    batch = get_batch_store().get_batch(batch_id)
    if batch:
        return jsonify({"batch": batch})
    return jsonify({"error": "Batch not found."}), 404


def _send_generated_doc(result_str):
    """Parse a document skill result string and send the file as download."""
    import re

    # Skill functions return strings like "Receipt saved to /path/to/file.docx"
    # or "Document saved to /path/to/file.docx"
    match = re.search(r"(?:saved to|created at|generated at|wrote)\s+(.+\.(?:docx|pdf))",
                      result_str, re.IGNORECASE)
    if match:
        filepath = match.group(1).strip()
        if os.path.isfile(filepath):
            return send_file(
                filepath,
                as_attachment=True,
                download_name=os.path.basename(filepath),
            )

    # If result looks like an error or no file found, return the raw message
    return jsonify({"message": result_str, "error": "Could not locate generated file."}), 500


# ============================================================
# Smart Intake API
# ============================================================


@app.route("/api/intake/process", methods=["POST"])
@require_auth
def api_intake_process():
    """Process a file through smart intake pipeline."""
    from familiar.skills.intake.skill import smart_intake
    data = request.get_json(silent=True) or {}
    if not data.get("filepath"):
        return jsonify({"error": "filepath required"}), 400
    result = smart_intake(data)
    return jsonify({"result": result})


@app.route("/api/intake/history")
@require_auth
def api_intake_history():
    """Get intake history entries."""
    from familiar.skills.intake.history import get_intake_history
    history = get_intake_history()
    limit = request.args.get("limit", 50, type=int)
    content_type = request.args.get("content_type")
    entries = history.list_entries(limit=limit, content_type=content_type)
    return jsonify({"entries": entries, "count": len(entries)})


@app.route("/api/intake/stats")
@require_auth
def api_intake_stats():
    """Get intake statistics."""
    from familiar.skills.intake.history import get_intake_history
    history = get_intake_history()
    return jsonify(history.get_stats())


@app.route("/api/intake/batch", methods=["POST"])
@require_auth
def api_intake_batch():
    """Process multiple files through intake."""
    from familiar.skills.intake.skill import batch_intake
    data = request.get_json(silent=True) or {}
    result = batch_intake(data)
    return jsonify({"result": result})


@app.route("/api/intake/upload", methods=["POST"])
@require_auth
def api_intake_upload():
    """Upload a file and process through intake pipeline."""
    if "file" not in request.files:
        return jsonify({"error": "No file uploaded"}), 400

    file = request.files["file"]
    if not file.filename:
        return jsonify({"error": "Empty filename"}), 400

    # Validate file extension
    from familiar.skills.intake.skill import _ALL_SUPPORTED
    ext = Path(file.filename).suffix.lower()
    if ext not in _ALL_SUPPORTED:
        return jsonify({
            "error": f"Unsupported file type: {ext}",
            "supported": "Images (JPG/PNG/TIFF), PDF, DOCX, XLSX, CSV, TXT, MP3, WAV, M4A",
        }), 400

    # Check file size (50MB limit)
    file.seek(0, 2)
    size = file.tell()
    file.seek(0)
    if size > 50 * 1024 * 1024:
        return jsonify({"error": "File too large (50MB limit)"}), 413

    # Save to temp intake directory
    intake_dir = Path.home() / ".familiar" / "data" / "intake_uploads"
    intake_dir.mkdir(parents=True, exist_ok=True)

    import secrets as _secrets
    safe_name = f"{_secrets.token_hex(4)}_{file.filename}"
    save_path = intake_dir / safe_name
    file.save(str(save_path))

    # Process through intake
    from familiar.skills.intake.skill import smart_intake
    dry_run = request.form.get("dry_run", "false").lower() == "true"
    result = smart_intake({"filepath": str(save_path), "dry_run": dry_run})

    # Clean up uploaded file after processing (data is now in stores)
    if not dry_run:
        try:
            save_path.unlink()
        except OSError:
            pass

    return jsonify({"result": result, "filepath": str(save_path)})


# ============================================================
# Scribe endpoints
# ============================================================


@app.route("/api/scribe/inbox")
@require_auth
def api_scribe_inbox():
    """Get scribe inbox (recent intake entries)."""
    from familiar.skills.intake.history import get_intake_history
    history = get_intake_history()
    limit = request.args.get("limit", 50, type=int)
    entries = history.list_entries(limit=limit)
    return jsonify({"entries": entries, "count": len(entries)})


@app.route("/api/scribe/inbox/process", methods=["POST"])
@require_auth
def api_scribe_inbox_process():
    """Process a file through scribe intake."""
    from familiar.skills.intake.skill import smart_intake
    data = request.get_json(silent=True) or {}
    if not data.get("filepath"):
        return jsonify({"error": "filepath required"}), 400
    result = smart_intake(data)
    return jsonify({"result": result})


@app.route("/api/scribe/review")
@require_auth
def api_scribe_review():
    """Get review queue items."""
    from familiar.skills.intake.review import get_review_queue
    queue = get_review_queue()
    limit = request.args.get("limit", 50, type=int)
    pending = queue.list_pending(limit=limit)
    stats = queue.get_stats()
    return jsonify({"items": pending, "stats": stats})


@app.route("/api/scribe/review/<item_id>/approve", methods=["POST"])
@require_auth
def api_scribe_review_approve(item_id):
    """Approve a review item."""
    from familiar.skills.intake.review import get_review_queue, process_review_decision
    data = request.get_json(silent=True) or {}
    result = process_review_decision(
        item_id=item_id,
        action="approve",
        corrected_type=data.get("corrected_type"),
        corrected_fields=data.get("corrected_fields"),
    )
    return jsonify({"result": result})


@app.route("/api/scribe/review/<item_id>/reject", methods=["POST"])
@require_auth
def api_scribe_review_reject(item_id):
    """Reject a review item."""
    from familiar.skills.intake.review import get_review_queue, process_review_decision
    data = request.get_json(silent=True) or {}
    result = process_review_decision(
        item_id=item_id,
        action="reject",
        reason=data.get("reason", ""),
    )
    return jsonify({"result": result})


@app.route("/api/scribe/search")
@require_auth
def api_scribe_search():
    """Search intake history."""
    from familiar.skills.intake.history import get_intake_history
    history = get_intake_history()
    content_type = request.args.get("content_type")
    limit = request.args.get("limit", 50, type=int)
    entries = history.list_entries(limit=limit, content_type=content_type)
    return jsonify({"entries": entries, "count": len(entries)})


@app.route("/api/scribe/stats")
@require_auth
def api_scribe_stats():
    """Get scribe statistics (intake + review)."""
    from familiar.skills.intake.history import get_intake_history
    from familiar.skills.intake.review import get_review_queue
    history = get_intake_history()
    queue = get_review_queue()
    return jsonify({
        "intake": history.get_stats(),
        "review": queue.get_stats(),
    })


@app.route("/api/scribe/watches", methods=["GET"])
@require_auth
def api_scribe_watches_list():
    """List watch folders."""
    from familiar.skills.intake.watcher import get_folder_watcher
    watcher = get_folder_watcher()
    return jsonify({"watches": watcher.list_watches()})


@app.route("/api/scribe/watches", methods=["POST"])
@require_auth
def api_scribe_watches_add():
    """Add a watch folder."""
    from familiar.skills.intake.watcher import get_folder_watcher
    data = request.get_json(silent=True) or {}
    if not data.get("directory"):
        return jsonify({"error": "directory required"}), 400
    watcher = get_folder_watcher()
    try:
        watch_id = watcher.add_watch(
            directory=data["directory"],
            recursive=data.get("recursive", False),
            file_types=data.get("file_types"),
            label=data.get("label", ""),
        )
        return jsonify({"watch_id": watch_id})
    except ValueError as e:
        return jsonify({"error": str(e)}), 400


@app.route("/api/scribe/watches/<watch_id>", methods=["DELETE"])
@require_auth
def api_scribe_watches_delete(watch_id):
    """Remove a watch folder."""
    from familiar.skills.intake.watcher import get_folder_watcher
    watcher = get_folder_watcher()
    if watcher.remove_watch(watch_id):
        return jsonify({"removed": True})
    return jsonify({"error": "Watch not found"}), 404


@app.route("/api/scribe/clips")
@require_auth
def api_scribe_clips():
    """List web clips."""
    from familiar.skills.intake.clipper import get_web_clipper
    clipper = get_web_clipper()
    limit = request.args.get("limit", 50, type=int)
    return jsonify({"clips": clipper.list_clips(limit=limit)})


@app.route("/api/scribe/templates")
@require_auth
def api_scribe_templates():
    """List extraction templates."""
    from familiar.skills.intake.templates import get_template_store
    store = get_template_store()
    limit = request.args.get("limit", 50, type=int)
    return jsonify({"templates": store.list_templates(limit=limit)})


# ============================================================
# Helpers
# ============================================================


def _get_uptime():
    """Get system uptime."""
    try:
        with open("/proc/uptime", "r") as f:
            uptime_seconds = float(f.readline().split()[0])
            days = int(uptime_seconds // 86400)
            hours = int((uptime_seconds % 86400) // 3600)
            minutes = int((uptime_seconds % 3600) // 60)

            if days > 0:
                return f"{days}d {hours}h {minutes}m"
            elif hours > 0:
                return f"{hours}h {minutes}m"
            else:
                return f"{minutes}m"
    except (IOError, OSError, ValueError, IndexError):
        return "unknown"


# ============================================================
# Main
# ============================================================


_first_run_mode = False


def run_dashboard(agent=None, gateway=None, host="127.0.0.1", port=5000, debug=False,
                  first_run=False):
    """Run the web dashboard."""
    global _agent, _gateway, _first_run_mode
    _agent = agent
    _gateway = gateway
    _first_run_mode = first_run

    # Load ~/.familiar/.env so API keys persist across restarts
    _env_path = AGENT_DIR / ".env"
    if _env_path.exists():
        try:
            from dotenv import load_dotenv
            load_dotenv(str(_env_path), override=False)
        except ImportError:
            # Manual fallback if python-dotenv isn't installed
            for _line in _env_path.read_text().splitlines():
                _line = _line.strip()
                if _line and not _line.startswith("#") and "=" in _line:
                    _k, _v = _line.split("=", 1)
                    _k, _v = _k.strip(), _v.strip().strip("'\"")
                    if _k and _k not in os.environ:
                        os.environ[_k] = _v

    # Enforce secure permissions on credential files
    try:
        from familiar.core.paths import enforce_token_permissions
        enforce_token_permissions()
    except Exception:
        pass

    # Pre-authorize the dashboard user as OWNER so they get full capabilities
    # (web search, email, files, etc.) from the very first message.
    if agent and hasattr(agent, "sessions"):
        from familiar.core.security import TrustLevel, TRUST_CAPABILITIES
        session = agent.sessions.get_or_create_session("dashboard", "web")
        if session.trust_level != TrustLevel.OWNER:
            session.set_trust_level(TrustLevel.OWNER)
            session._explicit_grants.update(
                c.value for c in TRUST_CAPABILITIES.get(TrustLevel.OWNER, set())
            )
            session.user_role = "admin"
            logger.info("Dashboard user promoted to OWNER trust")
        # Apply budget settings from config.yaml (or generous default if not set).
        # budget_enabled=False (default) → unlimited; True → user-specified amount.
        _budget_enabled = False
        _budget_amount = 10.0
        try:
            import yaml as _yaml
            _cfg_path = CONFIG_FILE
            if _cfg_path.exists():
                with open(_cfg_path) as _f:
                    _cfg = _yaml.safe_load(_f) or {}
                _budget_enabled = _cfg.get("agent", {}).get("budget_enabled", False)
                _budget_amount = _cfg.get("agent", {}).get("daily_budget_amount", 10.0)
        except Exception:
            pass
        session.daily_budget = _budget_amount if _budget_enabled else 999999.0
        agent.sessions.save_session(session)

    # Wire scheduler delivery callback so briefings show in the dashboard
    if agent and hasattr(agent, "scheduler") and agent.scheduler:
        def _deliver_to_dashboard(message, channel, chat_id):
            _scheduled_messages.append({
                "timestamp": datetime.now().isoformat(),
                "message": message,
                "channel": channel or "internal",
            })
            # Persist briefing-type messages so they survive page refreshes
            if message and len(message) > 50:
                _save_briefing(message, source="scheduled")
            logger.info("Scheduled message queued for dashboard (%d chars)", len(message or ""))

        agent.scheduler.set_delivery_callback(_deliver_to_dashboard)
        logger.info("Dashboard delivery callback registered")

    # Create template and static dirs if needed
    TEMPLATES_DIR.mkdir(parents=True, exist_ok=True)
    STATIC_DIR.mkdir(parents=True, exist_ok=True)

    config_path = CONFIG_FILE
    display_host = "127.0.0.1" if host == "0.0.0.0" else host
    base_url = f"http://{display_host}:{port}"

    import webbrowser

    if first_run:
        # First run — no auth needed, clean URL
        url = base_url
        print("\n  Open in your browser to get started:")
        print(f"  {url}\n")
    else:
        # Normal run — resolve or generate a persistent dashboard key
        key = os.environ.get("FAMILIAR_DASHBOARD_KEY")
        if not key:
            # Try to load from .env
            env_path = AGENT_DIR / ".env"
            if env_path.exists():
                for line in env_path.read_text().splitlines():
                    if line.startswith("FAMILIAR_DASHBOARD_KEY="):
                        key = line.split("=", 1)[1].strip().strip("'\"")
                        break
        if not key:
            # Generate and persist so the key survives restarts
            import secrets as _secrets

            key = _secrets.token_urlsafe(32)
            env_path = AGENT_DIR / ".env"
            try:
                env_path.parent.mkdir(parents=True, exist_ok=True)
                with open(env_path, "a") as f:
                    f.write(f"\nFAMILIAR_DASHBOARD_KEY={key}\n")
            except OSError:
                pass
        os.environ["FAMILIAR_DASHBOARD_KEY"] = key
        require_auth._auto_key = key
        url = f"{base_url}?api_key={key}"
        print(f"\n  Dashboard: {url}\n")

    # Auto-open browser after a short delay to let Flask bind
    import threading as _thr

    _thr.Timer(1.5, webbrowser.open, args=[url]).start()

    app.run(host=host, port=port, debug=debug, threaded=True)


if __name__ == "__main__":
    run_dashboard(debug=False)
