# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# ORCID: 0009-0006-4957-0540
# Email: Georgescottfoley@proton.me
# Licensed under the MIT License - see LICENSE file for details

"""
Workspace data layer for the dashboard.

Resolves workspace configs into live data: loads JSON files,
computes metrics, paginates tables, groups pipelines.
No LLM round-trips — pure file reads and computation.
"""

from __future__ import annotations

import json
import logging
import math
import os
import threading
import time as _time
from datetime import date, datetime
from pathlib import Path
from typing import Any, Optional

logger = logging.getLogger(__name__)


# ---------------------------------------------------------------------------
# Background-refresh cache for slow data sources (email, calendar, server)
# ---------------------------------------------------------------------------

_source_cache: dict[str, dict] = {}
_source_cache_ts: dict[str, float] = {}
_source_cache_lock = threading.Lock()
_SOURCE_TTL = 60  # seconds — serve stale data rather than block
_DISK_CACHE_DIR = Path.home() / ".familiar" / "data" / "cache"


def _disk_cache_save(key: str, data: dict):
    """Persist source data to disk so restarts don't lose state."""
    try:
        import json as _json

        _DISK_CACHE_DIR.mkdir(parents=True, exist_ok=True)
        path = _DISK_CACHE_DIR / f"ws_{key}.json"
        path.write_text(_json.dumps(data, default=str))
    except Exception:
        pass  # best-effort


def _disk_cache_load(key: str) -> dict | None:
    """Load cached source data from disk."""
    try:
        import json as _json

        path = _DISK_CACHE_DIR / f"ws_{key}.json"
        if path.exists():
            return _json.loads(path.read_text())
    except Exception:
        pass
    return None


def _disk_cache_warm():
    """On startup, pre-fill in-memory cache from disk so first paint is instant."""
    try:
        if not _DISK_CACHE_DIR.exists():
            return
        import json as _json

        for path in _DISK_CACHE_DIR.glob("ws_*.json"):
            key = path.stem[3:]  # strip "ws_" prefix
            try:
                data = _json.loads(path.read_text())
                with _source_cache_lock:
                    if key not in _source_cache:
                        _source_cache[key] = data
                        _source_cache_ts[key] = 0  # mark stale so bg refresh happens
            except Exception:
                pass
    except Exception:
        pass


_disk_cache_warm()

# Sources that do network I/O and should never block the UI thread
_SLOW_SOURCES = frozenset(
    {
        "server",
        "wazuh",
        "crowdsec",
        "spiderfoot",
        "netdata",
        "uptime_kuma",
        "dns",
        "network",
        "system",
        "providers",
    }
)


def _cached_source_get(key: str, loader):
    """Return cached data for slow sources; trigger background refresh if stale."""
    now = _time.monotonic()
    with _source_cache_lock:
        cached = _source_cache.get(key)
        ts = _source_cache_ts.get(key, 0)
        stale = (now - ts) > _SOURCE_TTL

    if cached is not None and not stale:
        return cached

    # If we have stale data, return it immediately and refresh in background
    if cached is not None:
        _bg_refresh(key, loader)
        return cached

    # First call ever — do a quick synchronous load with a short timeout
    # to avoid blocking the UI for 30+ seconds
    result = _bg_refresh_sync(key, loader)
    return result


def _bg_refresh(key: str, loader):
    """Kick off a background thread to refresh a slow source."""

    def _run():
        try:
            data = loader()
        except Exception as e:
            logger.debug("Background refresh %s failed: %s", key, e)
            data = {}
        with _source_cache_lock:
            _source_cache[key] = data
            _source_cache_ts[key] = _time.monotonic()
        if data:
            _disk_cache_save(key, data)

    t = threading.Thread(target=_run, daemon=True, name=f"ws-refresh-{key}")
    t.start()


def _bg_refresh_sync(key: str, loader):
    """First-ever load: run in a thread with a 3s timeout, return empty on timeout."""
    result_holder: list[dict] = [{}]

    def _run():
        try:
            result_holder[0] = loader()
        except Exception as e:
            logger.debug("Initial load %s failed: %s", key, e)

    t = threading.Thread(target=_run, daemon=True, name=f"ws-init-{key}")
    t.start()
    t.join(timeout=3.0)  # Wait at most 3 seconds

    data = result_holder[0]
    with _source_cache_lock:
        _source_cache[key] = data
        _source_cache_ts[key] = _time.monotonic()
    if data:
        _disk_cache_save(key, data)
    if t.is_alive():
        logger.debug("Source %s still loading after 3s — returning empty", key)
    return data


def _get_data_dir() -> Path:
    try:
        from familiar.core.paths import DATA_DIR

        return DATA_DIR
    except ImportError:
        return Path.home() / ".familiar" / "data"


# ---------------------------------------------------------------------------
# Data source registry
# ---------------------------------------------------------------------------


def _load_json(filename: str) -> dict:
    """Load a JSON file from the data directory."""
    path = _get_data_dir() / filename
    if not path.exists():
        return {}
    try:
        with open(path, encoding="utf-8") as f:
            return json.load(f)
    except (json.JSONDecodeError, IOError, OSError) as e:
        logger.warning("Failed to load %s: %s", filename, e)
        return {}


_WORKSPACE_SOURCES: dict[str, Any] = {
    "nonprofit": lambda: _load_json("nonprofit.json"),
    "bookkeeping": lambda: _load_json("bookkeeping.json"),
    "tasks": lambda: _load_tasks_data(),
    "contacts": lambda: _load_json("contacts.json"),
    "cases": lambda: _load_cases_data(),
    "research": lambda: _load_json("research.json"),
    "kitchen": lambda: _load_kitchen_data(),
    "studio": lambda: _load_studio_data(),
    # Security sources — live API calls, gracefully degrade when not configured
    "wazuh": lambda: _load_wazuh_data(),
    "crowdsec": lambda: _load_crowdsec_data(),
    "osint": lambda: _load_osint_data(),
    "spiderfoot": lambda: _load_spiderfoot_data(),
    # Network Admin sources — live API calls, gracefully degrade when not configured
    "netdata": lambda: _load_netdata_data(),
    "uptime_kuma": lambda: _load_uptime_kuma_data(),
    "network": lambda: _load_network_scan_data(),
    "dns": lambda: _load_dns_data(),
    # Digital Broker — analytics data from analytics.db
    "analytics": lambda: _load_analytics_data(),
    # System health — live host metrics for Helper Command Center
    "system": lambda: _load_system_health_data(),
    # Server infrastructure — runtime + service readiness
    "server": lambda: _load_server_readiness(),
    # AI provider health — live checks for all configured providers
    "providers": lambda: _load_provider_health_data(),
    # Grants.gov search — starts empty, populated via search endpoint
    "grants_gov": lambda: {"results": []},
    # Email pipeline — pending events + action queue from email triage
    "pending_events": lambda: _load_pending_events(),
    "action_queue": lambda: _load_action_queue(),
    # Event management
    "events": lambda: _load_events_data(),
    # Advocacy & legislative tracking
    "advocacy": lambda: _load_advocacy_data(),
    # Legislation search (companion to advocacy — empty by default, populated via search modal)
    "legislation": lambda: {"results": []},
    # Stewardship tracking
    "stewardship": lambda: _load_stewardship_data(),
    # Batch communications
    "batch_comms": lambda: _load_batch_comms_data(),
    # Smart Intake
    "intake": lambda: _load_intake_data(),
    # Scribe records office
    "scribe": lambda: _load_scribe_data(),
}


def _load_wazuh_data() -> dict:
    """Load Security Analyst workspace data from Wazuh API. Degrades gracefully."""
    empty = {
        "alerts": [],
        "vulnerabilities": [],
        "compliance": [],
        "threat_timeline": [],
        "agents": [],
        "decisions": [],
        "alerts.critical_count": 0,
        "alerts.high_count": 0,
        "agents.active_count": 0,
        "vulnerabilities.open_count": 0,
        "compliance.cis_pass_percent": 0,
    }
    try:
        from familiar.skills.wazuh.skill import _api_get, _indexer_search, _wazuh_available

        if not _wazuh_available():
            return empty

        # Fetch alert counts by level
        counts_body = {
            "size": 0,
            "query": {"match_all": {}},
            "aggs": {
                "by_level": {
                    "range": {
                        "field": "rule.level",
                        "ranges": [
                            {"key": "critical", "from": 13},
                            {"key": "high", "from": 10, "to": 13},
                            {"key": "medium", "from": 7, "to": 10},
                            {"key": "low", "from": 1, "to": 7},
                        ],
                    }
                }
            },
        }
        counts_result = _indexer_search("wazuh-alerts-*", counts_body)
        buckets = counts_result.get("aggregations", {}).get("by_level", {}).get("buckets", [])
        level_counts = {b.get("key", ""): b.get("doc_count", 0) for b in buckets}

        # Fetch last 20 alerts for the feed
        alerts_body = {
            "size": 20,
            "sort": [{"timestamp": {"order": "desc"}}],
            "query": {"range": {"rule.level": {"gte": 7}}},
            "_source": ["timestamp", "agent.name", "rule.level", "rule.description", "rule.id"],
        }
        alerts_result = _indexer_search("wazuh-alerts-*", alerts_body)
        raw_alerts = alerts_result.get("hits", {}).get("hits", [])
        alerts = []
        for hit in raw_alerts:
            src = hit.get("_source", {})
            level = src.get("rule", {}).get("level", 0)
            sev = "critical" if level >= 13 else "high" if level >= 10 else "medium"
            alerts.append(
                {
                    "rule_description": src.get("rule", {}).get("description", ""),
                    "agent_name": src.get("agent", {}).get("name", ""),
                    "timestamp": src.get("timestamp", ""),
                    "severity": sev,
                    "rule_id": src.get("rule", {}).get("id", ""),
                }
            )

        # Fetch agent summary
        agents_resp = _api_get("/agents", params={"limit": 50})
        agents = []
        if agents_resp.status_code == 200:
            raw_agents = agents_resp.json().get("data", {}).get("affected_items", [])
            active_count = sum(1 for a in raw_agents if a.get("status") == "active")
            agents = raw_agents
        else:
            active_count = 0

        return {
            "alerts": alerts,
            "vulnerabilities": [],
            "compliance": [],
            "threat_timeline": alerts[:7],
            "agents": agents,
            "alerts.critical_count": level_counts.get("critical", 0),
            "alerts.high_count": level_counts.get("high", 0),
            "agents.active_count": active_count,
            "vulnerabilities.open_count": 0,
            "compliance.cis_pass_percent": 0,
        }
    except Exception as e:
        logger.debug("Wazuh workspace load failed: %s", e)
        return empty


def _load_crowdsec_data() -> dict:
    """Load CrowdSec decisions for the Security Analyst workspace. Degrades gracefully."""
    empty = {"decisions": [], "decisions.ban_count": 0}
    try:
        from familiar.skills.crowdsec.skill import _api_get as _cs_get
        from familiar.skills.crowdsec.skill import _crowdsec_available

        if not _crowdsec_available():
            return empty

        resp = _cs_get("/v1/decisions", params={"limit": 50})
        if resp.status_code == 404:
            return {"decisions": [], "decisions.ban_count": 0}
        if resp.status_code != 200:
            return empty

        decisions = resp.json() or []
        ban_count = sum(1 for d in decisions if d.get("type") == "ban")
        rows = [
            {
                "value": d.get("value", ""),
                "type": d.get("type", ""),
                "reason": d.get("scenario", d.get("reason", "")),
                "origin": d.get("origin", ""),
                "duration": d.get("duration", ""),
            }
            for d in decisions
        ]
        return {"decisions": rows, "decisions.ban_count": ban_count}
    except Exception as e:
        logger.debug("CrowdSec workspace load failed: %s", e)
        return empty


def _load_osint_data() -> dict:
    """Load OSINT authorization log, investigations, and entity findings."""
    empty = {
        "authorizations": [],
        "investigations": [],
        "entity_findings": [],
        "authorizations.active_count": 0,
        "findings.total_count": 0,
        "investigations.active_count": 0,
    }
    try:
        import datetime as _dt

        from familiar.skills.osint.auth import list_authorizations, purge_expired

        purge_expired()
        auths = list_authorizations(include_expired=False)

        auth_rows = []
        for a in auths:
            expires = _dt.datetime.fromtimestamp(a["expires_at"], tz=_dt.timezone.utc).strftime(
                "%Y-%m-%dT%H:%M:%SZ"
            )
            auth_rows.append(
                {
                    "target": a["target"],
                    "justification": a["justification"],
                    "scope": a["scope"],
                    "created_at": _dt.datetime.fromtimestamp(
                        a["created_at"], tz=_dt.timezone.utc
                    ).strftime("%Y-%m-%dT%H:%M:%SZ"),
                    "expires_at": expires,
                    "status": "active",
                }
            )

        # Live investigations and findings from SQLite persistence
        investigations = []
        entity_findings = []
        total_findings = 0
        active_inv_count = 0
        try:
            from familiar.skills.osint.investigations import (
                get_recent_findings,
                list_investigations,
            )

            investigations = list_investigations()
            entity_findings = get_recent_findings(limit=20)
            active_inv_count = sum(
                1 for inv in investigations if inv.get("status") in ("open", "active")
            )
            total_findings = sum(inv.get("finding_count", 0) for inv in investigations)
        except Exception as inv_e:
            logger.debug("OSINT investigations load failed: %s", inv_e)

        return {
            "authorizations": auth_rows,
            "investigations": investigations,
            "entity_findings": entity_findings,
            "authorizations.active_count": len(auth_rows),
            "findings.total_count": total_findings,
            "investigations.active_count": active_inv_count,
        }
    except Exception as e:
        logger.debug("OSINT workspace load failed: %s", e)
        return empty


def _load_spiderfoot_data() -> dict:
    """Load SpiderFoot scan list for the OSINT workspace spiderfoot_scans section."""
    empty = {
        "scans": [],
        "scans.total": 0,
        "scans.running": 0,
    }
    try:
        from familiar.skills.spiderfoot.skill import list_spiderfoot_scans

        result = list_spiderfoot_scans()
        if result.get("format") == "degraded":
            return empty
        return {
            "scans": result.get("scans", []),
            "scans.total": result.get("total", 0),
            "scans.running": result.get("running", 0),
        }
    except Exception as e:
        logger.debug("SpiderFoot workspace load failed: %s", e)
        return empty


def _load_netdata_data() -> dict:
    """Load Network Admin workspace data from Netdata API.

    Falls back to live /proc metrics when Netdata isn't deployed,
    so the NOC shows useful host data immediately.
    """
    empty = {
        "hosts": [],
        "alarms": [],
        "hosts.monitored_count": 0,
        "alarms.active_count": 0,
        "system.cpu_avg_percent": 0.0,
        "system.ram_avg_percent": 0.0,
    }

    # Try Netdata API first
    try:
        from familiar.skills.netdata.skill import (
            _netdata_available,
            get_host_metrics,
            get_system_alarms,
        )

        if _netdata_available():
            metrics = get_host_metrics()
            # Check format AND that we got real data (not all zeros from connection failure)
            has_real_data = metrics.get("format") != "degraded" and (
                metrics.get("uptime_seconds", 0) > 0 or metrics.get("disk_total_gb", 0) > 0
            )
            if has_real_data:
                alarms_data = get_system_alarms()
                active_alarms = alarms_data.get("active", [])

                host_row = {
                    "hostname": metrics.get("hostname", "localhost"),
                    "cpu_pct": metrics.get("cpu_pct", 0.0),
                    "ram_pct": metrics.get("ram_pct", 0.0),
                    "disk_pct": metrics.get("disk_pct", 0.0),
                    "disk_used_gb": metrics.get("disk_used_gb", 0.0),
                    "disk_total_gb": metrics.get("disk_total_gb", 0.0),
                    "net_rx_mbps": metrics.get("net_rx_mbps", 0.0),
                    "net_tx_mbps": metrics.get("net_tx_mbps", 0.0),
                    "uptime_days": round(metrics.get("uptime_seconds", 0) / 86400, 1),
                    "alarms": alarms_data.get("count", 0),
                }

                return {
                    "hosts": [host_row],
                    "alarms": active_alarms,
                    "hosts.monitored_count": 1,
                    "alarms.active_count": alarms_data.get("count", 0),
                    "system.cpu_avg_percent": metrics.get("cpu_pct", 0.0),
                    "system.ram_avg_percent": metrics.get("ram_pct", 0.0),
                }
    except Exception as e:
        logger.debug("Netdata API unavailable: %s", e)

    # Fallback: read /proc directly for local host metrics
    return _load_netdata_from_proc()


def _load_netdata_from_proc() -> dict:
    """Read host metrics directly from /proc when Netdata isn't deployed."""
    import platform
    import shutil

    hostname = platform.node() or "localhost"
    cpu_pct = 0.0
    ram_pct = 0.0
    disk_pct = 0.0
    disk_used_gb = 0.0
    disk_total_gb = 0.0
    uptime_days = 0.0
    net_rx_mbps = 0.0
    net_tx_mbps = 0.0

    # CPU from /proc/loadavg
    try:
        with open("/proc/loadavg", encoding="utf-8") as f:
            load1 = float(f.read().split()[0])
        try:
            cores = len(os.sched_getaffinity(0))
        except Exception:
            cores = os.cpu_count() or 1
        cpu_pct = round(min(load1 / cores * 100, 100.0), 1)
    except Exception:
        pass

    # Memory from /proc/meminfo
    try:
        with open("/proc/meminfo", encoding="utf-8") as f:
            mi: dict[str, int] = {}
            for line in f:
                parts = line.split()
                if len(parts) >= 2:
                    mi[parts[0].rstrip(":")] = int(parts[1])
        total = mi.get("MemTotal", 0)
        avail = mi.get("MemAvailable", mi.get("MemFree", 0))
        if total:
            ram_pct = round((1 - avail / total) * 100, 1)
    except Exception:
        pass

    # Disk
    try:
        usage = shutil.disk_usage("/")
        disk_pct = round(usage.used / usage.total * 100, 1)
        disk_used_gb = round(usage.used / (1024**3), 1)
        disk_total_gb = round(usage.total / (1024**3), 1)
    except Exception:
        pass

    # Uptime
    try:
        with open("/proc/uptime", encoding="utf-8") as f:
            secs = float(f.read().split()[0])
        uptime_days = round(secs / 86400, 1)
    except Exception:
        pass

    # Network throughput snapshot from /proc/net/dev
    try:
        with open("/proc/net/dev", encoding="utf-8") as f:
            for line in f:
                parts = line.strip().split()
                if not parts or not parts[0].endswith(":"):
                    continue
                iface = parts[0].rstrip(":")
                if iface == "lo":
                    continue
                # bytes received = col 1, bytes transmitted = col 9
                rx_bytes = int(parts[1])
                tx_bytes = int(parts[9])
                if rx_bytes > 0 or tx_bytes > 0:
                    # Cumulative since boot, in GB for readability
                    net_rx_mbps = round(rx_bytes / (1024**3), 1)
                    net_tx_mbps = round(tx_bytes / (1024**3), 1)
                    break  # Use first non-loopback interface
    except Exception:
        pass

    host_row = {
        "hostname": hostname,
        "cpu_pct": cpu_pct,
        "ram_pct": ram_pct,
        "disk_pct": disk_pct,
        "disk_used_gb": disk_used_gb,
        "disk_total_gb": disk_total_gb,
        "net_rx_mbps": net_rx_mbps,
        "net_tx_mbps": net_tx_mbps,
        "uptime_days": uptime_days,
        "alarms": 0,
    }

    return {
        "hosts": [host_row],
        "alarms": [],
        "hosts.monitored_count": 1,
        "alarms.active_count": 0,
        "system.cpu_avg_percent": cpu_pct,
        "system.ram_avg_percent": ram_pct,
    }


def _load_uptime_kuma_data() -> dict:
    """Load service uptime data from Uptime Kuma API.

    Falls back to Familiar's own container status when Uptime Kuma
    isn't deployed, so the NOC shows all managed services immediately.
    """
    empty = {
        "monitors": [],
        "monitors.up_count": 0,
        "monitors.down_count": 0,
        "monitors.total": 0,
    }

    # Try Uptime Kuma API first
    try:
        from familiar.skills.uptime_kuma.skill import (
            _uptime_kuma_available,
            get_monitor_status,
        )

        if _uptime_kuma_available():
            data = get_monitor_status()
            if data.get("format") != "degraded":
                monitors = data.get("monitors", [])
                # If Uptime Kuma is running but has no monitors configured,
                # fall back to container status so the NOC isn't empty
                if monitors:
                    return {
                        "monitors": monitors,
                        "monitors.up_count": data.get("up_count", 0),
                        "monitors.down_count": data.get("down_count", 0),
                        "monitors.total": data.get("total", 0),
                    }
    except Exception as e:
        logger.debug("Uptime Kuma API unavailable: %s", e)

    # Fallback: use Familiar's own service manager to list container status.
    # This also handles the case where Uptime Kuma is running but has
    # no monitors configured yet (fresh install).
    return _load_uptime_from_containers()


def _load_uptime_from_containers() -> dict:
    """Build service monitor list from Familiar's managed containers."""
    empty = {
        "monitors": [],
        "monitors.up_count": 0,
        "monitors.down_count": 0,
        "monitors.total": 0,
    }
    try:
        from familiar.services.manager import get_service_manager
        from familiar.services.specs import SERVICE_SPECS, ServiceType

        mgr = get_service_manager()
        rt_info = mgr.get_runtime_info()
        if rt_info.get("runtime", "none") == "none":
            return empty

        statuses = mgr.get_all_statuses()
        monitors = []
        up = 0
        down = 0

        for s in statuses:
            key = s.get("key", "")
            spec = SERVICE_SPECS.get(key)
            if not spec or spec.service_type == ServiceType.NATIVE:
                continue  # Skip native services — not containers

            status_str = s.get("status", "not_installed")
            if status_str == "not_installed":
                continue  # Don't show services that were never deployed

            is_up = status_str == "running"
            if is_up:
                up += 1
            else:
                down += 1

            # Build a monitor row matching Uptime Kuma's schema
            url = s.get("url", "")
            response_ms = 0
            if is_up and spec.health_check_port and spec.health_check_path:
                # Quick health ping for response time
                try:
                    import time as _time
                    import urllib.request

                    hurl = f"http://localhost:{spec.health_check_port}{spec.health_check_path}"
                    t0 = _time.monotonic()
                    urllib.request.urlopen(hurl, timeout=3)
                    response_ms = round((_time.monotonic() - t0) * 1000)
                except Exception:
                    pass
            monitors.append(
                {
                    "name": spec.display_name,
                    "service_key": key,
                    "status": "up" if is_up else "down",
                    "uptime_24h": 100.0 if is_up else 0.0,
                    "uptime_30d": 100.0 if is_up else 0.0,
                    "avg_response_ms": response_ms,
                    "last_check": datetime.now().isoformat(),
                    "url": url,
                    "type": "container",
                }
            )

        return {
            "monitors": monitors,
            "monitors.up_count": up,
            "monitors.down_count": down,
            "monitors.total": up + down,
        }
    except Exception as e:
        logger.debug("Container status fallback failed: %s", e)
        return empty


def _load_network_scan_data() -> dict:
    """Load cached network scan results. Populated by scan_subnet() tool calls."""
    # Network scan data is written to disk by the network skill after each scan
    # so the workspace can display the last known state without re-scanning.
    empty = {"hosts": [], "last_scan": None, "hosts_up": 0}
    try:
        data_dir = _get_data_dir() / "network"
        scan_file = data_dir / "last_scan.json"
        if not scan_file.exists():
            return empty
        with open(scan_file, encoding="utf-8") as f:
            cached = json.load(f)
        return {
            "hosts": cached.get("hosts", []),
            "last_scan": cached.get("timestamp"),
            "hosts_up": cached.get("hosts_up", 0),
        }
    except Exception as e:
        logger.debug("Network scan cache load failed: %s", e)
        return empty


def _load_dns_data() -> dict:
    """Load DNS health check results from cache. Populated by dns_lookup() tool calls."""
    empty = {"checks": [], "last_check": None}
    try:
        data_dir = _get_data_dir() / "network"
        dns_file = data_dir / "dns_checks.json"
        if not dns_file.exists():
            return empty
        with open(dns_file, encoding="utf-8") as f:
            cached = json.load(f)
        return {
            "checks": cached.get("checks", []),
            "last_check": cached.get("timestamp"),
        }
    except Exception as e:
        logger.debug("DNS cache load failed: %s", e)
        return empty


def _load_provider_health_data() -> dict:
    """Load AI provider health and usage data for NOC workspace."""
    providers = []
    totals = {"ok": 0, "error": 0, "not_configured": 0}

    # Check each cloud provider
    _checks = [
        (
            "anthropic",
            "ANTHROPIC_API_KEY",
            "Anthropic (Claude)",
            "https://api.anthropic.com/v1/messages",
            "https://console.anthropic.com/settings/plans",
        ),
        (
            "openai",
            "OPENAI_API_KEY",
            "OpenAI (GPT)",
            "https://api.openai.com/v1/models",
            "https://platform.openai.com/account/billing",
        ),
        (
            "gemini",
            "GEMINI_API_KEY",
            "Google (Gemini)",
            None,
            "https://aistudio.google.com/billing",
        ),
    ]

    for key, env_var, display_name, url, billing_url in _checks:
        api_key = os.environ.get(env_var, "")
        if not api_key:
            providers.append(
                {
                    "name": display_name,
                    "key": key,
                    "status": "not_configured",
                    "message": "No API key",
                    "billing_url": billing_url,
                }
            )
            totals["not_configured"] += 1
            continue

        status = "ok"
        message = "Connected"
        try:
            import urllib.error
            import urllib.request

            if key == "anthropic":
                req = urllib.request.Request(
                    url,
                    method="POST",
                    headers={
                        "x-api-key": api_key,
                        "anthropic-version": "2023-06-01",
                        "content-type": "application/json",
                    },
                    data=json.dumps(
                        {
                            "model": "claude-haiku-4-5-20251001",
                            "max_tokens": 1,
                            "messages": [{"role": "user", "content": "ping"}],
                        }
                    ).encode(),
                )
            elif key == "openai":
                req = urllib.request.Request(
                    url,
                    headers={"Authorization": f"Bearer {api_key}"},
                )
            else:  # gemini
                req = urllib.request.Request(
                    f"https://generativelanguage.googleapis.com/v1beta/models?key={api_key}",
                )

            with urllib.request.urlopen(req, timeout=10) as resp:
                resp.read()
                status = "ok"
                message = "Connected"
        except urllib.error.HTTPError as e:
            body = e.read().decode("utf-8", errors="replace").lower()
            if any(
                kw in body
                for kw in [
                    "credit",
                    "insufficient",
                    "billing",
                    "payment required",
                ]
            ):
                status = "billing"
                message = "Insufficient credits"
            elif e.code == 401:
                status = "auth_error"
                message = "Invalid API key"
            else:
                # 2xx equivalent — some providers return errors for minimal requests
                # but the key is valid if we got past auth
                status = "ok"
                message = "Connected"
        except Exception as e:
            status = "unreachable"
            message = str(e)[:60]

        providers.append(
            {
                "name": display_name,
                "key": key,
                "status": status,
                "message": message,
                "billing_url": billing_url,
            }
        )
        totals["ok" if status == "ok" else "error"] += 1

    # Check Ollama
    ollama_status = "unreachable"
    ollama_msg = "Not running"
    ollama_models = 0
    try:
        import urllib.request

        req = urllib.request.Request("http://localhost:11434/api/tags", method="GET")
        with urllib.request.urlopen(req, timeout=3) as resp:
            data = json.loads(resp.read().decode())
            ollama_models = len(data.get("models", []))
            ollama_status = "ok"
            ollama_msg = f"{ollama_models} models installed"
    except Exception:
        pass

    providers.insert(
        0,
        {
            "name": "Ollama (Local)",
            "key": "ollama",
            "status": ollama_status,
            "message": ollama_msg,
            "billing_url": "",
            "models": ollama_models,
        },
    )
    if ollama_status == "ok":
        totals["ok"] += 1
    else:
        totals["error"] += 1

    # Load usage stats from analytics.db
    usage_rows = []
    try:
        from familiar.core.analytics import get_analytics_store

        store = get_analytics_store()
        # Get per-model usage from last 7 days
        import sqlite3

        conn = sqlite3.connect(str(store.db_path))
        rows = conn.execute(
            "SELECT model, COUNT(*) as calls, SUM(tokens_in) as t_in, "
            "SUM(tokens_out) as t_out, SUM(cost_usd) as cost "
            "FROM messages WHERE timestamp > datetime('now', '-7 days') "
            "GROUP BY model ORDER BY calls DESC"
        ).fetchall()
        conn.close()
        for row in rows:
            usage_rows.append(
                {
                    "model": row[0] or "unknown",
                    "calls_7d": row[1],
                    "tokens_in_7d": row[2] or 0,
                    "tokens_out_7d": row[3] or 0,
                    "cost_7d": round(row[4] or 0, 4),
                }
            )
    except Exception as e:
        logger.debug("Analytics load failed: %s", e)

    return {
        "providers": providers,
        "usage": usage_rows,
        "providers.ok_count": totals["ok"],
        "providers.error_count": totals["error"],
        "providers.total_count": totals["ok"] + totals["error"] + totals["not_configured"],
    }


def _load_system_health_data() -> dict:
    """Load live system health metrics for Helper Command Center."""
    import shutil

    health = []
    disk_pct = 0.0

    # CPU
    try:
        with open("/proc/loadavg", encoding="utf-8") as f:
            load1 = f.read().split()[0]
        try:
            cores = len(os.sched_getaffinity(0))
        except Exception:
            cores = os.cpu_count() or 1
        load_pct = round(float(load1) / cores * 100, 1)
        status = "ok" if load_pct < 80 else ("warning" if load_pct < 95 else "critical")
        health.append({"metric": "CPU Load", "value": f"{load1} ({load_pct}%)", "status": status})
    except Exception:
        pass

    # Memory
    try:
        with open("/proc/meminfo", encoding="utf-8") as f:
            mi = {}
            for line in f:
                parts = line.split()
                if len(parts) >= 2:
                    mi[parts[0].rstrip(":")] = int(parts[1])
        total = mi.get("MemTotal", 0)
        avail = mi.get("MemAvailable", mi.get("MemFree", 0))
        if total:
            used_pct = round((1 - avail / total) * 100, 1)
            used_gb = round((total - avail) / 1048576, 1)
            total_gb = round(total / 1048576, 1)
            status = "ok" if used_pct < 80 else ("warning" if used_pct < 95 else "critical")
            health.append(
                {
                    "metric": "Memory",
                    "value": f"{used_gb}/{total_gb} GB ({used_pct}%)",
                    "status": status,
                }
            )
    except Exception:
        pass

    # Disk
    try:
        usage = shutil.disk_usage("/")
        disk_pct = round(usage.used / usage.total * 100, 1)
        total_gb = round(usage.total / (1024**3), 1)
        used_gb = round(usage.used / (1024**3), 1)
        status = "ok" if disk_pct < 80 else ("warning" if disk_pct < 90 else "critical")
        health.append(
            {
                "metric": "Disk (/)",
                "value": f"{used_gb}/{total_gb} GB ({disk_pct}%)",
                "status": status,
            }
        )
    except Exception:
        pass

    # Uptime
    try:
        with open("/proc/uptime", encoding="utf-8") as f:
            secs = int(float(f.read().split()[0]))
        days, rem = divmod(secs, 86400)
        hours, rem = divmod(rem, 3600)
        mins = rem // 60
        health.append({"metric": "Uptime", "value": f"{days}d {hours}h {mins}m", "status": "ok"})
    except Exception:
        pass

    return {
        "health": health,
        "disk.usage_percent": disk_pct,
    }


def _check_service_credentials() -> dict[str, bool]:
    """Check which services have credentials configured.

    Returns a dict mapping connect_key -> bool for services that have
    a connect_key in their ServiceSpec. Only checks env-var-based
    credentials relevant to self-hosted services.
    """
    import os
    from pathlib import Path

    data_dir = Path.home() / ".familiar" / "data"

    has_joplin = bool(os.environ.get("JOPLIN_TOKEN"))
    has_nextcloud = bool(os.environ.get("NEXTCLOUD_URL") and os.environ.get("NEXTCLOUD_TOKEN"))
    has_gitea = bool(os.environ.get("GITEA_URL") and os.environ.get("GITEA_TOKEN"))
    has_jellyfin = bool(os.environ.get("JELLYFIN_URL") and os.environ.get("JELLYFIN_API_KEY"))
    has_pihole = bool(os.environ.get("PIHOLE_URL") and os.environ.get("PIHOLE_API_KEY"))
    try:
        from familiar.skills.email.accounts import get_all_accounts

        has_email = bool(get_all_accounts())
    except ImportError:
        has_email = bool(os.environ.get("EMAIL_ADDRESS"))

    return {
        "mealie": bool(os.environ.get("MEALIE_URL") and os.environ.get("MEALIE_TOKEN")),
        "joplin": has_joplin,
        "nextcloud": has_nextcloud,
        "gitea": has_gitea,
        "jellyfin": has_jellyfin,
        "pihole": has_pihole,
        "email": has_email,
        "searxng": bool(os.environ.get("SEARXNG_URL")),
        "uptime_kuma": bool(os.environ.get("UPTIME_KUMA_URL")),
        "mastodon": bool(
            os.environ.get("MASTODON_ACCESS_TOKEN") and os.environ.get("MASTODON_INSTANCE_URL")
        ),
        "bluesky": bool(
            os.environ.get("BLUESKY_HANDLE") and os.environ.get("BLUESKY_APP_PASSWORD")
        ),
    }


def _load_server_readiness() -> dict:
    """Check server infrastructure readiness for workspace CTA and status cards."""
    try:
        from familiar.services.manager import get_service_manager

        mgr = get_service_manager()
        rt = mgr.detect_runtime()
        rt_info = mgr.get_runtime_info()
        statuses = mgr.get_all_statuses()

        running = [s for s in statuses if s.get("status") == "running"]
        has_runtime = rt_info.get("runtime", "none") != "none"

        # Check which services have credentials configured
        credential_status: dict[str, bool] = {}
        try:
            credential_status = _check_service_credentials()
        except Exception:
            pass

        # Map service spec connect_key for lookup
        connect_keys: dict[str, str] = {}
        try:
            from familiar.services.specs import SERVICE_SPECS

            for skey, spec in SERVICE_SPECS.items():
                if spec.connect_key:
                    connect_keys[skey] = spec.connect_key
        except Exception:
            pass

        # Build per-service status lookup for workspace cards
        services: dict[str, dict] = {}
        for s in statuses:
            key = s.get("key", "")
            if key:
                ck = connect_keys.get(key, "")
                services[key] = {
                    "status": s.get("status", "not_installed"),
                    "url": s.get("url", ""),
                    "display_name": s.get("display_name", key),
                    "icon": s.get("icon", "fa-server"),
                    "health_check_port": s.get("health_check_port"),
                    "service_type": s.get("service_type", "container"),
                    "connect_key": ck,
                    "configured": credential_status.get(ck, False) if ck else False,
                }

        return {
            "has_runtime": has_runtime,
            "runtime_name": rt_info.get("runtime", "none"),
            "running_count": len(running),
            "total_count": len(statuses),
            "needs_setup": not has_runtime or len(running) == 0,
            "services": services,
        }
    except Exception as e:
        logger.debug("Server readiness check failed: %s", e)
        return {
            "has_runtime": False,
            "runtime_name": "none",
            "running_count": 0,
            "total_count": 0,
            "needs_setup": True,
            "services": {},
        }


def _load_events_data() -> dict:
    """Load event management data for workspace."""
    try:
        from familiar.skills.events.skill import get_event_store

        store = get_event_store()
        events = store.list_events()
        upcoming = store.count_upcoming()
        total_budget = sum(e.get("budget_total", 0) for e in events)
        return {
            "events": events,
            "events.upcoming_count": upcoming,
            "events.total_count": len(events),
            "events.total_budget": total_budget,
        }
    except Exception as e:
        logger.debug("Events load failed: %s", e)
        return {
            "events": [],
            "events.upcoming_count": 0,
            "events.total_count": 0,
            "events.total_budget": 0,
        }


def _load_advocacy_data() -> dict:
    """Load advocacy/legislative tracking data for workspace."""
    try:
        from familiar.skills.advocacy.skill import get_bill_store

        store = get_bill_store()
        bills = store.list_bills()
        active = store.count_active()
        hearings = store.count_hearings(days=14)
        critical = sum(
            1
            for b in bills
            if b.get("priority") == "critical"
            and b.get("status") not in ("signed", "vetoed", "dead")
        )
        return {
            "bills": bills,
            "advocacy.active_count": active,
            "advocacy.hearing_count": hearings,
            "advocacy.critical_count": critical,
        }
    except Exception as e:
        logger.debug("Advocacy load failed: %s", e)
        return {
            "bills": [],
            "advocacy.active_count": 0,
            "advocacy.hearing_count": 0,
            "advocacy.critical_count": 0,
        }


def _load_stewardship_data() -> dict:
    """Load stewardship tracking data for workspace."""
    try:
        from familiar.skills.nonprofit.stewardship import get_stewardship_store

        store = get_stewardship_store()
        overdue = store.get_overdue()
        pending = store.get_pending(days=7)
        rate = store.completion_rate()
        # Flatten for table display
        rows = []
        for entry in overdue + pending:
            actions = entry.get("overdue_actions", entry.get("pending_actions", []))
            for a in actions:
                rows.append(
                    {
                        "entry_id": entry["entry_id"],
                        "donor_name": entry["donor_name"],
                        "trigger_amount": entry.get("trigger_amount", 0),
                        "action_type": a["type"],
                        "action_label": a.get("label", a["type"]),
                        "due_date": a["due_date"],
                        "status": a["status"],
                        "overdue": entry in overdue,
                    }
                )
        return {
            "stewardship": rows,
            "stewardship.overdue_count": len(overdue),
            "stewardship.pending_count": len(pending),
            "stewardship.completion_rate": rate,
        }
    except Exception as e:
        logger.debug("Stewardship load failed: %s", e)
        return {
            "stewardship": [],
            "stewardship.overdue_count": 0,
            "stewardship.pending_count": 0,
            "stewardship.completion_rate": 0,
        }


def _load_batch_comms_data() -> dict:
    """Load batch communications data for workspace."""
    try:
        from familiar.skills.nonprofit.batch_comms import get_batch_store

        store = get_batch_store()
        batches = store.list_batches()
        active = sum(1 for b in batches if b["status"] not in ("sent", "cancelled"))
        total_sent = sum(b.get("sent_count", 0) for b in batches)
        return {
            "batches": batches,
            "batch_comms.active_count": active,
            "batch_comms.total_sent": total_sent,
        }
    except Exception as e:
        logger.debug("Batch comms load failed: %s", e)
        return {
            "batches": [],
            "batch_comms.active_count": 0,
            "batch_comms.total_sent": 0,
        }


def _load_intake_data() -> dict:
    """Load Smart Intake history data for workspace."""
    try:
        from familiar.skills.intake.history import get_intake_history

        history = get_intake_history()
        stats = history.get_stats()
        entries = history.list_entries(limit=10)
        return {
            "intake_entries": entries,
            "intake.total_count": stats.get("total", 0),
            "intake.success_rate": stats.get("success_rate", 0.0),
            "intake.by_type": stats.get("by_type", {}),
        }
    except Exception as e:
        logger.debug("Intake load failed: %s", e)
        return {
            "intake_entries": [],
            "intake.total_count": 0,
            "intake.success_rate": 0.0,
            "intake.by_type": {},
        }


def _load_scribe_data() -> dict:
    """Load Scribe workspace data — review queue, intake history, watches, clips."""
    result = {
        "inbox_entries": [],
        "recent_entries": [],
        "review_items": [],
        "search_results": [],
        "transcription_entries": [],
        "stats_breakdown": [],
        "inbox_count": 0,
        "review_count": 0,
        "processed_today": 0,
        "total_processed": 0,
        "accuracy_pct": 100.0,
        "watch_count": 0,
    }
    try:
        from familiar.skills.intake.history import get_intake_history

        history = get_intake_history()
        stats = history.get_stats()
        entries = history.list_entries(limit=50)

        result["inbox_entries"] = entries[:20]
        result["recent_entries"] = entries[:10]
        result["inbox_count"] = stats.get("total", 0)
        result["total_processed"] = stats.get("total", 0)

        # Transcriptions are audio entries
        result["transcription_entries"] = [
            e for e in entries if e.get("content_type") == "audio_transcript"
        ][:20]

        # Processed today
        from datetime import date as _date

        today_str = _date.today().isoformat()
        result["processed_today"] = sum(
            1 for e in entries if e.get("timestamp", "").startswith(today_str)
        )

        # Stats breakdown by type
        by_type = stats.get("by_type", {})
        result["stats_breakdown"] = [
            {"content_type": ct, "count": count, "success_rate": "—"}
            for ct, count in sorted(by_type.items(), key=lambda x: -x[1])
        ]
    except Exception as e:
        logger.debug("Scribe intake data load failed: %s", e)

    # Review queue
    try:
        from familiar.skills.intake.review import get_review_queue

        queue = get_review_queue()
        pending = queue.list_pending(limit=50)
        queue_stats = queue.get_stats()
        result["review_items"] = pending
        result["review_count"] = queue_stats.get("pending", 0)
        result["accuracy_pct"] = queue_stats.get("accuracy_pct", 100.0)
    except Exception as e:
        logger.debug("Scribe review queue load failed: %s", e)

    # Watch folders
    try:
        from familiar.skills.intake.watcher import get_folder_watcher

        watcher = get_folder_watcher()
        watches = watcher.list_watches()
        result["watch_count"] = len(watches)
    except Exception as e:
        logger.debug("Scribe watcher load failed: %s", e)

    return result


def _load_pending_events() -> dict:
    """Load pending calendar events from email triage pipeline."""
    try:
        from familiar.skills.email.pending_events import get_pending_event_store

        store = get_pending_event_store()
        pending = store.get_pending()
        return {
            "events": pending,
            "pending_events.count": len(pending),
        }
    except Exception as e:
        logger.debug("Pending events load failed: %s", e)
        return {"events": [], "pending_events.count": 0}


def _load_action_queue() -> dict:
    """Load email follow-up action queue from triage pipeline."""
    try:
        from familiar.skills.email.action_queue import get_action_store

        store = get_action_store()
        pending = store.get_pending()
        by_type = store.summary_by_type()
        return {
            "actions": pending,
            "action_queue.count": len(pending),
            "action_queue.by_type": by_type,
        }
    except Exception as e:
        logger.debug("Action queue load failed: %s", e)
        return {"actions": [], "action_queue.count": 0, "action_queue.by_type": {}}


def _load_analytics_data() -> dict:
    """Load analytics data for Digital Broker workspace.

    Pulls from analytics.db via AnalyticsStore: KPIs, monthly spend,
    recent call log, and per-skill costs.
    """
    empty = {
        "today_cost": 0,
        "month_cost": 0,
        "projected_cost": 0,
        "today_calls": 0,
        "month_tokens": 0,
        "proactive_pct": 0,
        "interactive_cost": 0,
        "proactive_cost": 0,
        "daily_avg": 0,
        "last_month_total": 0,
        "calls": [],
        "skill_costs": [],
    }
    try:
        from familiar.core.analytics import get_analytics_store

        store = get_analytics_store()
        kpis = store.get_period_kpis()
        spend = store.get_monthly_spend()
        calls, _ = store.get_call_log(page=1, per_page=50)

        today = kpis.get("today", {})
        month = kpis.get("month", {})
        total = spend.get("total", 0)
        proactive = spend.get("proactive_cost", 0)
        proactive_pct = (proactive / total * 100) if total > 0 else 0

        # Per-skill cost breakdown from daily_stats
        skill_costs = []
        try:
            from datetime import timedelta as _td
            from datetime import timezone as _tz

            start = datetime.now(_tz.utc) - _td(days=30)
            summary = store.get_summary(start=start)
            for skill_name, info in summary.by_skill.items():
                if skill_name:
                    skill_costs.append(
                        {
                            "skill": skill_name,
                            "calls": info.get("calls", 0),
                            "cost": round(info.get("cost", 0), 6),
                            "avg_latency": round(info.get("avg_latency", 0), 0),
                        }
                    )
            skill_costs.sort(key=lambda s: s["cost"], reverse=True)
        except Exception:
            pass

        return {
            "today_cost": today.get("cost", 0),
            "month_cost": month.get("cost", 0),
            "projected_cost": spend.get("projected", 0),
            "today_calls": today.get("calls", 0),
            "month_tokens": (month.get("tokens_in", 0) + month.get("tokens_out", 0)),
            "proactive_pct": round(proactive_pct, 1),
            "interactive_cost": spend.get("interactive_cost", 0),
            "proactive_cost": proactive,
            "daily_avg": spend.get("daily_avg", 0),
            "last_month_total": spend.get("last_month_total", 0),
            "calls": calls,
            "skill_costs": skill_costs,
        }
    except Exception as e:
        logger.debug("Analytics data load failed: %s", e)
        return empty


def _load_tasks_data() -> dict:
    """Load tasks.json (which is an array) and wrap in a dict."""
    path = _get_data_dir() / "tasks.json"
    if not path.exists():
        return {"tasks": [], "pending_count": 0, "overdue_count": 0}
    try:
        with open(path, encoding="utf-8") as f:
            tasks = json.load(f)
        if isinstance(tasks, list):
            today = date.today().isoformat()
            pending = [t for t in tasks if not t.get("completed")]
            overdue = [t for t in pending if t.get("due_date") and t["due_date"] < today]
            return {
                "tasks": pending,
                "pending_count": len(pending),
                "overdue_count": len(overdue),
            }
        return {"tasks": [], "pending_count": 0, "overdue_count": 0}
    except (json.JSONDecodeError, IOError, OSError) as e:
        logger.warning("Failed to load tasks.json: %s", e)
        return {"tasks": [], "pending_count": 0, "overdue_count": 0}


def _load_cases_data() -> dict:
    """Load cases.json for the Social Worker workspace."""
    data = _load_json("cases.json")
    if not data:
        return {
            "cases": [],
            "resources": [],
            "notes": [],
            "crisis_log": [],
            "crisis_week_count": 0,
        }

    crisis_log = data.get("crisis_log", [])

    # Compute crisis-this-week count
    crisis_week_count = 0
    try:
        from datetime import timedelta

        week_ago = (date.today() - timedelta(days=7)).isoformat()
        crisis_week_count = sum(1 for c in crisis_log if c.get("date", "") >= week_ago)
    except Exception:
        pass

    data["crisis_week_count"] = crisis_week_count
    return data


def _load_kitchen_data() -> dict:
    """Load kitchen.json for the Chef workspace with computed metrics."""
    data = _load_json("kitchen.json")
    if not data:
        return {
            "recipes": [],
            "menus": [],
            "vendors": [],
            "inventory": [],
            "schedule": [],
            "costs": [],
            "food_cost_pct": 0,
            "labor_cost_pct": 0,
            "revenue_mtd": 0,
            "food_cost_mtd": 0,
            "labor_cost_mtd": 0,
            "avg_plate_cost": 0,
        }

    costs = data.get("costs", [])
    current_month = date.today().strftime("%Y-%m")
    mtd = [c for c in costs if str(c.get("date", "")).startswith(current_month)]

    revenue = sum(c.get("amount", 0) for c in mtd if c.get("type") == "revenue")
    food = sum(c.get("amount", 0) for c in mtd if c.get("type") == "food_cost")
    labor = sum(c.get("amount", 0) for c in mtd if c.get("type") == "labor_cost")

    data["revenue_mtd"] = revenue
    data["food_cost_mtd"] = food
    data["labor_cost_mtd"] = labor
    data["food_cost_pct"] = round((food / revenue * 100) if revenue else 0, 1)
    data["labor_cost_pct"] = round((labor / revenue * 100) if revenue else 0, 1)

    # Average plate cost from recipes
    recipes = data.get("recipes", [])
    costs_per = [r.get("cost_per_serving", 0) for r in recipes if r.get("cost_per_serving")]
    data["avg_plate_cost"] = round(sum(costs_per) / len(costs_per), 2) if costs_per else 0

    return data


def _load_studio_data() -> dict:
    """Load studio.json for the Artist workspace with computed metrics."""
    data = _load_json("studio.json")
    if not data:
        return {
            "pieces": [],
            "commissions": [],
            "shows": [],
            "materials": [],
            "clients": [],
            "revenue_mtd": 0,
        }

    # Revenue this month from sold pieces
    pieces = data.get("pieces", [])
    current_month = date.today().strftime("%Y-%m")
    data["revenue_mtd"] = sum(
        p.get("price", 0)
        for p in pieces
        if p.get("status") == "sold" and str(p.get("sold_date", "")).startswith(current_month)
    )

    return data


# ---------------------------------------------------------------------------
# Metric resolver
# ---------------------------------------------------------------------------


def resolve_metric(source_data: dict, metric_path: str) -> Any:
    """Resolve a dot-path metric from source data.

    Supported patterns:
        donors.count            -> len(data["donors"])
        grants.active_count     -> count where status in (applied, awarded)
        grants.deadline_count   -> count grants with upcoming deadlines
        donors.needs_thanks_count -> count donors needing thank-you
        ytd_income / ytd_expense / ytd_net -> sum transactions for current year
        donor_revenue / grant_revenue -> sum by category
        transaction_count       -> count of transactions
        pending_count / overdue_count -> task filtering
        today_count             -> calendar event count
        unread_count / urgent_count / action_count / fyi_count -> email counts
    """
    # Direct key match (flat metrics like pending_count, today_count)
    if metric_path in source_data:
        return source_data[metric_path]

    parts = metric_path.split(".")
    if len(parts) == 2:
        collection_key, agg = parts
        collection = source_data.get(collection_key, [])
        if not isinstance(collection, list):
            return 0

        if agg == "count":
            return len(collection)

        if agg == "active_count":
            # Grants: applied/awarded. Cases: assessment/active/follow_up.
            return sum(
                1
                for item in collection
                if item.get("status") in ("applied", "awarded", "assessment", "active", "follow_up")
            )

        if agg == "intake_count":
            return sum(1 for item in collection if item.get("status") == "intake")

        if agg == "followup_count":
            return sum(1 for item in collection if item.get("status") == "follow_up")

        # Kitchen: menus with status "active"
        if collection_key == "menus" and agg == "active_count":
            return sum(1 for item in collection if item.get("status") == "active")

        # Kitchen: inventory items below par level
        if agg == "low_stock_count":
            return sum(
                1
                for item in collection
                if item.get("quantity") is not None
                and item.get("par_level" if "par_level" in item else "reorder_at") is not None
                and item.get("quantity", 0) <= item.get("par_level", item.get("reorder_at", 0))
            )

        # Studio: pieces in active work stages
        if agg == "in_progress_count":
            return sum(
                1
                for item in collection
                if item.get("status")
                in ("concept", "in_progress", "drying", "bisque_fire", "glazing", "glaze_fire")
            )

        # Studio: open commissions (not completed/cancelled)
        if agg == "open_count":
            return sum(
                1
                for item in collection
                if item.get("status") not in ("completed", "cancelled", "delivered")
            )

        # Studio: upcoming shows (date >= today)
        if agg == "upcoming_count":
            today = date.today().isoformat()
            return sum(
                1
                for item in collection
                if item.get("date", "") >= today and item.get("status") not in ("cancelled",)
            )

        if agg == "deadline_count":
            today = date.today().isoformat()
            future_30 = None
            try:
                from datetime import timedelta

                future_30 = (date.today() + timedelta(days=30)).isoformat()
            except Exception:
                pass
            return sum(
                1
                for item in collection
                if item.get("deadline")
                and item.get("deadline") >= today
                and (future_30 is None or item.get("deadline") <= future_30)
                and item.get("status") not in ("declined", "completed")
            )

        if agg == "needs_thanks_count":
            count = 0
            for donor in collection:
                interactions = donor.get("interactions", [])
                # Find gifts not followed by a thank-you
                last_gift_idx = -1
                last_thanks_idx = -1
                for i, ix in enumerate(interactions):
                    if ix.get("type") == "gift":
                        last_gift_idx = i
                    if ix.get("type") == "thank_you":
                        last_thanks_idx = i
                if last_gift_idx >= 0 and last_gift_idx > last_thanks_idx:
                    count += 1
            return count

    # Bookkeeping metrics
    transactions = source_data.get("transactions", [])
    current_year = str(date.today().year)

    if metric_path == "ytd_income":
        return sum(
            t.get("amount", 0)
            for t in transactions
            if t.get("type") == "income" and str(t.get("date", "")).startswith(current_year)
        )

    if metric_path == "ytd_expense":
        return sum(
            t.get("amount", 0)
            for t in transactions
            if t.get("type") == "expense" and str(t.get("date", "")).startswith(current_year)
        )

    if metric_path == "ytd_net":
        income = resolve_metric(source_data, "ytd_income")
        expense = resolve_metric(source_data, "ytd_expense")
        return income - expense

    if metric_path == "donor_revenue":
        return sum(
            t.get("amount", 0)
            for t in transactions
            if t.get("type") == "income"
            and t.get("category") in ("individual_donations", "donations", "donor")
            and str(t.get("date", "")).startswith(current_year)
        )

    if metric_path == "grant_revenue":
        return sum(
            t.get("amount", 0)
            for t in transactions
            if t.get("type") == "income"
            and t.get("category") in ("grants", "grant")
            and str(t.get("date", "")).startswith(current_year)
        )

    if metric_path == "transaction_count":
        return len(transactions)

    return 0


# ---------------------------------------------------------------------------
# Section data fetcher
# ---------------------------------------------------------------------------


def get_section_data(
    section_config: dict,
    sort: Optional[str] = None,
    sort_dir: str = "asc",
    page: int = 1,
    limit: int = 50,
) -> dict:
    """Load and shape section data based on type (table, pipeline, summary)."""
    source_key = section_config.get("source", "")
    loader = _WORKSPACE_SOURCES.get(source_key)
    if not loader:
        return {"error": f"Unknown source: {source_key}"}

    try:
        if source_key in _SLOW_SOURCES:
            source_data = _cached_source_get(source_key, loader)
        else:
            source_data = loader()
    except Exception as e:
        logger.warning("Source %s load failed: %s", source_key, e)
        return {"error": str(e)}

    section_type = section_config.get("type", "table")

    if section_type == "table":
        result = _build_table(section_config, source_data, sort, sort_dir, page, limit)
    elif section_type == "pipeline":
        result = _build_pipeline(section_config, source_data)
    elif section_type == "summary":
        result = _build_summary(section_config, source_data)
    else:
        return {"error": f"Unknown section type: {section_type}"}

    # Inject service status when section declares a service_key
    svc_key = section_config.get("service_key")
    if svc_key:
        try:
            server_data = _WORKSPACE_SOURCES["server"]()
            svc_info = server_data.get("services", {}).get(svc_key, {})
            result["service_status"] = {
                "key": svc_key,
                "status": svc_info.get("status", "not_installed"),
                "url": svc_info.get("url", ""),
                "display_name": svc_info.get("display_name", svc_key),
            }
        except Exception:
            pass

    return result


def _build_table(
    config: dict,
    source_data: dict,
    sort: Optional[str],
    sort_dir: str,
    page: int,
    limit: int,
) -> dict:
    """Extract, sort, and paginate table rows."""
    data_key = config.get("source_key", "")
    rows = source_data.get(data_key, [])
    if not isinstance(rows, list):
        rows = []

    # Enrich nonprofit donor rows with computed fields
    if data_key == "donors":
        for row in rows:
            interactions = row.get("interactions", [])
            row["interactions_count"] = len(interactions)
            # Compute date field from created_at for notes display
            if not row.get("date") and row.get("created_at"):
                row["date"] = row["created_at"][:10]

    # Enrich notes rows with date from created_at
    if data_key == "notes":
        for row in rows:
            if not row.get("date") and row.get("created_at"):
                row["date"] = row["created_at"][:10]

    total = len(rows)

    # Sort
    sort_field = sort or config.get("default_sort", "")
    if sort_field:
        reverse = (sort_dir or config.get("default_sort_dir", "asc")) == "desc"
        rows = sorted(rows, key=lambda r: r.get(sort_field) or "", reverse=reverse)

    # Paginate
    pages = max(1, math.ceil(total / limit))
    page = max(1, min(page, pages))
    start = (page - 1) * limit
    rows = rows[start : start + limit]

    return {"rows": rows, "total": total, "page": page, "pages": pages}


def _build_pipeline(config: dict, source_data: dict) -> dict:
    """Group items by stage_field into pipeline columns."""
    data_key = config.get("source_key", "")
    items = source_data.get(data_key, [])
    if not isinstance(items, list):
        items = []

    stage_field = config.get("stage_field", "status")
    stages_order = config.get("stages", [])

    grouped: dict[str, list] = {s: [] for s in stages_order}
    for item in items:
        stage = item.get(stage_field, "")
        if stage in grouped:
            grouped[stage].append(item)
        elif stages_order:
            grouped.setdefault(stage, []).append(item)

    totals = {s: len(v) for s, v in grouped.items()}
    return {"stages": grouped, "totals": totals}


def _build_summary(config: dict, source_data: dict) -> dict:
    """Compute summary stats from config."""
    stats = []
    for stat_config in config.get("stats", []):
        metric = stat_config.get("metric", "")
        value = resolve_metric(source_data, metric)
        stats.append(
            {
                "key": stat_config.get("key", ""),
                "label": stat_config.get("label", ""),
                "icon": stat_config.get("icon", ""),
                "color": stat_config.get("color", ""),
                "value": value,
                "format": stat_config.get("format", "number"),
            }
        )
    return {"stats": stats}


# ---------------------------------------------------------------------------
# Overview card resolver
# ---------------------------------------------------------------------------


def resolve_overview_cards(overview_config: dict) -> list[dict]:
    """Resolve all overview card values.

    Supports two card types:
    - **stat** (default): Displays a metric value (number, percent, etc.)
    - **cta**: Call-to-action banner with message + button. Only shown when
      its ``show_when`` metric resolves to a truthy value. Used for onboarding
      prompts like "No server detected — set up now".
    """
    cards = []
    # Cache source data to avoid re-loading
    source_cache: dict[str, dict] = {}

    for card in overview_config.get("cards", []):
        source_key = card.get("source", "")
        if source_key not in source_cache:
            loader = _WORKSPACE_SOURCES.get(source_key)
            if loader:
                try:
                    if source_key in _SLOW_SOURCES:
                        source_cache[source_key] = _cached_source_get(source_key, loader)
                    else:
                        source_cache[source_key] = loader()
                except Exception as e:
                    logger.debug("Source %s failed for overview: %s", source_key, e)
                    source_cache[source_key] = {}
            else:
                source_cache[source_key] = {}

        card_type = card.get("type", "stat")

        # CTA cards only appear when their condition is met
        if card_type == "cta":
            show_metric = card.get("show_when", "")
            source = source_cache.get(source_key, {})
            cta_svc_keys = card.get("service_keys", [])
            cta_message = card.get("message", "")
            if show_metric and cta_svc_keys and source.get("has_runtime"):
                # Job-class-aware: show CTA when any of THIS job's
                # container services aren't running yet (not the global flag).
                # Skip native/systemd services — they can't be auto-provisioned.
                svc_map = source.get("services", {})
                missing = [
                    svc_map.get(sk, {}).get("display_name", sk)
                    for sk in cta_svc_keys
                    if svc_map.get(sk, {}).get("status") != "running"
                    and svc_map.get(sk, {}).get("service_type") != "native"
                ]
                if not missing:
                    continue  # All this job's services are up — hide CTA
                # Rewrite message to list what still needs deploying
                names = ", ".join(missing[:-1]) + (
                    f" and {missing[-1]}" if len(missing) > 1 else missing[0]
                )
                cta_message = (
                    f"Your server is ready. Deploy {names} to complete your"
                    f" setup — one click, fully automated."
                )
            elif show_metric:
                show_val = resolve_metric(source, show_metric)
                if not show_val:
                    continue  # Condition not met — hide this card
            cta_card: dict[str, Any] = {
                "key": card.get("key", ""),
                "type": "cta",
                "label": card.get("label", ""),
                "icon": card.get("icon", ""),
                "color": card.get("color", ""),
                "message": cta_message,
                "button_label": card.get("button_label", "Set Up"),
                "button_action": card.get("button_action", ""),
            }
            if cta_svc_keys:
                cta_card["service_keys"] = cta_svc_keys
            cards.append(cta_card)
            continue

        # Service status cards — show live status badges for specific services
        if card_type == "service_status":
            svc_keys = card.get("service_keys", [])
            source = source_cache.get(source_key, {})
            svc_map = source.get("services", {})
            svc_items = []
            any_running = False
            for sk in svc_keys:
                info = svc_map.get(sk, {})
                status = info.get("status", "not_installed")
                if status == "running":
                    any_running = True
                svc_items.append(
                    {
                        "key": sk,
                        "display_name": info.get("display_name", sk),
                        "icon": info.get("icon", "fa-server"),
                        "status": status,
                        "url": info.get("url", ""),
                        "service_type": info.get("service_type", "container"),
                        "connect_key": info.get("connect_key", ""),
                        "configured": info.get("configured", False),
                    }
                )
            # Only show when at least one service has been provisioned
            show_metric = card.get("show_when", "")
            if show_metric:
                show_val = resolve_metric(source, show_metric)
                if not show_val:
                    continue
            # Hide if no services exist yet (CTA handles that case)
            if not any_running and all(s["status"] == "not_installed" for s in svc_items):
                continue
            cards.append(
                {
                    "key": card.get("key", ""),
                    "type": "service_status",
                    "label": card.get("label", "Infrastructure"),
                    "icon": card.get("icon", "fa-server"),
                    "color": card.get("color", ""),
                    "services": svc_items,
                }
            )
            continue

        # Client-resolved cards — value fetched in browser from existing API
        client_api = card.get("client_api")
        if client_api:
            cards.append(
                {
                    "key": card.get("key", ""),
                    "label": card.get("label", ""),
                    "icon": card.get("icon", ""),
                    "color": card.get("color", ""),
                    "value": "…",
                    "format": card.get("format", "number"),
                    "client_api": client_api,
                    "client_metric": card.get("client_metric", ""),
                }
            )
            continue

        value = resolve_metric(source_cache[source_key], card.get("metric", ""))
        cards.append(
            {
                "key": card.get("key", ""),
                "label": card.get("label", ""),
                "icon": card.get("icon", ""),
                "color": card.get("color", ""),
                "value": value,
                "format": card.get("format", "number"),
            }
        )
    return cards


# ---------------------------------------------------------------------------
# Smart alerts for nonprofit workspace
# ---------------------------------------------------------------------------


def get_nonprofit_alerts() -> list[dict]:
    """Generate actionable alerts for the nonprofit workspace overview.

    Returns a list of alert dicts with keys:
        level (info|warn|urgent), icon, message, action_label, action_fn
    """
    alerts: list[dict] = []
    today_str = date.today().isoformat()

    try:
        np_data = _load_json("nonprofit.json")
    except Exception:
        np_data = {}

    donors = np_data.get("donors", [])
    grants = np_data.get("grants", [])

    # Donors needing thank-you
    unthanked = []
    for donor in donors:
        interactions = donor.get("interactions", [])
        last_gift_idx = -1
        last_thanks_idx = -1
        for i, ix in enumerate(interactions):
            if ix.get("type") == "gift":
                last_gift_idx = i
            if ix.get("type") == "thank_you":
                last_thanks_idx = i
        if last_gift_idx >= 0 and last_gift_idx > last_thanks_idx:
            unthanked.append(donor.get("name", "Unknown"))

    if unthanked:
        names = ", ".join(unthanked[:3])
        extra = f" and {len(unthanked) - 3} more" if len(unthanked) > 3 else ""
        alerts.append(
            {
                "level": "warn",
                "icon": "fa-envelope-open-text",
                "message": f"{len(unthanked)} donor(s) need thank-you letters: {names}{extra}",
                "action_label": "Generate Letters",
                "action_fn": "npShowThankYouModal()",
            }
        )

    # Grant deadlines in next 14 days
    try:
        from datetime import timedelta

        deadline_cutoff = (date.today() + timedelta(days=14)).isoformat()
    except Exception:
        deadline_cutoff = None

    upcoming_deadlines = []
    for g in grants:
        dl = g.get("deadline", "")
        if (
            dl
            and dl >= today_str
            and (deadline_cutoff is None or dl <= deadline_cutoff)
            and g.get("status") not in ("declined", "completed")
        ):
            upcoming_deadlines.append(g)

    for g in upcoming_deadlines:
        days_left = 0
        try:
            d = date.fromisoformat(g["deadline"])
            days_left = (d - date.today()).days
        except Exception:
            pass
        level = "urgent" if days_left <= 3 else "warn"
        alerts.append(
            {
                "level": level,
                "icon": "fa-clock",
                "message": (
                    f'Grant "{g.get("name", "Unnamed")}" deadline in {days_left} day(s)'
                    f" ({g.get('deadline', '')})"
                ),
                "action_label": "View Grants",
                "action_fn": "switchWorkspaceTab('grants')",
            }
        )

    # Grant reports overdue
    for g in grants:
        rd = g.get("report_due", "")
        if rd and rd < today_str and g.get("status") not in ("declined", "completed"):
            alerts.append(
                {
                    "level": "urgent",
                    "icon": "fa-exclamation-triangle",
                    "message": (
                        f'Grant report overdue for "{g.get("name", "Unnamed")}" (was due {rd})'
                    ),
                    "action_label": "View Grants",
                    "action_fn": "switchWorkspaceTab('grants')",
                }
            )

    # No recent transactions (warn if no bookkeeping activity in 30+ days)
    try:
        bk_data = _load_json("bookkeeping.json")
        txns = bk_data.get("transactions", [])
        if txns:
            from datetime import timedelta

            cutoff_30 = (date.today() - timedelta(days=30)).isoformat()
            recent = [t for t in txns if t.get("date", "") >= cutoff_30]
            if not recent:
                alerts.append(
                    {
                        "level": "info",
                        "icon": "fa-chart-pie",
                        "message": "No transactions logged in the last 30 days.",
                        "action_label": "Log Transaction",
                        "action_fn": "npShowDonationModal()",
                    }
                )
    except Exception:
        pass

    # Getting-started guidance when workspace is empty
    if not donors and not grants and not alerts:
        alerts.extend(
            [
                {
                    "level": "info",
                    "icon": "fa-rocket",
                    "message": (
                        "Welcome to Nonprofit HQ! Start by adding your first"
                        " donor or searching for federal grants to fund your mission."
                    ),
                    "action_label": "Add First Donor",
                    "action_fn": "npShowDonorModal()",
                },
                {
                    "level": "info",
                    "icon": "fa-magnifying-glass-dollar",
                    "message": (
                        "Find federal funding on Grants.gov — search by keyword"
                        " and add opportunities directly to your grant pipeline."
                    ),
                    "action_label": "Search Grants",
                    "action_fn": "npShowGrantSearchModal()",
                },
            ]
        )

    return alerts


# ---------------------------------------------------------------------------
# Action executor
# ---------------------------------------------------------------------------


def execute_action(action_config: dict, row_data: dict, agent: Any = None) -> dict:
    """Execute a workspace action (tool call or agent chat).

    Returns {"result": str, "success": bool}.
    """
    mode = action_config.get("mode", "agent")

    if mode == "tool":
        return _execute_tool_action(action_config, row_data)
    elif mode == "agent":
        return _execute_agent_action(action_config, row_data, agent)
    else:
        return {"result": f"Unknown action mode: {mode}", "success": False}


def _execute_tool_action(action_config: dict, row_data: dict) -> dict:
    """Call a tool handler directly via the tool registry."""
    tool_name = action_config.get("tool", "")
    tool_args = dict(action_config.get("tool_args", {}))
    # Merge row_data into tool_args
    tool_args.update(row_data)

    try:
        from familiar.core.tool_registry import get_tool

        tool_def = get_tool(tool_name)
        if tool_def and tool_def.handler:
            result = tool_def.handler(tool_args)
            return {"result": str(result), "success": True}
        return {"result": f"Tool '{tool_name}' not found", "success": False}
    except Exception as e:
        logger.warning("Tool action %s failed: %s", tool_name, e)
        return {"result": str(e), "success": False}


def _execute_agent_action(action_config: dict, row_data: dict, agent: Any) -> dict:
    """Route through agent.chat_with_results()."""
    prompt = action_config.get("prompt", "")
    template = action_config.get("prompt_template", "")

    if template and row_data:
        try:
            prompt = template.format(**row_data)
        except KeyError:
            prompt = template

    if not prompt:
        return {"result": "No prompt provided", "success": False}

    if not agent:
        return {"result": prompt, "success": True}

    try:
        response = agent.chat_with_results(prompt, user_id="dashboard", channel="workspace")
        return {"result": response.text, "success": True}
    except Exception as e:
        logger.warning("Agent action failed: %s", e)
        return {"result": str(e), "success": False}
