#!/usr/bin/env python3
# Copyright (c) 2026 PULSE Contributors. Proprietary. All rights reserved.
"""
Pipeline Health Monitor — Symposium Phase 6

Near-real-time health aggregation (60s) complementing brain_health.py (6h).
Aggregates: daemon PIDs, scheduler heartbeats, worker heartbeats, capture log
growth, evidence freshness, task board state, and brain health alerts.

Public API:
  send_heartbeat(name, status, metrics) — scheduler/daemon reports liveness
  check_pipeline_health() — unified report → state/pipeline_health.json
"""
import json
import logging
import os
import sys
import time
from datetime import datetime, timezone
from pathlib import Path

SCRIPT_DIR = Path(__file__).resolve().parent
ROOT_DIR = SCRIPT_DIR.parent.parent.parent

if str(ROOT_DIR) not in sys.path:
    sys.path.insert(0, str(ROOT_DIR))

from pulse.core.brain_utils import (
    STATE_DIR, HIPPOCAMPUS_DIR, TASK_BOARD_FILE, is_process_running,
)

logger = logging.getLogger("pulse.pipeline_health")

HEALTH_DIR = STATE_DIR / "health"
DAEMON_HEALTH_FILE = HEALTH_DIR / "daemon_health.json"
PIPELINE_HEALTH_FILE = HEALTH_DIR / "pipeline_health.json"
CAPTURE_SNAPSHOT_FILE = HEALTH_DIR / "capture_snapshot.json"

# Thresholds
HEARTBEAT_STALE_S = 300       # Scheduler heartbeat stale after 5 min
CAPTURE_STALE_S = 600         # Capture logs stale after 10 min
EVIDENCE_STALE_S = 1800       # Evidence stale after 30 min


def send_heartbeat(daemon_name: str, status: str = "alive", metrics: dict = None):
    """Record heartbeat to state/daemon_health.json. 5s lock timeout, stderr fallback."""
    entry = {
        "status": status,
        "timestamp": datetime.now(timezone.utc).isoformat(),
        "pid": os.getpid(),
    }
    if metrics:
        entry["metrics"] = metrics
    HEALTH_DIR.mkdir(parents=True, exist_ok=True)
    try:
        from pulse.core.brain_io import _acquire
        with _acquire(DAEMON_HEALTH_FILE):
            data = {}
            if DAEMON_HEALTH_FILE.exists():
                try:
                    data = json.loads(DAEMON_HEALTH_FILE.read_text(encoding="utf-8"))
                except (json.JSONDecodeError, OSError):
                    data = {}
            data[daemon_name] = entry
            DAEMON_HEALTH_FILE.write_text(
                json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8"
            )
    except Exception as e:
        print(f"[HEARTBEAT] {daemon_name}: {e}", file=sys.stderr)


def _capture_log_growth() -> tuple:
    """Check if capture logs are growing since last snapshot. Returns (is_stale, info)."""
    current = {}
    for f in STATE_DIR.glob("pulse_captured_*.log"):
        try:
            current[f.name] = f.stat().st_size
        except OSError:
            pass

    prev_sizes, prev_ts = {}, 0.0
    if CAPTURE_SNAPSHOT_FILE.exists():
        try:
            snap = json.loads(CAPTURE_SNAPSHOT_FILE.read_text(encoding="utf-8"))
            prev_sizes = snap.get("sizes", {})
            ts_str = snap.get("timestamp", "")
            if ts_str:
                prev_ts = datetime.fromisoformat(ts_str.replace("Z", "+00:00")).timestamp()
        except Exception:
            pass

    growing = any(current.get(n, 0) > prev_sizes.get(n, 0) for n in current)
    elapsed = time.time() - prev_ts if prev_ts else 0
    stale = not growing and elapsed > CAPTURE_STALE_S and bool(prev_sizes)
    total_mb = round(sum(current.values()) / 1048576, 1)

    try:
        CAPTURE_SNAPSHOT_FILE.write_text(json.dumps({
            "sizes": current,
            "timestamp": datetime.now(timezone.utc).isoformat(),
        }, indent=2), encoding="utf-8")
    except Exception:
        pass

    return stale, {"growing": growing, "elapsed_s": round(elapsed), "total_mb": total_mb}


def check_pipeline_health(verbose: bool = False) -> dict:
    """Aggregate all health signals into unified report -> pipeline_health.json."""
    alerts, components = [], {}
    now = time.time()

    # 1. Orchestrator
    orch_pid_file = STATE_DIR / "pulse_orchestrator.pid"
    orch_running = False
    if orch_pid_file.exists():
        try:
            orch_running = is_process_running(int(orch_pid_file.read_text().strip()))
        except Exception:
            pass
    components["orchestrator"] = {"running": orch_running}
    if not orch_running:
        alerts.append("ORCH_DOWN: Orchestrator not running")

    # 2. Daemon processes
    daemon_pids = []
    dpf = STATE_DIR / "daemon_pids.json"
    if dpf.exists():
        try:
            daemon_pids = json.loads(dpf.read_text(encoding="utf-8"))
        except Exception:
            pass
    alive = sum(1 for p in daemon_pids if is_process_running(p))
    dead = len(daemon_pids) - alive
    components["daemons"] = {"total": len(daemon_pids), "alive": alive, "dead": dead}
    if dead > 0:
        alerts.append(f"DAEMON_DEAD: {dead}/{len(daemon_pids)} process(es) down")

    # 3. Scheduler/daemon heartbeats
    daemon_health = {}
    if DAEMON_HEALTH_FILE.exists():
        try:
            daemon_health = json.loads(DAEMON_HEALTH_FILE.read_text(encoding="utf-8"))
        except Exception:
            pass
    stale_hb, hung = [], []
    for name, entry in daemon_health.items():
        ts = entry.get("timestamp", "")
        if ts:
            try:
                age = now - datetime.fromisoformat(ts.replace("Z", "+00:00")).timestamp()
                if age > HEARTBEAT_STALE_S:
                    stale_hb.append(name)
            except Exception:
                pass
        metrics = entry.get("metrics", {})
        if entry.get("status") == "alive" and metrics.get("items_processed", -1) == 0:
            hung.append(name)
    components["heartbeats"] = {
        "reporting": len(daemon_health), "stale": stale_hb, "hung_suspects": hung,
    }
    if stale_hb:
        alerts.append(f"HEARTBEAT_STALE: {', '.join(stale_hb[:5])}")

    # 4. Workers
    hb_dir = STATE_DIR / "heartbeats"
    w_alive, w_stale, w_total = 0, 0, 0
    if hb_dir.exists():
        for hb_file in hb_dir.iterdir():
            if hb_file.suffix != ".json":
                continue
            w_total += 1
            try:
                if now - hb_file.stat().st_mtime < 90:
                    w_alive += 1
                else:
                    w_stale += 1
            except Exception:
                w_stale += 1
    components["workers"] = {"total": w_total, "alive": w_alive, "stale": w_stale}

    # 5. Capture log growth
    capture_stale, capture_info = _capture_log_growth()
    components["capture_logs"] = capture_info
    if capture_stale:
        alerts.append("CAPTURE_STALE: No new data in capture logs")

    # 6. Evidence freshness
    evidence_dir = HIPPOCAMPUS_DIR / "Evidence"
    newest = 0.0
    if evidence_dir.exists():
        for f in evidence_dir.rglob("*.json"):
            try:
                mt = f.stat().st_mtime
                if mt > newest:
                    newest = mt
            except Exception:
                pass
    ev_age = now - newest if newest else 999999
    components["evidence"] = {"age_s": round(ev_age), "fresh": ev_age < EVIDENCE_STALE_S}
    if EVIDENCE_STALE_S < ev_age < 999999:
        alerts.append(f"EVIDENCE_STALE: No writes in {ev_age / 60:.0f}min")

    # 7. Task board freshness
    board_age = 999999
    if TASK_BOARD_FILE.exists():
        try:
            board_age = now - TASK_BOARD_FILE.stat().st_mtime
        except Exception:
            pass
    components["task_board"] = {"age_s": round(board_age)}

    # 8. Brain health (from existing 6h check)
    brain_alerts = []
    bhf = STATE_DIR / "brain_health.json"
    if bhf.exists():
        try:
            bh = json.loads(bhf.read_text(encoding="utf-8"))
            brain_alerts = bh.get("alerts", [])
            components["brain_health"] = {
                "status": bh.get("status", "unknown"),
                "last_check": (bh.get("timestamp") or "unknown")[:19],
            }
        except Exception:
            pass

    # Build report
    status = "DOWN" if not orch_running else ("DEGRADED" if alerts or brain_alerts else "HEALTHY")
    report = {
        "timestamp": datetime.now(timezone.utc).isoformat(),
        "status": status,
        "alert_count": len(alerts) + len(brain_alerts),
        "alerts": alerts,
        "brain_alerts": brain_alerts[:5],
        "components": components,
    }
    try:
        PIPELINE_HEALTH_FILE.write_text(
            json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8"
        )
    except Exception as e:
        print(f"[HEALTH] Write failed: {e}", file=sys.stderr)

    if verbose:
        logger.info("[PIPELINE] %s — %d alert(s)", status, len(alerts) + len(brain_alerts))
        for a in alerts:
            logger.warning("  %s", a)
    return report


if __name__ == "__main__":
    import argparse
    logging.basicConfig(level=logging.INFO, format="%(message)s")
    ap = argparse.ArgumentParser(description="Pipeline Health Monitor")
    ap.add_argument("--once", action="store_true")
    ap.add_argument("--verbose", "-v", action="store_true")
    args = ap.parse_args()
    report = check_pipeline_health(verbose=True)
    print(json.dumps(report, indent=2))
