"""Watcher agent -- Health monitor and self-healer.

Monitors application health via HTTP endpoints, process checks,
log monitoring, and resource usage. Applies 4-tier self-healing:
  Level 0: Process restart (auto, 0-30s)
  Level 1: Known fix from cache (auto, 1-3 min)
  Level 2: AI diagnosis (collaborative, 3-10 min)
  Level 3: Human escalation (async)

Scope boundary: Watcher can restart processes and apply pre-approved fixes.
For anything else, it escalates.
"""

from __future__ import annotations

import contextlib
import json
import logging
import subprocess
import time
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any

if TYPE_CHECKING:
    from pathlib import Path

import httpx

from hive.agents.base import AgentRole, BaseAgent

logger = logging.getLogger("hive.watcher")


class CrashLoopDetector:
    """Detects crash loops to prevent infinite restart cycles."""

    def __init__(self, max_restarts: int = 3, window_seconds: int = 300) -> None:
        self.max_restarts = max_restarts
        self.window_seconds = window_seconds
        self.restart_timestamps: list[float] = []

    def record_restart(self) -> None:
        """Record a restart event."""
        now = time.time()
        self.restart_timestamps.append(now)
        self.restart_timestamps = [t for t in self.restart_timestamps if now - t < self.window_seconds]

    def is_crash_looping(self) -> bool:
        """Check if we're in a crash loop."""
        return len(self.restart_timestamps) >= self.max_restarts

    def reset(self) -> None:
        """Reset the detector (after admin resolution)."""
        self.restart_timestamps = []


class Watcher(BaseAgent):
    """Health monitor and self-healer.

    Real implementation with:
    - HTTP health endpoint polling
    - Process status checks via subprocess
    - Resource usage monitoring
    - Error rate tracking from log files
    - 4-tier escalation with real actions
    - Known-fix cache with success rate tracking
    """

    role = AgentRole.WATCHER

    def __init__(
        self,
        workspace_root: Path,
        health_url: str = "http://localhost:4000/health",
        check_interval: int = 60,
    ) -> None:
        super().__init__(workspace_root)
        self._health_url = health_url
        self._check_interval = check_interval
        self._crash_detector = CrashLoopDetector()
        self._health_history: list[dict[str, Any]] = []
        self._known_fixes: dict[str, dict[str, Any]] = {}
        self._incidents: list[dict[str, Any]] = []
        self._load_known_fixes()

    def _load_known_fixes(self) -> None:
        """Load known fixes from .hive/memory/incidents.md."""
        incidents_path = self.memory_dir / "incidents.md"
        if not incidents_path.exists():
            self._known_fixes = {}
            return

        content = incidents_path.read_text(encoding="utf-8")
        # Parse structured incident entries
        current_fix: dict[str, Any] = {}
        for line in content.splitlines():
            line = line.strip()
            if line.startswith("- signature:"):
                if current_fix.get("signature"):
                    self._known_fixes[current_fix["signature"]] = current_fix
                current_fix = {"signature": line.split(":", 1)[1].strip().strip('"')}
            elif line.startswith("- fix:") and current_fix:
                current_fix["fix_type"] = line.split(":", 1)[1].strip()
            elif line.startswith("- command:") and current_fix:
                current_fix["command"] = line.split(":", 1)[1].strip().strip('"')
            elif line.startswith("- success_rate:") and current_fix:
                try:
                    current_fix["success_rate"] = float(line.split(":", 1)[1].strip())
                except ValueError:
                    current_fix["success_rate"] = 0.0
            elif line.startswith("- auto_approved:") and current_fix:
                current_fix["auto_approved"] = line.split(":", 1)[1].strip().lower() == "true"

        if current_fix.get("signature"):
            self._known_fixes[current_fix["signature"]] = current_fix

    async def check_health(self) -> dict[str, Any]:
        """Run a comprehensive health check.

        Checks: HTTP health endpoint, process status, resource usage, error rate.
        Returns scored health result.
        """
        checks: dict[str, Any] = {}
        score = 0.0

        # Check 1: HTTP health endpoint (weight: 0.30)
        http_check = await self._check_http_health()
        checks["http_health"] = http_check
        score += http_check["score"] * 0.30

        # Check 2: Process status (weight: 0.25)
        process_check = self._check_process_status()
        checks["process_status"] = process_check
        score += process_check["score"] * 0.25

        # Check 3: Error rate from logs (weight: 0.20)
        error_check = self._check_error_rate()
        checks["error_rate"] = error_check
        score += error_check["score"] * 0.20

        # Check 4: Resource usage (weight: 0.15)
        resource_check = self._check_resource_usage()
        checks["resource_usage"] = resource_check
        score += resource_check["score"] * 0.15

        # Check 5: Dependency health (weight: 0.10)
        dep_check = await self._check_dependency_health()
        checks["dependency_health"] = dep_check
        score += dep_check["score"] * 0.10

        result = {
            "timestamp": datetime.now(timezone.utc).isoformat(),
            "score": round(score, 1),
            "status": self._score_to_status(score),
            "checks": checks,
        }

        # Persist
        self._update_health_snapshot(result)
        self._update_trends(result)
        self._health_history.append(result)

        # Auto-escalate if unhealthy
        if score < 50:
            logger.warning("Health CRITICAL: score=%.1f", score)
        elif score < 70:
            logger.warning("Health DEGRADED: score=%.1f", score)

        return result

    async def _check_http_health(self) -> dict[str, Any]:
        """Ping the application's /health endpoint."""
        try:
            start = time.monotonic()
            async with httpx.AsyncClient(timeout=5.0) as client:
                r = await client.get(self._health_url)
                latency_ms = int((time.monotonic() - start) * 1000)

                if r.status_code == 200:
                    return {"status": "pass", "latency_ms": latency_ms, "score": 100}
                else:
                    return {"status": "fail", "status_code": r.status_code, "latency_ms": latency_ms, "score": 0}
        except httpx.ConnectError:
            return {"status": "fail", "error": "connection_refused", "score": 0}
        except httpx.TimeoutException:
            return {"status": "fail", "error": "timeout", "score": 10}
        except Exception as e:
            return {"status": "fail", "error": str(e), "score": 0}

    def _check_process_status(self) -> dict[str, Any]:
        """Check if expected processes are running."""
        checks: dict[str, str] = {}

        # Check workspace has expected structure
        workspace_ok = (self._workspace_root / "pyproject.toml").exists()
        checks["workspace"] = "ok" if workspace_ok else "missing"

        # Check for Docker containers (if docker-compose based)
        compose_file = self._workspace_root / "docker-compose.yml"
        if compose_file.exists():
            try:
                result = subprocess.run(
                    ["docker", "compose", "ps", "--format", "json"],
                    cwd=self._workspace_root,
                    capture_output=True,
                    text=True,
                    timeout=10,
                    check=False,
                )
                if result.returncode == 0 and result.stdout.strip():
                    checks["docker_compose"] = "running"
                else:
                    checks["docker_compose"] = "stopped"
            except (FileNotFoundError, subprocess.TimeoutExpired):
                checks["docker_compose"] = "unknown"

        # Check CI config exists
        ci_ok = (self._workspace_root / ".github" / "workflows" / "ci.yml").exists()
        checks["ci_config"] = "ok" if ci_ok else "missing"

        # Score: 100 if all pass, proportional otherwise
        total = len(checks)
        passed = sum(1 for v in checks.values() if v in ("ok", "running"))
        score = (passed / max(total, 1)) * 100

        return {"status": "pass" if score >= 80 else "warning", "checks": checks, "score": round(score)}

    def _check_error_rate(self) -> dict[str, Any]:
        """Check error rate from recent log entries."""
        # Look for common Python log locations
        log_candidates = [
            self._workspace_root / "logs" / "app.log",
            self._workspace_root / "app.log",
            self._hive_dir / "metrics" / "errors.log",
        ]

        for log_path in log_candidates:
            if log_path.exists():
                try:
                    content = log_path.read_text(encoding="utf-8")
                    lines = content.splitlines()
                    recent = lines[-1000:] if len(lines) > 1000 else lines

                    error_count = sum(1 for line in recent if "ERROR" in line or "CRITICAL" in line)
                    warning_count = sum(1 for line in recent if "WARNING" in line)

                    rate = error_count / max(len(recent), 1)
                    score = max(0, 100 - (rate * 1000))  # Each 0.1% error rate costs 1 point

                    return {
                        "status": "pass" if score >= 80 else "warning" if score >= 50 else "fail",
                        "error_count": error_count,
                        "warning_count": warning_count,
                        "total_lines": len(recent),
                        "rate_per_line": round(rate, 4),
                        "score": round(score),
                    }
                except Exception:
                    pass

        # No logs found -- not a failure, just unknown
        return {"status": "pass", "note": "no log files found", "score": 80}

    def _check_resource_usage(self) -> dict[str, Any]:
        """Check CPU and memory usage."""
        try:
            # Use /proc for Linux
            with open("/proc/loadavg") as f:
                load_1m = float(f.read().split()[0])

            with open("/proc/meminfo") as f:
                meminfo = {}
                for line in f:
                    key, value = line.split(":", 1)
                    meminfo[key.strip()] = int(value.strip().split()[0])

            total_mb = meminfo.get("MemTotal", 0) / 1024
            available_mb = meminfo.get("MemAvailable", 0) / 1024
            used_pct = ((total_mb - available_mb) / max(total_mb, 1)) * 100

            cpu_score = max(0, 100 - (load_1m * 10))  # Penalize high load
            mem_score = max(0, 100 - used_pct)  # Penalize high memory use
            score = (cpu_score + mem_score) / 2

            return {
                "status": "pass" if score >= 70 else "warning",
                "load_1m": load_1m,
                "memory_used_pct": round(used_pct, 1),
                "memory_total_mb": round(total_mb),
                "score": round(score),
            }
        except (FileNotFoundError, ValueError, KeyError):
            # Not Linux or /proc not accessible (e.g., Docker)
            return {"status": "pass", "note": "/proc not available", "score": 80}

    async def _check_dependency_health(self) -> dict[str, Any]:
        """Check if external dependencies are reachable."""
        deps: dict[str, str] = {}

        # Check Ollama
        try:
            async with httpx.AsyncClient(timeout=3.0) as client:
                r = await client.get("http://localhost:11434/api/tags")
                deps["ollama"] = "up" if r.status_code == 200 else "down"
        except Exception:
            deps["ollama"] = "unreachable"

        # Check GitHub API (if GH_TOKEN set)
        import os

        if os.getenv("GH_TOKEN") or os.getenv("GITHUB_TOKEN"):
            try:
                async with httpx.AsyncClient(timeout=5.0) as client:
                    token = os.getenv("GH_TOKEN") or os.getenv("GITHUB_TOKEN")
                    r = await client.get(
                        "https://api.github.com/rate_limit",
                        headers={"Authorization": f"Bearer {token}"},
                    )
                    deps["github"] = "up" if r.status_code == 200 else "limited"
            except Exception:
                deps["github"] = "unreachable"

        total = len(deps)
        up_count = sum(1 for v in deps.values() if v == "up")
        score = (up_count / max(total, 1)) * 100 if total > 0 else 80

        return {"status": "pass" if score >= 70 else "warning", "dependencies": deps, "score": round(score)}

    def handle_error(self, error_signature: str, error_details: dict[str, Any]) -> dict[str, Any]:
        """Handle a detected error through the 4-tier escalation.

        Returns: action taken and result.
        """
        incident_id = f"INC-{len(self._incidents) + 1:03d}"
        start_time = time.time()

        # Check for crash loop first
        if self._crash_detector.is_crash_looping():
            self._record_incident(incident_id, error_signature, 3, "crash_loop_detected", start_time)
            return {
                "level": 3,
                "action": "crash_loop_detected",
                "incident_id": incident_id,
                "message": "Crash loop detected. Manual intervention required.",
                "details": error_details,
            }

        # Level 0: Simple restart if applicable
        if error_details.get("type") == "process_crash":
            self._crash_detector.record_restart()
            fix_result = self._attempt_restart(error_details)
            if fix_result["success"]:
                self._record_incident(incident_id, error_signature, 0, "restart", start_time)
                return {"level": 0, "action": "restart", "incident_id": incident_id, **fix_result}

        # Level 1: Check known fixes
        if error_signature in self._known_fixes:
            fix = self._known_fixes[error_signature]
            if fix.get("auto_approved") and fix.get("success_rate", 0) >= 0.80:
                fix_result = self._apply_known_fix(fix)
                if fix_result["success"]:
                    self._record_incident(incident_id, error_signature, 1, "known_fix", start_time)
                    return {"level": 1, "action": "known_fix", "incident_id": incident_id, **fix_result}

        # Level 2: Request AI diagnosis (via CEO → Analyst + Advisor)
        self._record_incident(incident_id, error_signature, 2, "diagnosis_needed", start_time)
        return {
            "level": 2,
            "action": "diagnosis_needed",
            "incident_id": incident_id,
            "error_signature": error_signature,
            "message": "Novel error. Requesting AI diagnosis.",
            "details": error_details,
        }

    def _attempt_restart(self, error_details: dict[str, Any]) -> dict[str, Any]:
        """Level 0: Attempt to restart a crashed process."""
        container = error_details.get("container")
        service = error_details.get("service")

        if container:
            try:
                result = subprocess.run(
                    ["docker", "restart", container],
                    capture_output=True,
                    text=True,
                    timeout=30,
                    check=False,
                )
                return {"success": result.returncode == 0, "method": "docker_restart", "output": result.stdout}
            except (FileNotFoundError, subprocess.TimeoutExpired):
                return {"success": False, "method": "docker_restart", "error": "docker not available or timeout"}

        if service:
            try:
                result = subprocess.run(
                    ["systemctl", "restart", service],
                    capture_output=True,
                    text=True,
                    timeout=30,
                    check=False,
                )
                return {"success": result.returncode == 0, "method": "systemctl_restart", "output": result.stdout}
            except (FileNotFoundError, subprocess.TimeoutExpired):
                return {"success": False, "method": "systemctl_restart", "error": "systemctl not available"}

        return {"success": False, "method": "none", "error": "no restart target specified"}

    def _apply_known_fix(self, fix: dict[str, Any]) -> dict[str, Any]:
        """Level 1: Apply a known fix from the cache."""
        command = fix.get("command")
        if not command:
            return {"success": False, "error": "no command in known fix"}

        try:
            result = subprocess.run(
                command.split(),
                cwd=self._workspace_root,
                capture_output=True,
                text=True,
                timeout=fix.get("timeout_seconds", 60),
                check=False,
            )
            return {"success": result.returncode == 0, "output": result.stdout[:500], "method": "known_fix"}
        except (subprocess.TimeoutExpired, FileNotFoundError) as e:
            return {"success": False, "error": str(e), "method": "known_fix"}

    def _record_incident(
        self,
        incident_id: str,
        signature: str,
        level: int,
        action: str,
        start_time: float,
    ) -> dict[str, Any]:
        """Record an incident in memory."""
        duration = time.time() - start_time
        incident = {
            "id": incident_id,
            "timestamp": datetime.now(timezone.utc).isoformat(),
            "signature": signature,
            "level": level,
            "action": action,
            "duration_seconds": round(duration, 2),
        }
        self._incidents.append(incident)

        # Write to institutional memory
        entry = (
            f"\n### {incident_id}: {signature}\n"
            f"- date: {incident['timestamp']}\n"
            f"- level: {level}\n"
            f"- action: {action}\n"
            f"- duration: {duration:.1f}s\n"
        )
        self.write_memory("incidents.md", entry)
        return incident

    def _update_health_snapshot(self, result: dict[str, Any]) -> None:
        """Write latest health check to .hive/metrics/health.json."""
        health_path = self._hive_dir / "metrics" / "health.json"
        health_path.parent.mkdir(parents=True, exist_ok=True)
        health_path.write_text(json.dumps(result, indent=2), encoding="utf-8")

    def _update_trends(self, result: dict[str, Any]) -> None:
        """Append health data point to trends file."""
        trends_path = self._hive_dir / "metrics" / "trends.json"
        trends_path.parent.mkdir(parents=True, exist_ok=True)

        trends: dict[str, Any] = {"retention_days": 30, "data_points": []}
        if trends_path.exists():
            with contextlib.suppress(json.JSONDecodeError):
                trends = json.loads(trends_path.read_text(encoding="utf-8"))

        trends["data_points"].append(
            {
                "timestamp": result["timestamp"],
                "score": result["score"],
                "status": result["status"],
            }
        )

        # Trim to retention period (keep last 30 days worth at 1/hour = ~720 points)
        if len(trends["data_points"]) > 720:
            trends["data_points"] = trends["data_points"][-720:]

        trends_path.write_text(json.dumps(trends, indent=2), encoding="utf-8")

    @staticmethod
    def _score_to_status(score: float) -> str:
        """Convert numeric score to status label."""
        if score >= 90:
            return "healthy"
        if score >= 70:
            return "degraded"
        if score >= 50:
            return "unhealthy"
        return "critical"

    def get_status(self) -> dict[str, Any]:
        """Watcher-specific status."""
        status = super().get_status()
        latest = self._health_history[-1] if self._health_history else None
        status.update(
            {
                "health_score": latest["score"] if latest else None,
                "health_status": latest["status"] if latest else "unknown",
                "total_checks": len(self._health_history),
                "known_fixes": len(self._known_fixes),
                "total_incidents": len(self._incidents),
                "crash_loop_active": self._crash_detector.is_crash_looping(),
                "check_interval_seconds": self._check_interval,
            }
        )
        return status
