# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# ORCID: 0009-0006-4957-0540
# Email: Georgescottfoley@proton.me
# Licensed under the MIT License - see LICENSE file for details

"""
Service Manager — provisions and manages self-hosted services via Podman/Docker.

Detects available container runtime, pulls images, creates/starts/stops containers,
and persists state to ``~/.familiar/services/state.json``.
"""

import json
import logging
import os
import secrets
import shutil
import socket
import subprocess
import threading
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from enum import Enum
from pathlib import Path
from typing import Any, Dict, List, Optional

from familiar.services.specs import SERVICE_SPECS, ServiceSpec, ServiceType

logger = logging.getLogger(__name__)

SERVICES_DIR = Path.home() / ".familiar" / "services"
STATE_FILE = SERVICES_DIR / "state.json"


class ContainerRuntime(str, Enum):
    """Detected container engine."""

    PODMAN = "podman"
    DOCKER = "docker"
    NONE = "none"


class ServiceStatus(str, Enum):
    """Current status of a managed service."""

    NOT_INSTALLED = "not_installed"
    STOPPED = "stopped"
    RUNNING = "running"
    ERROR = "error"
    PULLING = "pulling"
    STARTING = "starting"


@dataclass
class ServiceState:
    """Persisted state for a single service."""

    key: str
    status: str = ServiceStatus.NOT_INSTALLED.value
    container_id: str = ""
    container_name: str = ""
    ports: Dict[int, int] = field(default_factory=dict)
    started_at: str = ""
    error_message: str = ""
    url: str = ""

    def to_dict(self) -> dict:
        return asdict(self)


class ServiceManager:
    """
    Provisions and manages containerized (and native) services.

    Uses Podman by default, falls back to Docker.  All container names
    are prefixed ``familiar-`` to avoid collisions.
    """

    def __init__(self):
        self._runtime: Optional[ContainerRuntime] = None
        self._runtime_path: Optional[str] = None
        self._unusable_runtime: Optional[str] = None  # e.g. "docker" when socket is denied
        self._lock = threading.Lock()
        self._states: Dict[str, ServiceState] = {}
        SERVICES_DIR.mkdir(parents=True, exist_ok=True)
        self._load_state()

    # ------------------------------------------------------------------ runtime

    def detect_runtime(self) -> ContainerRuntime:
        """Detect the best available container runtime.

        Checks that the runtime is actually usable (not just installed).
        Docker without socket permission is treated as unavailable so that
        Podman can be installed as a rootless fallback.
        """
        if self._runtime is not None:
            return self._runtime

        for candidate in ("podman", "docker"):
            path = shutil.which(candidate)
            if path is None:
                continue
            try:
                result = subprocess.run(
                    [path, "info"],
                    capture_output=True,
                    text=True,
                    timeout=10,
                )
                # Docker exits 0 even when the server is unreachable —
                # check stderr for "permission denied" on the socket
                combined = (result.stdout + result.stderr).lower()
                if "permission denied" in combined:
                    logger.warning(
                        "%s found at %s but socket permission denied — skipping",
                        candidate,
                        path,
                    )
                    self._unusable_runtime = candidate
                    continue
                if result.returncode != 0 and "error" in combined:
                    logger.warning(
                        "%s at %s returned errors — skipping: %s",
                        candidate,
                        path,
                        result.stderr[:200],
                    )
                    self._unusable_runtime = candidate
                    continue

                self._runtime_path = path
                self._runtime = ContainerRuntime(candidate)
                logger.info("Detected container runtime: %s (%s)", candidate, path)
                return self._runtime
            except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
                continue

        self._runtime = ContainerRuntime.NONE
        return self._runtime

    def invalidate_runtime(self):
        """Clear cached runtime so next detect_runtime() re-probes."""
        self._runtime = None
        self._runtime_path = None
        self._unusable_runtime = None

    @staticmethod
    def _detect_platform() -> str:
        """Return 'linux', 'darwin', or 'windows'."""
        import platform

        return platform.system().lower()

    def bootstrap_runtime(self) -> Dict[str, Any]:
        """Install a container runtime if none is available.

        If Docker is installed but unusable (e.g. socket permission denied),
        installs Podman as a rootless fallback — no sudo, no group changes,
        works immediately.

        Platform support:
          - Linux: Podman via system package manager (dnf/apt/pacman/zypper/apk)
          - macOS: Podman via Homebrew (``brew install podman && podman machine init && podman machine start``)
          - Windows: Podman via winget, or Docker Desktop download hint

        Returns a dict with success, runtime, and install output.
        """
        rt = self.detect_runtime()
        if rt != ContainerRuntime.NONE:
            return {
                "success": True,
                "runtime": rt.value,
                "message": f"{rt.value} already installed",
                "already_installed": True,
            }

        # If we got here because Docker exists but is unusable, log it
        unusable = getattr(self, "_unusable_runtime", None)
        if unusable:
            logger.info(
                "%s found but not usable (permission denied) — installing Podman as fallback",
                unusable,
            )

        plat = self._detect_platform()

        if plat == "darwin":
            return self._bootstrap_macos()
        elif plat == "windows":
            return self._bootstrap_windows()
        else:
            return self._bootstrap_linux()

    def _bootstrap_linux(self) -> Dict[str, Any]:
        """Install Podman on Linux via system package manager."""
        pm = None
        for candidate in ("dnf", "apt-get", "pacman", "zypper", "apk"):
            if shutil.which(candidate):
                pm = candidate
                break

        if pm is None:
            if shutil.which("brew"):
                pm = "brew"
            else:
                return {
                    "success": False,
                    "error": "No supported package manager found (dnf, apt, pacman, zypper, apk, brew)",
                }

        pkg = "podman"
        install_map = {
            "dnf": ["dnf", "install", "-y", pkg],
            "apt-get": ["apt-get", "install", "-y", pkg],
            "pacman": ["pacman", "-S", "--noconfirm", pkg],
            "zypper": ["zypper", "install", "-y", pkg],
            "apk": ["apk", "add", pkg],
            "brew": ["brew", "install", pkg],
        }
        cmd = install_map.get(pm, [])
        if not cmd:
            return {"success": False, "error": f"Unsupported package manager: {pm}"}

        needs_sudo = pm in ("dnf", "apt-get", "pacman", "zypper", "apk")
        if needs_sudo:
            cmd = ["sudo"] + cmd

        return self._run_install(cmd, pkg, pm)

    def _bootstrap_macos(self) -> Dict[str, Any]:
        """Install Podman on macOS via Homebrew, then init+start the VM."""
        if not shutil.which("brew"):
            return {
                "success": False,
                "error": (
                    "Homebrew is required to install Podman on macOS. "
                    "Install it first: https://brew.sh"
                ),
                "platform": "macos",
            }

        # Step 1: brew install podman
        result = self._run_install(["brew", "install", "podman"], "podman", "brew")
        if not result.get("success"):
            return result

        # Step 2: podman machine init + start (macOS runs containers in a VM)
        podman_path = shutil.which("podman")
        if not podman_path:
            return {
                "success": False,
                "error": "Podman installed but not found in PATH — restart your terminal",
            }

        init_output = ""
        try:
            # Init the default machine (skip if already exists)
            init = subprocess.run(
                [podman_path, "machine", "init"],
                capture_output=True,
                text=True,
                timeout=120,
            )
            init_output = (init.stdout + init.stderr).strip()

            # Start the machine
            start = subprocess.run(
                [podman_path, "machine", "start"],
                capture_output=True,
                text=True,
                timeout=120,
            )
            init_output += "\n" + (start.stdout + start.stderr).strip()

            if start.returncode != 0 and "already running" not in init_output.lower():
                return {
                    "success": False,
                    "error": "Podman installed but VM failed to start",
                    "output": init_output[:4000],
                    "platform": "macos",
                }
        except subprocess.TimeoutExpired:
            return {
                "success": False,
                "error": "Podman machine init/start timed out",
                "platform": "macos",
            }
        except Exception as e:
            return {
                "success": False,
                "error": f"Podman machine setup failed: {e}",
                "platform": "macos",
            }

        # Re-detect
        self.invalidate_runtime()
        rt = self.detect_runtime()
        return {
            "success": rt != ContainerRuntime.NONE,
            "runtime": rt.value if rt != ContainerRuntime.NONE else "none",
            "message": "Installed Podman via Homebrew (macOS VM)",
            "output": init_output[:2000],
            "already_installed": False,
            "platform": "macos",
        }

    def _bootstrap_windows(self) -> Dict[str, Any]:
        """Install Podman on Windows via winget, or provide Docker Desktop instructions."""
        # Try winget first (built into Windows 10/11)
        if shutil.which("winget"):
            result = self._run_install(
                ["winget", "install", "--id", "RedHat.Podman", "-e", "--accept-source-agreements"],
                "podman",
                "winget",
            )
            if result.get("success"):
                # Init the Podman machine (WSL2 backend)
                try:
                    podman_path = shutil.which("podman") or "podman"
                    subprocess.run(
                        [podman_path, "machine", "init"],
                        capture_output=True,
                        text=True,
                        timeout=120,
                    )
                    subprocess.run(
                        [podman_path, "machine", "start"],
                        capture_output=True,
                        text=True,
                        timeout=120,
                    )
                except Exception:
                    pass  # Best effort — user may need to restart
                result["platform"] = "windows"
                result["note"] = "You may need to restart your terminal for podman to be available."
                return result

        # Try choco
        if shutil.which("choco"):
            result = self._run_install(
                ["choco", "install", "podman-desktop", "-y"],
                "podman",
                "choco",
            )
            if result.get("success"):
                result["platform"] = "windows"
                return result

        # No package manager — give instructions
        return {
            "success": False,
            "error": (
                "No package manager found (winget or choco). "
                "Install Podman Desktop from https://podman-desktop.io/downloads "
                "or Docker Desktop from https://docker.com/products/docker-desktop"
            ),
            "platform": "windows",
        }

    def _run_install(self, cmd: List[str], pkg: str, pm: str) -> Dict[str, Any]:
        """Execute an install command and re-detect the runtime."""
        logger.info("Bootstrapping container runtime: %s", " ".join(cmd))
        try:
            result = subprocess.run(
                cmd,
                capture_output=True,
                text=True,
                timeout=300,
            )
            output = (result.stdout + result.stderr).strip()
            if result.returncode != 0:
                return {
                    "success": False,
                    "error": f"Install failed (exit {result.returncode})",
                    "output": output[:4000],
                    "command": " ".join(cmd),
                }

            # Re-detect runtime
            self.invalidate_runtime()
            rt = self.detect_runtime()
            if rt == ContainerRuntime.NONE:
                return {
                    "success": False,
                    "error": "Package installed but runtime not detected — may need a shell restart",
                    "output": output[:4000],
                }

            return {
                "success": True,
                "runtime": rt.value,
                "message": f"Installed {pkg} via {pm}",
                "output": output[:2000],
                "already_installed": False,
            }
        except subprocess.TimeoutExpired:
            return {"success": False, "error": "Install timed out (5 min)"}
        except Exception as e:
            return {"success": False, "error": str(e)}

    def get_runtime_info(self) -> Dict[str, Any]:
        """Return runtime name, version, and rootless status."""
        rt = self.detect_runtime()
        if rt == ContainerRuntime.NONE:
            info: Dict[str, Any] = {"runtime": "none", "version": "", "rootless": False}
            # Report unusable runtime so the dashboard can explain the problem
            unusable = getattr(self, "_unusable_runtime", None)
            if unusable:
                info["unusable_runtime"] = unusable
                info["unusable_reason"] = "permission_denied"
                info["fix_hint"] = (
                    f"{unusable} is installed but your user can't access it. "
                    f"Familiar will install Podman (rootless) automatically, "
                    f"or you can run: sudo usermod -aG docker $USER && newgrp docker"
                )
            return info

        info: Dict[str, Any] = {"runtime": rt.value, "version": "", "rootless": False}
        try:
            result = subprocess.run(
                [self._runtime_path, "version", "--format", "{{.Client.Version}}"],
                capture_output=True,
                text=True,
                timeout=10,
            )
            info["version"] = result.stdout.strip() or ""
        except Exception:
            try:
                result = subprocess.run(
                    [self._runtime_path, "version"],
                    capture_output=True,
                    text=True,
                    timeout=10,
                )
                for line in result.stdout.splitlines():
                    if "version" in line.lower():
                        info["version"] = line.split(":")[-1].strip()
                        break
            except Exception as e:
                logger.debug("Could not detect container runtime version: %s", e)

        if rt == ContainerRuntime.PODMAN:
            try:
                result = subprocess.run(
                    [self._runtime_path, "info", "--format", "{{.Host.Security.Rootless}}"],
                    capture_output=True,
                    text=True,
                    timeout=10,
                )
                info["rootless"] = result.stdout.strip().lower() == "true"
            except Exception as e:
                logger.debug("Could not detect Podman rootless mode: %s", e)
        elif rt == ContainerRuntime.DOCKER:
            # Docker rootless: check SecurityOptions for "rootless"
            try:
                result = subprocess.run(
                    [self._runtime_path, "info", "--format", "{{.SecurityOptions}}"],
                    capture_output=True,
                    text=True,
                    timeout=10,
                )
                info["rootless"] = "rootless" in result.stdout.lower()
            except Exception:
                pass

        return info

    def get_security_posture(self) -> Dict[str, Any]:
        """Return a summary of the server's security posture for dashboard display."""
        rt_info = self.get_runtime_info()
        statuses = self.get_all_statuses()
        running = [s for s in statuses if s.get("status") == "running"]

        posture: Dict[str, Any] = {
            "runtime": rt_info.get("runtime", "none"),
            "rootless": rt_info.get("rootless", False),
            "services_running": len(running),
            "localhost_only": True,  # All specs default to localhost_only
            "caps_dropped": True,  # All provisions use --cap-drop=ALL
            "memory_limited": True,  # All specs have memory_limit
            "credentials_generated": True,  # No hardcoded passwords
            "tls_enabled": False,  # Until Caddy is deployed
        }

        # Check if Caddy is running for TLS
        for s in running:
            if s.get("key") == "caddy":
                posture["tls_enabled"] = True
                break

        # Overall score: count of True values out of possible
        checks = [
            "rootless",
            "localhost_only",
            "caps_dropped",
            "memory_limited",
            "credentials_generated",
            "tls_enabled",
        ]
        posture["score"] = sum(1 for c in checks if posture.get(c))
        posture["score_max"] = len(checks)

        return posture

    def full_server_setup(self, service_keys: List[str]) -> Dict[str, Any]:
        """End-to-end server setup from zero.

        1. Check for container runtime → install Podman if missing
        2. Provision each requested service (pull image + create container)
        3. Start all services in dependency order
        4. Health-check each running service

        Returns a dict with overall status and per-service results.
        This is the "one button" path: user picks a job class,
        the agent handles everything else.
        """
        results: Dict[str, Any] = {
            "bootstrap": None,
            "services": [],
            "summary": {"total": len(service_keys), "ok": 0, "failed": 0, "skipped": 0},
        }

        # Step 1: Ensure we have a container runtime
        rt = self.detect_runtime()
        if rt == ContainerRuntime.NONE:
            bootstrap = self.bootstrap_runtime()
            results["bootstrap"] = bootstrap
            if not bootstrap.get("success"):
                results["summary"]["failed"] = len(service_keys)
                return results
        else:
            results["bootstrap"] = {
                "success": True,
                "runtime": rt.value,
                "already_installed": True,
            }

        # Step 2 + 3: Provision and start each service
        # Separate native vs container services
        from familiar.services.specs import SERVICE_SPECS

        container_keys = []
        for key in service_keys:
            spec = SERVICE_SPECS.get(key)
            if not spec:
                results["services"].append(
                    {
                        "service": key,
                        "status": "unknown_spec",
                        "ok": False,
                    }
                )
                results["summary"]["failed"] += 1
                continue

            if spec.service_type == ServiceType.NATIVE:
                # Native services can't be auto-provisioned via containers
                results["services"].append(
                    {
                        "service": key,
                        "display_name": spec.display_name,
                        "status": "native_skipped",
                        "ok": True,
                        "hint": "System service — install via package manager if needed",
                    }
                )
                results["summary"]["skipped"] += 1
                continue

            # Check if already running
            status = self.get_status(key)
            if status.get("status") == ServiceStatus.RUNNING.value:
                results["services"].append(
                    {
                        "service": key,
                        "display_name": spec.display_name,
                        "status": "already_running",
                        "ok": True,
                        "url": status.get("url", ""),
                    }
                )
                results["summary"]["ok"] += 1
                continue

            container_keys.append(key)

        # Separate stack services (have depends_on) from standalone
        stack_keys = []
        standalone_keys = []
        for key in container_keys:
            spec = SERVICE_SPECS.get(key)
            if spec and spec.depends_on:
                stack_keys.append(key)
            else:
                # Also include services that other stack members depend on
                standalone_keys.append(key)

        # Check if any standalone service is a dependency of a stack service
        # If so, move it into the stack group
        stack_deps = set()
        for key in stack_keys:
            spec = SERVICE_SPECS.get(key)
            if spec:
                stack_deps.update(spec.depends_on)
        promoted = [k for k in standalone_keys if k in stack_deps]
        if promoted:
            stack_keys = promoted + stack_keys  # deps first
            standalone_keys = [k for k in standalone_keys if k not in stack_deps]

        # Deploy stacks via compose (shared network, dependency ordering)
        if stack_keys:
            stack_result = self.provision_compose_stack(stack_keys)
            results["services"].extend(stack_result["services"])
            results["summary"]["ok"] += stack_result["summary"]["ok"]
            results["summary"]["failed"] += stack_result["summary"]["failed"]

        # Provision standalone container services individually
        for key in standalone_keys:
            spec = SERVICE_SPECS.get(key)
            svc_result: Dict[str, Any] = {
                "service": key,
                "display_name": spec.display_name if spec else key,
            }
            try:
                prov = self.provision(key)
                if not prov.get("success"):
                    svc_result["status"] = "provision_failed"
                    svc_result["error"] = prov.get("error", "unknown")
                    svc_result["ok"] = False
                    results["summary"]["failed"] += 1
                    results["services"].append(svc_result)
                    continue
            except Exception as e:
                svc_result["status"] = "provision_error"
                svc_result["error"] = str(e)
                svc_result["ok"] = False
                results["summary"]["failed"] += 1
                results["services"].append(svc_result)
                continue

            # Start
            try:
                start = self.start(key)
                if not start.get("success"):
                    svc_result["status"] = "start_failed"
                    svc_result["error"] = start.get("error", "unknown")
                    svc_result["ok"] = False
                    results["summary"]["failed"] += 1
                    results["services"].append(svc_result)
                    continue
            except Exception as e:
                svc_result["status"] = "start_error"
                svc_result["error"] = str(e)
                svc_result["ok"] = False
                results["summary"]["failed"] += 1
                results["services"].append(svc_result)
                continue

            # Health verification — wait for service to become responsive
            health = self.health_check_with_retry(key)
            svc_result["url"] = start.get("url", "")
            if spec and spec.setup_notes:
                svc_result["setup_notes"] = spec.setup_notes
            if health.get("healthy"):
                svc_result["status"] = "running"
                svc_result["ok"] = True
                svc_result["health_attempts"] = health.get("attempts", 0)
                results["summary"]["ok"] += 1
            else:
                # Started but not healthy — report as degraded, not failed
                svc_result["status"] = "degraded"
                svc_result["ok"] = True  # Container is running, just not responding yet
                svc_result["health_error"] = health.get("error", "")
                svc_result["health_attempts"] = health.get("attempts", 0)
                results["summary"]["ok"] += 1
                logger.warning(
                    "Service %s started but health check failed after %d attempts: %s",
                    key,
                    health.get("attempts", 0),
                    health.get("error", ""),
                )

            results["services"].append(svc_result)

        # Deploy Caddy reverse proxy if any container services succeeded
        running_keys = [
            s["service"]
            for s in results["services"]
            if s.get("ok")
            and s.get("status") in ("running", "degraded", "already_running")
            and s["service"] != "caddy"
        ]
        if running_keys and "caddy" not in service_keys:
            try:
                from familiar.services.proxy import write_caddyfile

                write_caddyfile(services=running_keys)
                # Provision and start Caddy
                caddy_result: Dict[str, Any] = {
                    "service": "caddy",
                    "display_name": "Caddy (Reverse Proxy)",
                }
                prov = self.provision("caddy")
                if prov.get("success"):
                    start = self.start("caddy")
                    if start.get("success"):
                        caddy_result["status"] = "running"
                        caddy_result["ok"] = True
                        results["summary"]["ok"] += 1
                    else:
                        caddy_result["status"] = "start_failed"
                        caddy_result["error"] = start.get("error", "")
                        caddy_result["ok"] = False
                        results["summary"]["failed"] += 1
                else:
                    caddy_result["status"] = "provision_failed"
                    caddy_result["error"] = prov.get("error", "")
                    caddy_result["ok"] = False
                    results["summary"]["failed"] += 1
                results["services"].append(caddy_result)
                results["summary"]["total"] += 1
            except Exception as e:
                logger.warning("Caddy deployment failed: %s", e)

        # Include credential summaries for services that generated them
        cred_summaries = []
        for svc in results["services"]:
            if svc.get("ok"):
                spec = SERVICE_SPECS.get(svc["service"])
                if spec and spec.credential_keys:
                    from familiar.services.credentials import get_credential_summary

                    summary = get_credential_summary(svc["service"])
                    if summary:
                        cred_summaries.append(summary)
        if cred_summaries:
            results["credentials"] = cred_summaries

        # Include security posture
        results["security"] = self.get_security_posture()

        return results

    # ----------------------------------------------------------- compose stacks

    def _find_compose_bin(self) -> Optional[str]:
        """Find podman-compose or docker-compose binary.

        Prefers the compose tool matching the detected runtime to avoid
        cross-runtime issues (e.g. docker-compose trying to use Docker
        socket when Podman is the active runtime).
        """
        import sys

        # Order candidates to match the detected runtime
        if self._runtime == ContainerRuntime.DOCKER:
            candidates = ["docker-compose", "podman-compose"]
        else:
            candidates = ["podman-compose", "docker-compose"]

        for name in candidates:
            path = shutil.which(name)
            if path:
                return path
        # Check in the same venv/prefix as the running Python
        venv_bin = Path(sys.executable).parent / "podman-compose"
        if venv_bin.exists():
            return str(venv_bin)
        # Check ~/.local/bin (pip install --user)
        local_bin = Path.home() / ".local" / "bin" / "podman-compose"
        if local_bin.exists():
            return str(local_bin)
        # Try 'docker compose' (v2 plugin) — only if Docker is the runtime
        if self._runtime == ContainerRuntime.DOCKER and shutil.which("docker"):
            try:
                result = subprocess.run(
                    ["docker", "compose", "version"],
                    capture_output=True,
                    text=True,
                    timeout=5,
                )
                if result.returncode == 0:
                    return "docker compose"
            except Exception:
                pass
        return None

    def _ensure_compose(self) -> Optional[str]:
        """Ensure a compose tool is available, installing if needed."""
        import sys

        compose = self._find_compose_bin()
        if compose:
            return compose
        # Install podman-compose into the same Python environment
        pip = str(Path(sys.executable).parent / "pip3")
        if not Path(pip).exists():
            pip = str(Path(sys.executable).parent / "pip")
        try:
            subprocess.run(
                [pip, "install", "podman-compose"],
                capture_output=True,
                text=True,
                timeout=120,
            )
            return self._find_compose_bin()
        except Exception as e:
            logger.warning("Failed to install podman-compose: %s", e)
            return None

    def provision_compose_stack(self, service_keys: List[str]) -> Dict[str, Any]:
        """Provision a group of interdependent services via compose.

        Uses podman-compose/docker-compose to handle shared networking,
        dependency ordering, and inter-container DNS automatically.
        """
        from familiar.services.compose import generate_compose
        from familiar.services.specs import SERVICE_SPECS

        rt = self.detect_runtime()
        if rt == ContainerRuntime.NONE:
            return {
                "services": [
                    {
                        "service": k,
                        "display_name": SERVICE_SPECS[k].display_name if k in SERVICE_SPECS else k,
                        "status": "no_runtime",
                        "ok": False,
                        "error": "No container runtime (podman/docker) found.",
                    }
                    for k in service_keys
                ],
                "summary": {"total": len(service_keys), "ok": 0, "failed": len(service_keys)},
            }

        results: Dict[str, Any] = {
            "services": [],
            "summary": {"total": len(service_keys), "ok": 0, "failed": 0},
        }

        # Ensure compose tool is available
        compose_bin = self._ensure_compose()
        if not compose_bin:
            results["summary"]["failed"] = len(service_keys)
            for key in service_keys:
                spec = SERVICE_SPECS.get(key)
                results["services"].append(
                    {
                        "service": key,
                        "display_name": spec.display_name if spec else key,
                        "status": "compose_unavailable",
                        "ok": False,
                        "error": "podman-compose not found. Install via: pip3 install podman-compose",
                    }
                )
            return results

        # Pre-flight checks for all services
        for key in service_keys:
            spec = SERVICE_SPECS.get(key)
            if spec:
                preflight = self._preflight_check(key, spec)
                if not preflight.get("success", True):
                    results["services"].append(
                        {
                            "service": key,
                            "display_name": spec.display_name,
                            "status": "preflight_failed",
                            "ok": False,
                            "error": preflight.get("error", ""),
                        }
                    )
                    results["summary"]["failed"] += 1
                    results["summary"]["total"] -= 1
                    service_keys = [k for k in service_keys if k != key]

        if not service_keys:
            return results

        # Pull images for all services first
        for key in service_keys:
            pull = self.pull_image(key)
            if not pull.get("success"):
                spec = SERVICE_SPECS.get(key)
                results["services"].append(
                    {
                        "service": key,
                        "display_name": spec.display_name if spec else key,
                        "status": "pull_failed",
                        "ok": False,
                        "error": pull.get("error", ""),
                    }
                )
                results["summary"]["failed"] += 1
                service_keys = [k for k in service_keys if k != key]

        if not service_keys:
            return results

        # Create volume directories and fix permissions
        home = str(Path.home())
        for key in service_keys:
            spec = SERVICE_SPECS.get(key)
            if not spec:
                continue
            # Determine which host paths are file mounts (written by config_files)
            svc_dir = Path(home) / ".familiar" / "services" / key
            config_file_paths = set()
            if spec.config_files:
                for rel_path in spec.config_files:
                    config_file_paths.add(str(svc_dir / rel_path))
            for host_path in spec.get_resolved_volumes(home):
                p = Path(host_path)
                if str(p) in config_file_paths:
                    # File mount — create parent dir only, file written by config_files
                    p.parent.mkdir(parents=True, exist_ok=True)
                else:
                    p.mkdir(parents=True, exist_ok=True)
                    try:
                        os.chmod(p, 0o755)
                    except OSError:
                        pass

            # Reclaim ownership from prior deploys before writing configs
            if spec.container_uid and self._runtime == ContainerRuntime.PODMAN:
                for host_path in spec.get_resolved_volumes(home):
                    try:
                        subprocess.run(
                            [self._runtime_path, "unshare", "chown", "-R", "0:0", host_path],
                            capture_output=True,
                            text=True,
                            timeout=30,
                        )
                    except Exception:
                        pass

            # Write config files into volume dirs
            if spec.config_files:
                svc_dir = Path(home) / ".familiar" / "services" / key
                for rel_path, content in spec.config_files.items():
                    cfg_path = svc_dir / rel_path
                    cfg_path.parent.mkdir(parents=True, exist_ok=True)
                    cfg_path.write_text(content)

            # Set ownership to container UID for rootless Podman
            if spec.container_uid and self._runtime == ContainerRuntime.PODMAN:
                uid = str(spec.container_uid)
                for host_path in spec.get_resolved_volumes(home):
                    try:
                        subprocess.run(
                            [
                                self._runtime_path,
                                "unshare",
                                "chown",
                                "-R",
                                f"{uid}:{uid}",
                                host_path,
                            ],
                            capture_output=True,
                            text=True,
                            timeout=30,
                        )
                    except Exception:
                        pass

        # Generate and write compose file
        compose_content = generate_compose(
            service_keys=service_keys,
            runtime=self._runtime.value if self._runtime != ContainerRuntime.NONE else "podman",
        )
        compose_dir = Path.home() / ".familiar" / "services"
        compose_dir.mkdir(parents=True, exist_ok=True)
        compose_file = compose_dir / "familiar-compose.yml"
        compose_file.write_text(compose_content)

        # Remove existing containers to avoid name conflicts
        for key in service_keys:
            spec = SERVICE_SPECS.get(key)
            if spec:
                try:
                    subprocess.run(
                        [self._runtime_path, "rm", "-f", spec.default_container_name],
                        capture_output=True,
                        text=True,
                        timeout=30,
                    )
                except Exception:
                    pass

        # Run compose up
        compose_cmd = compose_bin.split()  # handles "docker compose" (2 words)
        cmd = compose_cmd + ["-f", str(compose_file), "up", "-d"]
        try:
            result = subprocess.run(
                cmd,
                capture_output=True,
                text=True,
                timeout=600,
                cwd=str(compose_dir),
            )
            if result.returncode != 0:
                error_msg = result.stderr.strip() or result.stdout.strip()
                for key in service_keys:
                    spec = SERVICE_SPECS.get(key)
                    results["services"].append(
                        {
                            "service": key,
                            "display_name": spec.display_name if spec else key,
                            "status": "compose_failed",
                            "ok": False,
                            "error": error_msg[:500],
                        }
                    )
                    results["summary"]["failed"] += 1
                return results
        except subprocess.TimeoutExpired:
            for key in service_keys:
                spec = SERVICE_SPECS.get(key)
                results["services"].append(
                    {
                        "service": key,
                        "display_name": spec.display_name if spec else key,
                        "status": "compose_timeout",
                        "ok": False,
                        "error": "Compose up timed out (10 min)",
                    }
                )
                results["summary"]["failed"] += 1
            return results

        # Check status of each service
        for key in service_keys:
            spec = SERVICE_SPECS.get(key)
            if not spec:
                continue
            svc_result: Dict[str, Any] = {
                "service": key,
                "display_name": spec.display_name,
            }
            # Update internal state
            status = self.get_status(key)
            if status.get("status") == ServiceStatus.RUNNING.value:
                svc_result["status"] = "running"
                svc_result["ok"] = True
                svc_result["url"] = status.get("url", "")
                results["summary"]["ok"] += 1
            else:
                svc_result["status"] = "degraded"
                svc_result["ok"] = True  # Container was created, may need time
                results["summary"]["ok"] += 1
            if spec.setup_notes:
                svc_result["setup_notes"] = spec.setup_notes
            results["services"].append(svc_result)

        return results

    # ------------------------------------------------------------------ ports

    def _check_port_available(self, port: int) -> bool:
        """Return True if the port is free on localhost."""
        try:
            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
                s.settimeout(1)
                s.bind(("127.0.0.1", port))
            return True
        except OSError:
            return False

    def check_port_conflicts(self, spec: ServiceSpec) -> List[int]:
        """Return list of host ports that are already in use."""
        conflicts = []
        for host_port in spec.ports:
            if not self._check_port_available(host_port):
                conflicts.append(host_port)
        return conflicts

    # ------------------------------------------------------------------ lifecycle

    def _qualify_image(self, image: str) -> str:
        """Fully qualify a Docker Hub image name for Podman compatibility.

        Podman's default registries.conf may not include docker.io as an
        unqualified search registry. Prepending ``docker.io/`` ensures pulls
        work without requiring system-level config changes.

        Images that already have a registry prefix (contain a dot in the
        registry portion, before the first slash) are returned unchanged.
        Examples:
          - ``caddy:2.9`` → ``docker.io/library/caddy:2.9`` (official image)
          - ``pihole/pihole:2024.07.0`` → ``docker.io/pihole/pihole:2024.07.0``
          - ``ghcr.io/mealie-recipes/mealie:v2.6.0`` → unchanged
        """
        if self._runtime == ContainerRuntime.PODMAN:
            # Split off the tag/digest, check only the name portion
            name_part = image.split(":")[0]
            first_segment = name_part.split("/")[0]
            if "." not in first_segment:
                # No registry domain — it's a Docker Hub reference
                if "/" not in name_part:
                    # Official image (e.g. caddy, nextcloud) → docker.io/library/
                    return f"docker.io/library/{image}"
                else:
                    # User image (e.g. pihole/pihole) → docker.io/
                    return f"docker.io/{image}"
        return image

    def pull_image(self, key: str) -> Dict[str, Any]:
        """Pull the container image for a service."""
        spec = SERVICE_SPECS.get(key)
        if not spec:
            return {"success": False, "error": f"Unknown service: {key}"}
        if spec.service_type == ServiceType.NATIVE:
            return {"success": True, "message": "Native service, no image to pull"}

        rt = self.detect_runtime()
        if rt == ContainerRuntime.NONE:
            return {"success": False, "error": "No container runtime found"}

        image = self._qualify_image(spec.image)
        self._set_status(key, ServiceStatus.PULLING)
        try:
            result = subprocess.run(
                [self._runtime_path, "pull", image],
                capture_output=True,
                text=True,
                timeout=600,
            )
            if result.returncode != 0:
                self._set_status(key, ServiceStatus.ERROR, error=result.stderr.strip())
                return {"success": False, "error": result.stderr.strip()}
            self._set_status(key, ServiceStatus.STOPPED)
            return {"success": True, "message": f"Pulled {spec.image}"}
        except subprocess.TimeoutExpired:
            self._set_status(key, ServiceStatus.ERROR, error="Image pull timed out")
            return {"success": False, "error": "Image pull timed out (10 min)"}
        except Exception as e:
            self._set_status(key, ServiceStatus.ERROR, error=str(e))
            return {"success": False, "error": str(e)}

    @staticmethod
    def _detect_selinux() -> bool:
        """Check if SELinux is enabled and enforcing."""
        try:
            result = subprocess.run(
                ["getenforce"],
                capture_output=True,
                text=True,
                timeout=5,
            )
            return result.returncode == 0 and result.stdout.strip().lower() in (
                "enforcing",
                "permissive",
            )
        except (FileNotFoundError, subprocess.TimeoutExpired):
            return False

    def _preflight_check(self, key: str, spec) -> Dict[str, Any]:
        """Run pre-flight checks and auto-fix system issues for provisioning.

        Handles:
        - Privileged port binding (< 1024) for rootless Podman
        - systemd-resolved conflict on port 53 (Pi-hole)
        - vm.max_map_count for OpenSearch/Wazuh Indexer
        """
        import platform

        if platform.system() != "Linux":
            return {"success": True}

        # OpenSearch / Wazuh Indexer needs vm.max_map_count >= 262144
        if key in ("wazuh_indexer",):
            try:
                current_map = int(Path("/proc/sys/vm/max_map_count").read_text().strip())
                if current_map < 262144:
                    result = subprocess.run(
                        ["sudo", "-n", "sysctl", "-w", "vm.max_map_count=262144"],
                        capture_output=True,
                        text=True,
                        timeout=10,
                    )
                    if result.returncode != 0:
                        return {
                            "success": False,
                            "error": (
                                "Wazuh Indexer requires vm.max_map_count >= 262144 "
                                f"(current: {current_map}). Run this command and try again:\n"
                                "  sudo sysctl -w vm.max_map_count=262144\n"
                                "  echo 'vm.max_map_count=262144' | "
                                "sudo tee -a /etc/sysctl.d/99-familiar.conf"
                            ),
                        }
                    # Persist across reboots
                    try:
                        sysctl_conf = Path("/etc/sysctl.d/99-familiar.conf")
                        existing = sysctl_conf.read_text() if sysctl_conf.exists() else ""
                        if "vm.max_map_count" not in existing:
                            subprocess.run(
                                ["sudo", "-n", "tee", "-a", str(sysctl_conf)],
                                input="vm.max_map_count=262144\n",
                                capture_output=True,
                                text=True,
                                timeout=10,
                            )
                    except OSError:
                        pass
                    logger.info("Set vm.max_map_count=262144 for %s", key)
            except (OSError, ValueError) as e:
                logger.debug("vm.max_map_count check failed: %s", e)

        # Check if any host port is privileged (< 1024)
        privileged_ports = [p for p in spec.ports if p < 1024]
        if not privileged_ports:
            return {"success": True}

        # Read current unprivileged port start
        min_port = min(privileged_ports)
        try:
            current = int(Path("/proc/sys/net/ipv4/ip_unprivileged_port_start").read_text().strip())
        except (OSError, ValueError):
            current = 1024

        if current > min_port:
            # Need to lower the threshold — requires sudo
            try:
                result = subprocess.run(
                    [
                        "sudo",
                        "-n",
                        "sysctl",
                        "-w",
                        f"net.ipv4.ip_unprivileged_port_start={min_port}",
                    ],
                    capture_output=True,
                    text=True,
                    timeout=10,
                )
                if result.returncode != 0:
                    # sudo without password failed — try with pkexec or give clear instructions
                    return {
                        "success": False,
                        "error": (
                            f"Port {min_port} is privileged. Run this command and try again:\n"
                            f"  sudo sysctl -w net.ipv4.ip_unprivileged_port_start={min_port}\n"
                            f"  echo 'net.ipv4.ip_unprivileged_port_start={min_port}' | "
                            f"sudo tee -a /etc/sysctl.d/99-familiar.conf"
                        ),
                    }
                # Persist across reboots
                sysctl_conf = Path("/etc/sysctl.d/99-familiar.conf")
                sysctl_line = f"net.ipv4.ip_unprivileged_port_start={min_port}\n"
                try:
                    existing = sysctl_conf.read_text() if sysctl_conf.exists() else ""
                    if f"ip_unprivileged_port_start={min_port}" not in existing:
                        subprocess.run(
                            ["sudo", "-n", "tee", "-a", str(sysctl_conf)],
                            input=sysctl_line,
                            capture_output=True,
                            text=True,
                            timeout=10,
                        )
                except OSError:
                    pass  # Non-critical — runtime sysctl is enough
                logger.info("Set ip_unprivileged_port_start=%d for %s", min_port, key)
            except subprocess.TimeoutExpired:
                return {
                    "success": False,
                    "error": (
                        f"Timed out trying to allow port {min_port}. "
                        f"Run manually: sudo sysctl -w net.ipv4.ip_unprivileged_port_start={min_port}"
                    ),
                }
            except FileNotFoundError:
                return {
                    "success": False,
                    "error": "sudo not found. Cannot configure privileged port access.",
                }

        # Check for systemd-resolved conflict on port 53
        if 53 in privileged_ports:
            try:
                import socket

                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
                    s.settimeout(1)
                    if s.connect_ex(("127.0.0.1", 53)) == 0:
                        # Something is on port 53 — likely systemd-resolved
                        resolved_active = subprocess.run(
                            ["systemctl", "is-active", "systemd-resolved"],
                            capture_output=True,
                            text=True,
                            timeout=5,
                        )
                        if resolved_active.stdout.strip() == "active":
                            # Disable the stub listener, keep resolved running
                            subprocess.run(
                                [
                                    "sudo",
                                    "-n",
                                    "sed",
                                    "-i",
                                    "s/^#\\?DNSStubListener=.*/DNSStubListener=no/",
                                    "/etc/systemd/resolved.conf",
                                ],
                                capture_output=True,
                                text=True,
                                timeout=10,
                            )
                            subprocess.run(
                                ["sudo", "-n", "systemctl", "restart", "systemd-resolved"],
                                capture_output=True,
                                text=True,
                                timeout=15,
                            )
                            logger.info("Disabled systemd-resolved stub listener for Pi-hole")
            except Exception as e:
                logger.debug("Port 53 conflict check: %s", e)

        return {"success": True}

    def _build_env_vars(self, key: str, spec) -> Dict[str, str]:
        """Build environment variables: spec defaults + generated credentials.

        Credentials are generated once and persisted in secrets.json.
        Existing credentials are never overwritten (idempotent).
        Bluesky PDS gets additional hex-encoded keys.
        """
        from familiar.services.credentials import get_service_credentials

        env = dict(spec.env_vars)

        # Generate or load credentials for any declared credential_keys
        if spec.credential_keys:
            creds = get_service_credentials(key, spec.credential_keys)
            env.update(creds)

        # Bluesky PDS needs extra hex-encoded keys beyond token_urlsafe
        if key == "bluesky_pds":
            env.setdefault("PDS_JWT_SECRET", secrets.token_hex(16))
            env.setdefault(
                "PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX",
                secrets.token_hex(32),
            )

        return env

    def provision(self, key: str) -> Dict[str, Any]:
        """Pull image and create a hardened container for the service.

        Security measures applied (per NIST SP 800-190 / OWASP Agentic):
        - --cap-drop=ALL + only spec-declared capabilities added back
        - --security-opt=no-new-privileges:true
        - --read-only filesystem where supported (+ tmpfs for writable dirs)
        - --memory and --cpus resource limits
        - 127.0.0.1 port binding (localhost only by default)
        - Volume directories created with chmod 700
        - SELinux :Z labels when SELinux is detected
        - Credentials generated per-service, stored in secrets.json
        """
        spec = SERVICE_SPECS.get(key)
        if not spec:
            return {"success": False, "error": f"Unknown service: {key}"}

        if spec.service_type == ServiceType.NATIVE:
            self._set_status(key, ServiceStatus.STOPPED)
            return {"success": True, "message": f"{spec.display_name} ready (native)"}

        rt = self.detect_runtime()
        if rt == ContainerRuntime.NONE:
            return {
                "success": False,
                "error": "No container runtime found. Install Podman or Docker.",
            }

        # Pre-flight: handle privileged ports and system conflicts
        preflight = self._preflight_check(key, spec)
        if not preflight.get("success", True):
            return preflight

        # Pull image
        pull = self.pull_image(key)
        if not pull.get("success"):
            return pull

        # Create volume directories — use 755 so rootless Podman's mapped
        # container UID (via subuid) can traverse the directory tree.
        home = str(Path.home())
        svc_dir = Path(home) / ".familiar" / "services" / key
        config_file_paths = set()
        if spec.config_files:
            for rel_path in spec.config_files:
                config_file_paths.add(str(svc_dir / rel_path))
        for host_path in spec.get_resolved_volumes(home):
            p = Path(host_path)
            if str(p) in config_file_paths:
                p.parent.mkdir(parents=True, exist_ok=True)
            else:
                p.mkdir(parents=True, exist_ok=True)
                try:
                    os.chmod(p, 0o755)
                except OSError:
                    pass  # Windows fallback

        # Reclaim ownership from prior deploys before writing configs
        if spec.container_uid and self._runtime == ContainerRuntime.PODMAN:
            for host_path in spec.get_resolved_volumes(home):
                try:
                    subprocess.run(
                        [self._runtime_path, "unshare", "chown", "-R", "0:0", host_path],
                        capture_output=True,
                        text=True,
                        timeout=30,
                    )
                except Exception:
                    pass

        # Write config files into volume dirs
        if spec.config_files:
            svc_dir = Path(home) / ".familiar" / "services" / key
            for rel_path, content in spec.config_files.items():
                cfg_path = svc_dir / rel_path
                cfg_path.parent.mkdir(parents=True, exist_ok=True)
                cfg_path.write_text(content)

        # Set ownership to container UID for rootless Podman
        if spec.container_uid and self._runtime == ContainerRuntime.PODMAN:
            uid = str(spec.container_uid)
            for host_path in spec.get_resolved_volumes(home):
                try:
                    subprocess.run(
                        [self._runtime_path, "unshare", "chown", "-R", f"{uid}:{uid}", host_path],
                        capture_output=True,
                        text=True,
                        timeout=30,
                    )
                except Exception as e:
                    logger.debug("podman unshare chown failed for %s: %s", host_path, e)

        # Build environment variables with generated credentials
        env_vars = self._build_env_vars(key, spec)

        # Detect SELinux for volume label suffix
        selinux = self._detect_selinux()

        # Build create command
        name = spec.default_container_name
        cmd = [self._runtime_path, "create", "--name", name]

        # ── Security flags ──────────────────────────────────────────
        # Drop ALL capabilities, add back only what the service needs
        # Services with no_new_privileges=False need default caps (no drop)
        if spec.no_new_privileges:
            cmd.extend(["--cap-drop", "ALL"])
        for cap in spec.capabilities:
            cmd.extend(["--cap-add", cap])

        # Prevent privilege escalation
        if spec.no_new_privileges:
            cmd.extend(["--security-opt", "no-new-privileges:true"])

        # Read-only root filesystem
        if spec.read_only_root:
            cmd.append("--read-only")

        # Writable tmpfs mounts (for /tmp, /run, etc.)
        for tmpfs in spec.tmpfs_mounts:
            cmd.extend(["--tmpfs", f"{tmpfs}:rw,noexec,nosuid"])

        # Resource limits — prevent one container from killing the host
        if spec.memory_limit:
            cmd.extend(["--memory", spec.memory_limit])
        if spec.cpu_limit:
            cmd.extend(["--cpus", str(spec.cpu_limit)])

        # ── Port mappings ───────────────────────────────────────────
        for host_port, container_port in spec.ports.items():
            if spec.localhost_only:
                cmd.extend(["-p", f"127.0.0.1:{host_port}:{container_port}"])
            else:
                cmd.extend(["-p", f"{host_port}:{container_port}"])

        # ── Volume mounts ───────────────────────────────────────────
        for host_path, container_path in spec.get_resolved_volumes(home).items():
            vol_str = f"{host_path}:{container_path}"
            if selinux:
                vol_str += ":Z"  # Private SELinux label
            cmd.extend(["-v", vol_str])

        # ── Environment variables ───────────────────────────────────
        for env_key, env_val in env_vars.items():
            cmd.extend(["-e", f"{env_key}={env_val}"])

        cmd.extend(["--restart", "unless-stopped"])
        cmd.append(self._qualify_image(spec.image))

        self._set_status(key, ServiceStatus.STARTING)
        try:
            # Remove existing container if any
            subprocess.run(
                [self._runtime_path, "rm", "-f", name],
                capture_output=True,
                text=True,
                timeout=30,
            )

            result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
            if result.returncode != 0:
                self._set_status(key, ServiceStatus.ERROR, error=result.stderr.strip())
                return {"success": False, "error": result.stderr.strip()}

            container_id = result.stdout.strip()
            state = self._get_or_create_state(key)
            state.container_id = container_id
            state.container_name = name
            state.ports = spec.ports
            state.status = ServiceStatus.STOPPED.value
            state.error_message = ""
            self._save_state()

            return {
                "success": True,
                "message": f"{spec.display_name} provisioned",
                "container_id": container_id,
            }
        except subprocess.TimeoutExpired:
            self._set_status(key, ServiceStatus.ERROR, error="Container create timed out")
            return {"success": False, "error": "Container create timed out"}
        except Exception as e:
            self._set_status(key, ServiceStatus.ERROR, error=str(e))
            return {"success": False, "error": str(e)}

    def start(self, key: str) -> Dict[str, Any]:
        """Start a provisioned service."""
        spec = SERVICE_SPECS.get(key)
        if not spec:
            return {"success": False, "error": f"Unknown service: {key}"}

        if spec.service_type == ServiceType.NATIVE:
            return self.start_native(key)

        rt = self.detect_runtime()
        if rt == ContainerRuntime.NONE:
            return {"success": False, "error": "No container runtime found"}

        name = spec.default_container_name
        try:
            result = subprocess.run(
                [self._runtime_path, "start", name],
                capture_output=True,
                text=True,
                timeout=30,
            )
            if result.returncode != 0:
                self._set_status(key, ServiceStatus.ERROR, error=result.stderr.strip())
                return {"success": False, "error": result.stderr.strip()}

            # Determine URL
            url = spec.web_url

            state = self._get_or_create_state(key)
            state.status = ServiceStatus.RUNNING.value
            state.started_at = datetime.now(timezone.utc).isoformat()
            state.url = url
            state.error_message = ""
            self._save_state()

            return {"success": True, "message": f"{spec.display_name} started", "url": url}
        except subprocess.TimeoutExpired:
            self._set_status(key, ServiceStatus.ERROR, error="Start timed out")
            return {"success": False, "error": "Start timed out"}
        except Exception as e:
            self._set_status(key, ServiceStatus.ERROR, error=str(e))
            return {"success": False, "error": str(e)}

    def stop(self, key: str) -> Dict[str, Any]:
        """Stop a running service."""
        spec = SERVICE_SPECS.get(key)
        if not spec:
            return {"success": False, "error": f"Unknown service: {key}"}

        if spec.service_type == ServiceType.NATIVE:
            return self.stop_native(key)

        rt = self.detect_runtime()
        if rt == ContainerRuntime.NONE:
            return {"success": False, "error": "No container runtime found"}

        name = spec.default_container_name
        try:
            result = subprocess.run(
                [self._runtime_path, "stop", name],
                capture_output=True,
                text=True,
                timeout=30,
            )
            if result.returncode != 0:
                self._set_status(key, ServiceStatus.ERROR, error=result.stderr.strip())
                return {"success": False, "error": result.stderr.strip()}

            state = self._get_or_create_state(key)
            state.status = ServiceStatus.STOPPED.value
            state.url = ""
            state.error_message = ""
            self._save_state()

            return {"success": True, "message": f"{spec.display_name} stopped"}
        except subprocess.TimeoutExpired:
            return {"success": False, "error": "Stop timed out"}
        except Exception as e:
            return {"success": False, "error": str(e)}

    def restart(self, key: str) -> Dict[str, Any]:
        """Restart a service."""
        spec = SERVICE_SPECS.get(key)
        if not spec:
            return {"success": False, "error": f"Unknown service: {key}"}

        if spec.service_type == ServiceType.NATIVE:
            self.stop_native(key)
            return self.start_native(key)

        rt = self.detect_runtime()
        if rt == ContainerRuntime.NONE:
            return {"success": False, "error": "No container runtime found"}

        name = spec.default_container_name
        try:
            result = subprocess.run(
                [self._runtime_path, "restart", name],
                capture_output=True,
                text=True,
                timeout=30,
            )
            if result.returncode != 0:
                self._set_status(key, ServiceStatus.ERROR, error=result.stderr.strip())
                return {"success": False, "error": result.stderr.strip()}

            url = spec.web_url

            state = self._get_or_create_state(key)
            state.status = ServiceStatus.RUNNING.value
            state.started_at = datetime.now(timezone.utc).isoformat()
            state.url = url
            state.error_message = ""
            self._save_state()

            return {"success": True, "message": f"{spec.display_name} restarted", "url": url}
        except subprocess.TimeoutExpired:
            return {"success": False, "error": "Restart timed out"}
        except Exception as e:
            return {"success": False, "error": str(e)}

    def remove(self, key: str, remove_data: bool = False) -> Dict[str, Any]:
        """Remove a service container and optionally its data."""
        spec = SERVICE_SPECS.get(key)
        if not spec:
            return {"success": False, "error": f"Unknown service: {key}"}

        if spec.service_type == ServiceType.NATIVE:
            self.stop_native(key)
            self._set_status(key, ServiceStatus.NOT_INSTALLED)
            return {"success": True, "message": f"{spec.display_name} removed"}

        rt = self.detect_runtime()
        if rt == ContainerRuntime.NONE:
            return {"success": False, "error": "No container runtime found"}

        name = spec.default_container_name
        try:
            # Stop first
            subprocess.run(
                [self._runtime_path, "stop", name],
                capture_output=True,
                text=True,
                timeout=30,
            )
            # Remove container
            result = subprocess.run(
                [self._runtime_path, "rm", "-f", name],
                capture_output=True,
                text=True,
                timeout=30,
            )
            if result.returncode != 0:
                self._set_status(key, ServiceStatus.ERROR, error=result.stderr.strip())
                return {"success": False, "error": result.stderr.strip()}

            # Optionally remove data
            if remove_data:
                home = str(Path.home())
                for host_path in spec.get_resolved_volumes(home):
                    p = Path(host_path)
                    if p.exists():
                        shutil.rmtree(p, ignore_errors=True)

            self._set_status(key, ServiceStatus.NOT_INSTALLED)
            state = self._get_or_create_state(key)
            state.container_id = ""
            state.container_name = ""
            state.url = ""
            state.started_at = ""
            self._save_state()

            return {"success": True, "message": f"{spec.display_name} removed"}
        except subprocess.TimeoutExpired:
            return {"success": False, "error": "Remove timed out"}
        except Exception as e:
            return {"success": False, "error": str(e)}

    # ------------------------------------------------------------------ status

    def get_status(self, key: str) -> Dict[str, Any]:
        """Get live status of a service by inspecting its container."""
        spec = SERVICE_SPECS.get(key)
        if not spec:
            return {"key": key, "status": "unknown", "error": f"Unknown service: {key}"}

        if spec.service_type == ServiceType.NATIVE:
            state = self._get_or_create_state(key)
            # Check systemd unit status for native services
            if spec.systemctl_unit:
                try:
                    result = subprocess.run(
                        ["systemctl", "is-active", spec.systemctl_unit],
                        capture_output=True,
                        text=True,
                        timeout=5,
                    )
                    if result.stdout.strip() == "active":
                        state.status = ServiceStatus.RUNNING.value
                        if spec.health_check_port:
                            state.url = spec.web_url
                    elif result.stdout.strip() == "activating":
                        state.status = ServiceStatus.PULLING.value
                    else:
                        # Unit exists but not active — could be socket-activated
                        # Check if the unit file exists (socket activation = installed)
                        check = subprocess.run(
                            ["systemctl", "list-unit-files", spec.systemctl_unit],
                            capture_output=True,
                            text=True,
                            timeout=5,
                        )
                        if spec.systemctl_unit in check.stdout:
                            state.status = ServiceStatus.STOPPED.value
                        # else: leave as not_installed
                    self._save_state()
                except (subprocess.TimeoutExpired, OSError):
                    pass  # Leave state as-is if systemctl unavailable
            return {**state.to_dict(), **spec.to_dict()}

        rt = self.detect_runtime()
        if rt == ContainerRuntime.NONE:
            state = self._get_or_create_state(key)
            return {**state.to_dict(), **spec.to_dict()}

        name = spec.default_container_name
        try:
            result = subprocess.run(
                [self._runtime_path, "inspect", "--format", "json", name],
                capture_output=True,
                text=True,
                timeout=10,
            )
            if result.returncode != 0:
                # Container doesn't exist
                state = self._get_or_create_state(key)
                if state.status not in (ServiceStatus.PULLING.value, ServiceStatus.ERROR.value):
                    state.status = ServiceStatus.NOT_INSTALLED.value
                    self._save_state()
                return {**state.to_dict(), **spec.to_dict()}

            data = json.loads(result.stdout)
            if isinstance(data, list) and data:
                data = data[0]

            container_status = data.get("State", {}).get("Status", "unknown")
            state = self._get_or_create_state(key)

            if container_status == "running":
                state.status = ServiceStatus.RUNNING.value
                url = spec.web_url
                state.url = url
            elif container_status in ("exited", "stopped", "created"):
                state.status = ServiceStatus.STOPPED.value
                state.url = ""
            else:
                state.status = ServiceStatus.ERROR.value

            state.container_id = data.get("Id", "")[:12]
            state.container_name = name
            state.error_message = ""
            self._save_state()

            return {**state.to_dict(), **spec.to_dict()}
        except (json.JSONDecodeError, KeyError, IndexError):
            state = self._get_or_create_state(key)
            return {**state.to_dict(), **spec.to_dict()}
        except Exception:
            state = self._get_or_create_state(key)
            return {**state.to_dict(), **spec.to_dict()}

    def get_all_statuses(self) -> List[Dict[str, Any]]:
        """Get status of all registered services."""
        return [self.get_status(key) for key in SERVICE_SPECS]

    def get_logs(self, key: str, tail: int = 100) -> Dict[str, Any]:
        """Get recent log lines for a container service."""
        spec = SERVICE_SPECS.get(key)
        if not spec:
            return {"success": False, "error": f"Unknown service: {key}", "logs": ""}
        if spec.service_type == ServiceType.NATIVE:
            return {"success": True, "logs": "(Native service — check Familiar logs)"}

        rt = self.detect_runtime()
        if rt == ContainerRuntime.NONE:
            return {"success": False, "error": "No container runtime", "logs": ""}

        name = spec.default_container_name
        try:
            result = subprocess.run(
                [self._runtime_path, "logs", "--tail", str(tail), name],
                capture_output=True,
                text=True,
                timeout=10,
            )
            output = result.stdout or result.stderr or "(no logs)"
            return {"success": True, "logs": output}
        except subprocess.TimeoutExpired:
            return {"success": False, "error": "Log fetch timed out", "logs": ""}
        except Exception as e:
            return {"success": False, "error": str(e), "logs": ""}

    # ------------------------------------------------------------------ health

    def health_check(self, key: str) -> Dict[str, Any]:
        """HTTP health check against a service's health endpoint."""
        spec = SERVICE_SPECS.get(key)
        if not spec:
            return {"healthy": False, "error": f"Unknown service: {key}"}
        if not spec.health_check_port or not spec.health_check_path:
            return {"healthy": False, "error": "No health check configured"}

        try:
            import httpx

            url = f"http://localhost:{spec.health_check_port}{spec.health_check_path}"
            resp = httpx.get(url, timeout=5.0)
            return {"healthy": resp.status_code < 400, "status_code": resp.status_code, "url": url}
        except ImportError:
            # Fall back to stdlib
            import urllib.error
            import urllib.request

            url = f"http://localhost:{spec.health_check_port}{spec.health_check_path}"
            try:
                req = urllib.request.urlopen(url, timeout=5)
                return {"healthy": req.status < 400, "status_code": req.status, "url": url}
            except urllib.error.URLError as e:
                return {"healthy": False, "error": str(e), "url": url}
        except Exception as e:
            return {"healthy": False, "error": str(e)}

    def health_check_with_retry(
        self,
        key: str,
        max_attempts: int = 5,
        initial_delay: float = 2.0,
    ) -> Dict[str, Any]:
        """Health check with exponential backoff retry.

        Waits for a service to become healthy after starting. Returns the
        health check result from the first successful attempt, or the
        last failed attempt if all retries are exhausted.
        """
        import time

        delay = initial_delay
        last_result: Dict[str, Any] = {"healthy": False, "error": "No attempts made"}

        for attempt in range(1, max_attempts + 1):
            last_result = self.health_check(key)
            if last_result.get("healthy"):
                last_result["attempts"] = attempt
                return last_result

            if attempt < max_attempts:
                logger.debug(
                    "Health check %s attempt %d/%d failed, retrying in %.1fs",
                    key,
                    attempt,
                    max_attempts,
                    delay,
                )
                time.sleep(delay)
                delay = min(delay * 1.5, 15.0)  # Cap at 15s

        last_result["attempts"] = max_attempts
        return last_result

    # ------------------------------------------------------------------ native

    def start_native(self, key: str) -> Dict[str, Any]:
        """Start a native (in-process) service."""
        spec = SERVICE_SPECS.get(key)
        if not spec or spec.service_type != ServiceType.NATIVE:
            return {"success": False, "error": "Not a native service"}

        state = self._get_or_create_state(key)
        state.status = ServiceStatus.RUNNING.value
        state.started_at = datetime.now(timezone.utc).isoformat()
        state.error_message = ""
        self._save_state()
        return {"success": True, "message": f"{spec.display_name} started (native)"}

    def stop_native(self, key: str) -> Dict[str, Any]:
        """Stop a native service."""
        spec = SERVICE_SPECS.get(key)
        if not spec or spec.service_type != ServiceType.NATIVE:
            return {"success": False, "error": "Not a native service"}

        state = self._get_or_create_state(key)
        state.status = ServiceStatus.STOPPED.value
        state.url = ""
        state.error_message = ""
        self._save_state()
        return {"success": True, "message": f"{spec.display_name} stopped (native)"}

    # ------------------------------------------------------------------ dependency ordering

    @staticmethod
    def _topo_sort(keys: List[str]) -> List[str]:
        """Return ``keys`` sorted so that each service's ``depends_on`` come first.

        Uses Kahn's algorithm.  Raises ``ValueError`` on a cycle.

        Args:
            keys: Service keys to order.

        Returns:
            Ordered list where dependencies precede dependents.
        """
        from collections import deque

        # Build adjacency and in-degree only for the requested set
        key_set = set(keys)
        in_degree: Dict[str, int] = {k: 0 for k in keys}
        graph: Dict[str, List[str]] = {k: [] for k in keys}

        for key in keys:
            spec = SERVICE_SPECS.get(key)
            if not spec:
                continue
            for dep in spec.depends_on:
                if dep in key_set:
                    graph[dep].append(key)
                    in_degree[key] += 1

        queue: deque = deque(k for k in keys if in_degree[k] == 0)
        order: List[str] = []
        while queue:
            node = queue.popleft()
            order.append(node)
            for neighbor in graph[node]:
                in_degree[neighbor] -= 1
                if in_degree[neighbor] == 0:
                    queue.append(neighbor)

        if len(order) != len(keys):
            raise ValueError("Circular dependency detected in service depends_on graph")
        return order

    def start_ordered(self, keys: List[str]) -> List[Dict[str, Any]]:
        """Start multiple services in dependency order.

        Services listed in ``depends_on`` of other requested services will be
        started first.  Each service is started sequentially to preserve
        startup ordering guarantees (e.g. Wazuh indexer must not start before
        the manager is accepting connections).

        Args:
            keys: Service keys to start.

        Returns:
            List of per-service result dicts from :meth:`start`.
        """
        try:
            ordered = self._topo_sort(keys)
        except ValueError as e:
            logger.error("Dependency ordering failed: %s", e)
            ordered = keys  # best-effort fallback

        results = []
        for key in ordered:
            results.append(self.start(key))
        return results

    # ------------------------------------------------------------------ state persistence

    def _get_or_create_state(self, key: str) -> ServiceState:
        if key not in self._states:
            self._states[key] = ServiceState(key=key)
        return self._states[key]

    def _set_status(
        self,
        key: str,
        status: ServiceStatus,
        error: str = "",
    ) -> None:
        state = self._get_or_create_state(key)
        state.status = status.value
        state.error_message = error
        self._save_state()

    def _load_state(self) -> None:
        with self._lock:
            if not STATE_FILE.exists():
                return
            try:
                with open(STATE_FILE) as f:
                    data = json.load(f)
                for key, vals in data.items():
                    self._states[key] = ServiceState(
                        key=key, **{k: v for k, v in vals.items() if k != "key"}
                    )
            except (json.JSONDecodeError, OSError, TypeError) as e:
                logger.warning("Failed to load service state: %s", e)

    def _save_state(self) -> None:
        with self._lock:
            try:
                SERVICES_DIR.mkdir(parents=True, exist_ok=True)
                data = {key: state.to_dict() for key, state in self._states.items()}
                with open(STATE_FILE, "w") as f:
                    json.dump(data, f, indent=2)
            except OSError as e:
                logger.warning("Failed to save service state: %s", e)


# =============================================================================
# SINGLETON
# =============================================================================

_service_manager: Optional[ServiceManager] = None
_manager_lock = threading.Lock()


def get_service_manager() -> ServiceManager:
    """Get the singleton service manager."""
    global _service_manager
    if _service_manager is None:
        with _manager_lock:
            if _service_manager is None:
                _service_manager = ServiceManager()
    return _service_manager
