# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# ORCID: 0009-0006-4957-0540
# Email: Georgescottfoley@proton.me
# Licensed under the MIT License - see LICENSE file for details

"""
Familiar - Main Agent Class (Coordinator)

Thin coordinator that delegates to focused managers:
- ToolExecutor       (tool_executor.py)  — tool execution, RBAC, retry, metrics
- ProviderManager    (provider_manager.py) — provider init, routing, fallback
- PromptBuilder      (prompt_builder.py)  — system prompt, context injection
- MemoryManager      (memory_manager.py)  — episodic memory, remember/recall
- StatusReporter     (status_reporter.py) — status, health, metrics, probes

All public attributes remain on Agent for backward compatibility.
Managers read live state through back-references (self._agent).

Phase 1 (v1.6): Multi-user support via UserContext
Phase 2.1 (v2.1): Input sanitization and structured exceptions
"""

import logging
import os
import time
import uuid
from datetime import datetime
from datetime import timezone as _dt_timezone
from pathlib import Path
from typing import Any, Dict, Generator, Optional

from .backup import (
    BackupConfig,
    BackupManager,
    BackupManifest,
    BackupType,
)
from .bus import (
    SkillBus,
    set_skill_bus,
)
from .config import Config, load_config
from .context import (
    EmbeddingProvider as ContextEmbeddingProvider,
)
from .context import (
    InMemoryVectorStore,
    RetrievalConfig,
    SemanticRetriever,
)
from .embeddings import (
    EmbeddingProvider as RawEmbeddingProvider,
)
from .embeddings import (
    TFIDFEmbeddings,
    get_embedding_provider,
)
from .event_emitter import EventEmitter
from .exceptions import (
    BudgetExceededError,
    SessionExpiredError,
)
from .experiments import (
    ExperimentManager,
    set_experiment_manager,
)
from .guardrails import (
    GuardrailConfig,
    Guardrails,
    GuardrailViolation,
    UsageTracker,
    ViolationType,
)
from .health import (
    DependencyHealth,
    DependencyType,
    HealthChecker,
    create_memory_agent_health_check,
    create_planner_health_check,
)
from .memory import ConversationHistory, Memory
from .memory_agent import MemoryExtractor

# Mesh networking — lazy imports inside methods to avoid port binding at module load
# Full imports happen in start_mesh() and __init__ (gateway object only)
from .mesh import (
    MeshGateway,
)
from .metrics import (
    MetricsCollector,
    get_metrics_collector,
)
from .observability import calculate_cost
from .orchestration import (
    Orchestrator,
)
from .planner import TaskPlanner
from .providers import (
    LLMResponse,
    StreamEvent,
    StreamEventType,
)

try:
    from .providers import OllamaProvider as _OllamaProvider
except ImportError:
    _OllamaProvider = None
from .agent_response import AgentResponse

# ── Decomposed managers ──────────────────────────────────────────────────────
from .memory_manager import (
    MemoryManager,
    _get_episodic_encryption,  # noqa: F401 — re-export
)
from .prompt_builder import PromptBuilder
from .provider_manager import ProviderManager
from .resilience import RetryConfig
from .saga import (
    InMemorySagaLog,
    Saga,
    SagaExecution,
    SagaOrchestrator,
)

# Phase 2: Input sanitization and exceptions
from .sanitization import SanitizationLevel, sanitize_user_input
from .scheduler import TaskFrequency, get_scheduler
from .security import (
    Capability,
    SecureSession,
    SecurityConfig,
    SecurityMode,
    SessionManager,
    TrustLevel,
    is_encryption_enabled,
)
from .skills import SkillLoader
from .status_reporter import StatusReporter
from .structured_output import (
    ExtractionMethod,
    StructuredGenerator,
    ValidationStrategy,
)

# Re-export _capture_context_from_result so channels can still import it
# from familiar.core.agent (backward compat).
from .tool_executor import (
    ToolExecutor,
    _capture_context_from_result,  # noqa: F401
)
from .tool_router import RouterConfig, ToolRouter
from .tools import get_tool_registry

# Phase 1: Multi-user support
try:
    from .data_store import DataStore, get_data_store
    from .users import User, UserContext, UserRole

    HAS_USER_MANAGEMENT = True
except ImportError:
    HAS_USER_MANAGEMENT = False
    UserContext = None
    UserRole = None
    User = None
    DataStore = None
    get_data_store = None

# Import compliance module (optional)
try:
    from .compliance import (
        ComplianceMode,
        validate_compliance,
    )

    COMPLIANCE_AVAILABLE = True
except ImportError:
    COMPLIANCE_AVAILABLE = False
    ComplianceMode = None

logger = logging.getLogger(__name__)


class _GuardrailBlock(Exception):
    """
    Internal sentinel raised by _setup_request when a guardrail fires before
    the session is established.  Caught in chat() and chat_stream() and
    converted directly to a user-facing string — never propagates to callers.
    """


class Agent:
    """
    The main agent class.

    Usage:
        agent = Agent()
        response = agent.chat("Hello!", user_id="user123", channel="telegram")

    Security:
        Each user gets a SecureSession with trust level, capabilities, and budget.
        Tools are checked against capabilities before execution.
    """

    def __init__(
        self, config: Optional[Config] = None, security_mode: Optional[SecurityMode] = None
    ):
        self.config = config or load_config()

        # Initialize compliance mode
        self.compliance_mode = None
        self._compliance_profile = None
        self._audit_encrypted_storage = None
        self._compliance_audit_logger = None
        self._validate_compliance()

        # Initialize security
        self.security_mode = security_mode or SecurityMode(
            getattr(self.config.agent, "security_mode", "balanced")
        )
        self.sessions = SessionManager(self.security_mode)

        # Initialize components
        self.memory = Memory() if self.config.agent.memory_enabled else None
        self.history = ConversationHistory(max_per_chat=self.config.agent.max_conversation_history)

        # Initialize tool registry with sandboxed directories from config
        self.tools = get_tool_registry(sandboxed_dirs=self.config.agent.sandboxed_directories)

        self.skills = SkillLoader(self.config.agent.skills_dirs)
        self.scheduler = get_scheduler() if self.config.agent.scheduler_enabled else None

        # Set agent reference on scheduler for built-in handlers
        if self.scheduler:
            self.scheduler.set_agent(self)

            # Register compliance cleanup task when compliance mode is active
            if self.compliance_mode and self._compliance_profile:
                existing = self.scheduler.list_tasks()
                has_cleanup = any(t["name"] == "compliance_cleanup" for t in existing)
                if not has_cleanup:
                    from .scheduler import TaskFrequency, TaskType

                    retention = self._compliance_profile.data_retention_days or 365
                    self.scheduler.add_task(
                        name="compliance_cleanup",
                        description=f"Compliance data retention cleanup (max_age={retention} days)",
                        frequency=TaskFrequency.DAILY,
                        task_type=TaskType.CLEANUP,
                        payload={"max_age_days": retention},
                        time_of_day="03:00",
                    )
                    logger.info(f"Registered compliance_cleanup task (retention={retention} days)")

            # Auto-start the scheduler thread here so backup tasks and other
            # registered tasks fire on all deployments — not just CLI/Telegram.
            # Channels that want to register delivery callbacks before any task
            # fires can still call start_scheduler() again (idempotent — the
            # thread-alive check inside prevents double-starting the loop).
            self.scheduler.start()

        # Initialize tool router — pre-filters the tool list per message so the
        # LLM receives only the most relevant subset rather than all 189+ tools
        # a TRUSTED user would otherwise see.  Works without TOOLS.yaml guidance
        # via schema-based keyword scoring.
        #
        # Local models (Ollama) are much slower per input token, so we send
        # fewer tools to keep inference fast on CPU-only hardware.
        _is_local = (
            _OllamaProvider is not None
            and hasattr(self, "provider")
            and isinstance(self.provider, _OllamaProvider)
        ) or getattr(getattr(self.config, "llm", None), "default_provider", "") == "ollama"
        self.tool_router = ToolRouter(
            guidance=None,
            tool_registry=self.tools,
            config=RouterConfig(
                max_tools=6 if _is_local else 15,
                fallback_count=10 if _is_local else 20,
                confidence_threshold=0.10,
                min_routing_confidence=0.15,
                enabled=True,
            ),
        )

        # Initialize observability
        self.tracer = None  # Tracer disabled for compatibility

        # Initialize resilience config
        self.retry_config = (
            RetryConfig(
                max_attempts=self.config.resilience.max_attempts,
                base_delay=self.config.resilience.base_delay,
                max_delay=self.config.resilience.max_delay,
                exponential_base=self.config.resilience.exponential_base,
                jitter=self.config.resilience.jitter,
            )
            if self.config.resilience.enabled
            else None
        )

        # ── ProviderManager ────────────────────────────────────────────────────
        self._provider_manager = ProviderManager(self)
        self.capacity_tracker = self._provider_manager.capacity_tracker
        self.provider = self._provider_manager.init_provider()
        self._provider_manager.init_model_router()

        # ── ToolExecutor ──────────────────────────────────────────────────────
        self._tool_executor = ToolExecutor(self)

        # Initialize task planner (uses provider for plan generation)
        planning_enabled = getattr(self.config.agent, "planning_enabled", True)

        # Initialize agentic memory extractor (async post-conversation)
        extraction_enabled = getattr(self.config.agent, "memory_extraction_enabled", True)

        # Try to get a lightweight provider for background tasks.
        # Falls back to main provider if none configured.
        self._lightweight_provider = None
        if planning_enabled or extraction_enabled:
            try:
                from .providers import get_lightweight_provider

                self._lightweight_provider = get_lightweight_provider(self.config)
            except Exception as e:
                logger.debug(f"No lightweight provider: {e}")

        bg_provider = self._lightweight_provider or self.provider

        self.planner = (
            TaskPlanner(
                provider=bg_provider,
                enabled=planning_enabled,
            )
            if planning_enabled
            else None
        )

        self.memory_extractor = (
            MemoryExtractor(
                provider=bg_provider,
                memory=self.memory,
                enabled=extraction_enabled and self.memory is not None,
            )
            if extraction_enabled
            else None
        )

        # Load skills
        if self.config.agent.skills_enabled:
            self.skills.load_all()
            # Rebuild router index now that skill tools are registered
            self.tool_router._build_index()

        # Guardrails — content safety, PII, rate limits, token/cost/iteration caps.
        #
        # Limits come from agent config where present, falling back to sensible
        # defaults that match the existing session-level budget system.
        # All limits are permissive by default so existing behaviour is unchanged
        # unless the operator explicitly tightens them.
        guardrail_cfg = GuardrailConfig(
            max_tokens_per_request=getattr(self.config.agent, "guardrail_max_tokens", 200_000),
            max_cost_per_request=getattr(self.config.agent, "guardrail_max_cost_request", 2.00),
            max_cost_per_hour=getattr(self.config.agent, "guardrail_max_cost_hour", 20.00),
            max_cost_per_day=getattr(self.config.agent, "guardrail_max_cost_day", 100.00),
            max_iterations=getattr(self.config.agent, "max_iterations", 15),
            max_requests_per_minute=getattr(self.config.agent, "guardrail_rpm", 30),
            request_timeout_seconds=getattr(self.config.agent, "guardrail_timeout", 300.0),
            enable_content_filter=True,
            enable_pii_detection=getattr(self.config.agent, "enable_pii_detection", False),
            blocked_tools=getattr(self.config.agent, "blocked_tools", []),
            require_approval_tools=getattr(self.config.agent, "require_approval_tools", []),
        )
        self.guardrails = Guardrails(guardrail_cfg)

        # Propagate compliance mode to guardrails (HIPAA → PII BLOCK)
        if self.compliance_mode:
            self.guardrails.set_compliance_mode(self.compliance_mode)

        logger.debug(
            f"Guardrails active: rpm={guardrail_cfg.max_requests_per_minute} "
            f"max_iter={guardrail_cfg.max_iterations} "
            f"pii={'on' if guardrail_cfg.enable_pii_detection else 'off'}"
        )

        # ── MemoryManager ─────────────────────────────────────────────────────
        self._memory_manager = MemoryManager(self)

        # SagaOrchestrator — shared across requests, thread-safe internally.
        # Tools call tool_context["sagas"].run(saga, context) (sync bridge).
        # Skills register sagas at load time via agent.sagas.register(saga).
        # Uses InMemorySagaLog by default; swap for FileSagaLog for persistence.
        self.sagas = SagaOrchestrator(
            saga_log=InMemorySagaLog(),
            default_timeout=getattr(self.config.agent, "saga_default_timeout", 300.0),
        )
        logger.debug("SagaOrchestrator active (InMemorySagaLog)")

        # ExperimentManager — feature flags and A/B experiments.
        # Persists to disk so flags survive restarts.
        # Skills register flags at load time: agent.experiments.create_flag("my_feature")
        # Tool handlers check flags: tool_context["experiments"].is_enabled("flag", user_id)
        _exp_dir = Path.home() / ".familiar" / "experiments"
        _exp_dir.mkdir(parents=True, exist_ok=True)
        _exp_path = str(_exp_dir / "experiments.json")
        self.experiments = ExperimentManager(persistence_path=_exp_path)
        set_experiment_manager(self.experiments)
        logger.debug(f"ExperimentManager active: store={_exp_path}")

        # ── Orchestrator ──────────────────────────────────────────────────────
        # Multi-agent coordination: HIERARCHICAL, CONSENSUS, DEBATE strategies.
        # RemoteAgents (mesh peers) auto-register here when they connect.
        self.orchestrator = Orchestrator()

        # ── SkillBus ──────────────────────────────────────────────────────────
        # Inter-skill communication: pub/sub events, direct invocation with
        # retry + dead-letter queue, shared state, workflow orchestration.
        #
        # Skills register actions at load time:
        #   agent.bus.register_skill("donor", {"intake": handle_intake})
        #
        # Tools fire events after execution (tool.executed topic).
        # Subscribers react without the LLM needing to chain calls manually:
        #   agent.bus.subscribe("donor.intake.completed", trigger_crm_update)
        #   agent.bus.subscribe("triage.case.created", assign_to_volunteer)
        #
        # Module-level publish()/subscribe()/invoke() route to this instance.
        self.bus = SkillBus(
            async_processing=True,
            dead_letter_queue_size=1000,
        )
        set_skill_bus(self.bus)  # wire module-level helpers
        logger.debug("SkillBus active: pub/sub, invoke, workflows, shared state, DLQ")

        # ── SemanticRetriever (RAG pipeline) ──────────────────────────────────
        # Chunked document retrieval for arbitrary knowledge sources:
        # org handbooks, grant guidelines, donor intake forms, policy docs.
        #
        # Uses the same embedding provider cascade as memory:
        #   Ollama (nomic-embed-text) → ONNX → HTTP endpoint → Voyage → OpenAI → TF-IDF
        #
        # Skills and tools add documents at load time or runtime:
        #   agent.retriever.add_text("Grant eligibility: ...", metadata={"source": "smith_foundation"})
        #
        # The current message is used to retrieve top-k relevant chunks
        # that are injected into the LLM system prompt alongside memory.
        #
        # Accessible as tool_context["retriever"] — tools add docs on demand:
        #   tool_context["retriever"].add_text(fetched_doc, metadata={"url": url})
        #   results = tool_context["retriever"].retrieve("query", top_k=5)
        self.retriever: SemanticRetriever = self._init_retriever()
        _emb_name = (
            type(self.memory._embedder).__name__
            if (self.memory and self.memory._embedder)
            else "TFIDFEmbeddings"
        )
        logger.debug(f"SemanticRetriever active: {_emb_name} embeddings")

        # ── HealthChecker ─────────────────────────────────────────────────────
        # Liveness / readiness / startup probes for Kubernetes, Docker, Fly.io.
        # Agent-aware checks registered here:
        #   filesystem   — always registered (built-in)
        #   llm_provider — checks provider connectivity
        #   memory_agent — checks background extraction thread
        #   planner      — checks for stuck plans
        #   bus          — checks SkillBus worker thread
        #
        # Expose via tool_context["health"] so skills can register custom checks.
        # Call agent.mark_ready() once your entry point finishes init.
        #
        # HTTP probes (Kubernetes):
        #   GET /healthz         → liveness_check()
        #   GET /readyz          → readiness_check()
        #   GET /startupz        → startup_check()
        #   GET /health          → check_health() (full report)
        # ── MetricsCollector ──────────────────────────────────────────────────
        # Per-call tool latency (p50/p95/p99), success rates, error counts.
        # Per-call LLM latency, token usage, cost tracking.
        # Custom counters, gauges, histograms for skill-level metrics.
        # Alert thresholds fire callbacks when failure rates exceed limits.
        #
        # Automatically records every tool call via _execute_tool_secure.
        # Automatically records every LLM chat call in _run_agent_loop.
        # Accessible via tool_context["metrics"] for skill-level recording.
        self.metrics: MetricsCollector = get_metrics_collector()
        logger.debug("MetricsCollector active: tool + LLM call recording enabled")

        # ── BackupManager ─────────────────────────────────────────────────────
        # Encrypted incremental backups of all user data:
        #   memory.json           — semantic facts
        #   episodic/*.json(.enc) — per-user conversation history
        #   history.json          — global conversation log
        #   users.db              — user database
        #   analytics.db          — analytics
        #   skills/               — skill data
        #
        # Disabled by default (no-op BackupManager) — enable via config:
        #   config.backup.enabled = True
        #   config.backup.encrypt = True
        #   config.backup.encryption_key = Fernet.generate_key().decode()
        #   config.backup.path = Path("/var/backups/familiar")
        #
        # Manual trigger: agent.create_backup()
        # Restore:        agent.restore_backup(backup_id)
        # Health check:   agent.get_health_status()  (backup_freshness dep)
        self.backup: BackupManager = self._init_backup()
        logger.debug("BackupManager active: data protection enabled")

        # Register scheduled backup task when both backup and scheduler enabled.
        # Safe to call here: scheduler exists (wired above), backup exists,
        # episodic_dir is set. _schedule_backup is a no-op if scheduler=None.
        if getattr(self.backup.config, "enabled", False) and self.scheduler is not None:
            self._schedule_backup()

        # ── HealthChecker — MUST come after backup, metrics, retriever ─────────
        # _init_health_checker references self.backup, self.bus, self.planner,
        # self.memory — all must exist before this call.
        self.health: HealthChecker = self._init_health_checker()
        logger.debug("HealthChecker active: liveness/readiness/startup probes registered")

        # ── Mesh Gateway ──────────────────────────────────────────────────────
        # NOT started here — mesh binds a port and needs an event loop.
        # Call agent.start_mesh() from your entry point when ready.
        #
        # Enabled by:  FAMILIAR_MESH_ENABLED=true  (env var)
        #              or config.agent.mesh_enabled = True
        #
        # Default port: 18789 (configurable via FAMILIAR_MESH_PORT)
        _mesh_enabled = getattr(self.config.agent, "mesh_enabled", False) or os.environ.get(
            "FAMILIAR_MESH_ENABLED", ""
        ).lower() in ("1", "true", "yes")
        _mesh_host = os.environ.get("FAMILIAR_MESH_HOST", "127.0.0.1")
        _mesh_port = int(os.environ.get("FAMILIAR_MESH_PORT", "18789"))

        self.mesh_gateway: Optional[MeshGateway] = None
        self._mesh_enabled = _mesh_enabled
        self._mesh_host = _mesh_host
        self._mesh_port = _mesh_port

        if _mesh_enabled:
            self.mesh_gateway = MeshGateway(self, host=_mesh_host, port=_mesh_port)
            # Wire orchestrator so RemoteAgents auto-register on peer connect
            self.mesh_gateway.set_orchestrator(self.orchestrator)
            # Wire memory bridge if memory is available
            if self.memory is not None:
                hipaa = getattr(self.config.agent, "hipaa_enabled", False) or os.environ.get(
                    "FAMILIAR_HIPAA", ""
                ).lower() in ("1", "true", "yes")
                self.mesh_gateway.init_memory_bridge(self.memory, hipaa_enabled=hipaa)
            logger.info(
                f"Mesh gateway configured: {_mesh_host}:{_mesh_port} "
                f"(call start_mesh() to activate)"
            )
        else:
            logger.debug("Mesh disabled. Set FAMILIAR_MESH_ENABLED=true to enable.")

        # Log initialization summary
        encryption_status = "encrypted" if is_encryption_enabled() else "unencrypted"
        compliance_status = self.compliance_mode.value if self.compliance_mode else "none"
        logger.info(
            f"Agent initialized: provider={self.provider.name}, "
            f"security={self.security_mode.value}, "
            f"compliance={compliance_status}, "
            f"storage={encryption_status}"
        )

        # ── PromptBuilder ─────────────────────────────────────────────────────
        self._prompt_builder = PromptBuilder(self)

        # ── StatusReporter ────────────────────────────────────────────────────
        self._status_reporter = StatusReporter(self)

        # Mark startup complete — readiness probe returns ready=True from here.
        self.health.mark_startup_complete()

    # ════════════════════════════════════════════════════════════════
    # BACKWARD-COMPAT DELEGATES — managers now own the logic
    # ════════════════════════════════════════════════════════════════

    # ── ProviderManager properties ────────────────────────────────────────
    @property
    def model_router(self):
        return self._provider_manager.model_router

    @model_router.setter
    def model_router(self, value):
        self._provider_manager.model_router = value

    @property
    def _routing_mode(self):
        return self._provider_manager._routing_mode

    @_routing_mode.setter
    def _routing_mode(self, value):
        self._provider_manager._routing_mode = value

    @property
    def _manual_routes(self):
        return self._provider_manager._manual_routes

    @_manual_routes.setter
    def _manual_routes(self, value):
        self._provider_manager._manual_routes = value

    # ── ToolExecutor delegates ────────────────────────────────────────────
    def set_tool_event_callback(self, user_id, callback):
        self._tool_executor.set_tool_event_callback(user_id, callback)

    def clear_tool_event_callback(self, user_id):
        self._tool_executor.clear_tool_event_callback(user_id)

    def _execute_tool_secure(self, *args, **kwargs):
        return self._tool_executor.execute_tool_secure(*args, **kwargs)

    def _execute_tools(self, *args, **kwargs):
        return self._tool_executor.execute_tools(*args, **kwargs)

    def _get_allowed_tools(self, *args, **kwargs):
        return self._tool_executor.get_allowed_tools(*args, **kwargs)

    def _get_skill_for_tool(self, tool_name):
        return self._tool_executor.get_skill_for_tool(tool_name)

    # ── ProviderManager delegates ─────────────────────────────────────────
    def _init_provider(self):
        return self._provider_manager.init_provider()

    def _init_model_router(self):
        return self._provider_manager.init_model_router()

    def _select_provider(self, message, session):
        return self._provider_manager.select_provider(message, session)

    def _provider_fallback(self, active_provider, primary_error):
        return self._provider_manager.provider_fallback(active_provider, primary_error)

    def get_routing_config(self):
        return self._provider_manager.get_routing_config()

    def set_routing_config(self, mode, routes=None):
        return self._provider_manager.set_routing_config(mode, routes)

    def switch_provider(self, provider_name):
        return self._provider_manager.switch_provider(provider_name)

    # ── PromptBuilder delegates ───────────────────────────────────────────
    def get_system_prompt(
        self,
        session=None,
        user_context=None,
        current_message=None,
        channel="default",
        is_proactive=False,
        routed_skills=None,
    ):
        return self._prompt_builder.build_system_prompt(
            session, user_context, current_message, channel, is_proactive, routed_skills
        )

    # ── MemoryManager delegates ───────────────────────────────────────────
    @property
    def _episodic_enabled(self):
        return self._memory_manager._episodic_enabled

    @_episodic_enabled.setter
    def _episodic_enabled(self, value):
        self._memory_manager._episodic_enabled = value

    @property
    def _episodic_dir(self):
        return self._memory_manager._episodic_dir

    @_episodic_dir.setter
    def _episodic_dir(self, value):
        self._memory_manager._episodic_dir = value

    @property
    def _episodic_stores(self):
        return self._memory_manager._episodic_stores

    @_episodic_stores.setter
    def _episodic_stores(self, value):
        self._memory_manager._episodic_stores = value

    def _get_episodic_memory(self, user_id):
        return self._memory_manager.get_episodic_memory(user_id)

    def _save_episodic_memory_async(self, user_id):
        return self._memory_manager.save_episodic_memory_async(user_id)

    def remember(self, key, value, category="fact"):
        self._memory_manager.remember(key, value, category)

    def recall(self, query):
        return self._memory_manager.recall(query)

    def clear_history(self, user_id="default", channel="default"):
        self._memory_manager.clear_history(user_id, channel)

    def _get_user_history(self, user_context, channel, limit=20):
        return self._memory_manager.get_user_history(user_context, channel, limit)

    def _save_user_history(self, user_context, channel, user_message, assistant_response):
        self._memory_manager.save_user_history(
            user_context, channel, user_message, assistant_response
        )

    # ── StatusReporter delegates ──────────────────────────────────────────
    def get_status(self):
        return self._status_reporter.get_status()

    def get_health_status(self, include_resources=True):
        return self._status_reporter.get_health_status(include_resources)

    def get_metrics_summary(
        self, tool_name=None, include_llm=True, include_tools=True, include_alerts=True
    ):
        return self._status_reporter.get_metrics_summary(
            tool_name, include_llm, include_tools, include_alerts
        )

    def get_bus_status(self):
        return self._status_reporter.get_bus_status()

    def get_mesh_status(self):
        return self._status_reporter.get_mesh_status()

    def get_retriever_status(self):
        return self._status_reporter.get_retriever_status()

    def mark_ready(self):
        self._status_reporter.mark_ready()

    def liveness(self):
        return self._status_reporter.liveness()

    def readiness(self):
        return self._status_reporter.readiness()

    def startup_probe(self):
        return self._status_reporter.startup_probe()

    # ── Error helpers (backward compat) ───────────────────────────────────
    _ERROR_PREFIXES = ToolExecutor._ERROR_PREFIXES
    _ERROR_SUBSTRINGS = ToolExecutor._ERROR_SUBSTRINGS
    _ERROR_CLASSES = ToolExecutor._ERROR_CLASSES

    def _is_tool_error(self, result):
        return self._tool_executor._is_tool_error(result)

    def _classify_error(self, error_text):
        return self._tool_executor._classify_error(error_text)

    def _format_tool_error(self, tool_name, tool_input, error_text, attempt, max_attempts):
        return self._tool_executor._format_tool_error(
            tool_name, tool_input, error_text, attempt, max_attempts
        )

    def _format_tool_exhausted(self, tool_name, attempts):
        return self._tool_executor._format_tool_exhausted(tool_name, attempts)

    # ════════════════════════════════════════════════════════════════
    # ORIGINAL METHODS THAT STAY IN agent.py
    # ════════════════════════════════════════════════════════════════

    def run_saga(
        self,
        saga: "Saga",
        context: Optional[Dict[str, Any]] = None,
        timeout: Optional[float] = None,
    ) -> "SagaExecution":
        """
        Execute a saga synchronously from sync tool-handler context.

        SagaOrchestrator.execute() is async. Tool handlers run in the agent's
        sync call path, so a direct ``await`` is not possible. This bridge
        runs the coroutine in a dedicated thread with its own event loop —
        safe regardless of whether an outer loop is running.

        Args:
            saga:    Saga definition (built with SagaBuilder or inline).
            context: Initial context dict passed to every step.
            timeout: Override saga-level timeout (seconds). None = saga default.

        Returns:
            SagaExecution with .status, .current_context, .step_executions, etc.

        Example (inside a tool handler)::

            def process_donor_intake(input_data, tool_context):
                agent = tool_context["agent"]
                saga = (
                    SagaBuilder("donor_intake")
                    .add_step("validate",  validate_donor,   undo_validate)
                    .add_step("crm_add",   add_to_crm,       remove_from_crm)
                    .add_step("welcome",   send_welcome,      None)
                    .add_step("schedule",  schedule_followup, cancel_followup)
                    .build()
                )
                execution = agent.run_saga(saga, context={
                    "donor": input_data,
                    "user_id": tool_context["user_id"],
                })
                if execution.status == SagaStatus.COMPLETED:
                    return json.dumps({"ok": True, "id": execution.execution_id})
                return json.dumps({"error": execution.error, "status": execution.status})
        """
        import asyncio
        import concurrent.futures

        async def _run():
            if timeout is not None:
                # Temporarily override saga timeout for this call
                original = saga.timeout_seconds
                saga.timeout_seconds = timeout
                try:
                    return await self.sagas.execute(saga, context)
                finally:
                    saga.timeout_seconds = original
            return await self.sagas.execute(saga, context)

        # Always run in a new thread with its own loop — avoids "cannot run
        # nested event loop" errors when called from within an existing loop.
        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
            fut = pool.submit(lambda: asyncio.run(_run()))
            eff_timeout = timeout or saga.timeout_seconds + 5  # +5s grace
            try:
                return fut.result(timeout=eff_timeout)
            except concurrent.futures.TimeoutError:
                logger.error(f"run_saga timed out after {eff_timeout}s: {saga.name}")
                # Return a synthetic timeout execution
                exec_ = SagaExecution(saga_name=saga.name)
                exec_.mark_timeout()
                return exec_

    # ── Mesh lifecycle ──────────────────────────────────────────────────────

    async def start_mesh(
        self,
        host: Optional[str] = None,
        port: Optional[int] = None,
    ) -> None:
        """
        Start the mesh gateway WebSocket server.

        Must be called from an async context (e.g. inside an asyncio.run() or
        existing event loop). Binds the port and begins accepting connections.

        If mesh was not enabled at __init__ time (FAMILIAR_MESH_ENABLED not set),
        this method will create and start a gateway on demand.

        Args:
            host: Override the host (default: value from env/config)
            port: Override the port (default: value from env/config)

        Example::

            async def main():
                agent = Agent()
                await agent.start_mesh()          # runs forever
                # or run other tasks alongside:
                await asyncio.gather(
                    agent.start_mesh(),
                    my_telegram_bot.run(),
                )
        """
        import asyncio as _asyncio

        _host = host or self._mesh_host
        _port = port or self._mesh_port

        if self.mesh_gateway is None:
            self.mesh_gateway = MeshGateway(self, host=_host, port=_port)
            self.mesh_gateway.set_orchestrator(self.orchestrator)
            if self.memory is not None:
                hipaa = getattr(self.config.agent, "hipaa_enabled", False) or os.environ.get(
                    "FAMILIAR_HIPAA", ""
                ).lower() in ("1", "true", "yes")
                self.mesh_gateway.init_memory_bridge(self.memory, hipaa_enabled=hipaa)

        logger.info(f"Starting mesh gateway on {_host}:{_port}")
        await self.mesh_gateway.start()

        try:
            while True:
                await _asyncio.sleep(1)
        except (KeyboardInterrupt, _asyncio.CancelledError):
            logger.info("Mesh gateway shutting down...")
            await self.mesh_gateway.stop()

    async def stop_mesh(self) -> None:
        """Stop the mesh gateway and disconnect all peers."""
        if self.mesh_gateway and self.mesh_gateway._running:
            await self.mesh_gateway.stop()
            logger.info("Mesh gateway stopped")

    def get_mesh_context(self, query: str, timeout: float = 1.5) -> str:
        """
        Query peer memories relevant to the current conversation turn.

        Called from get_system_prompt() when mesh is active and memory
        bridge is wired. Returns a formatted string injected into the
        LLM system prompt so the model sees cross-device context.

        Returns "" if mesh is not active, HIPAA mode is on, or no
        relevant memories are found within the timeout.

        Args:
            query:   The user's current message (used as search query)
            timeout: Max seconds to wait for peer responses (default 1.5)
        """
        if not self.mesh_gateway:
            return ""
        bridge = getattr(self.mesh_gateway, "memory_bridge", None)
        if bridge is None:
            return ""
        try:
            return bridge.get_mesh_context(query, timeout=timeout)
        except Exception as e:
            logger.debug(f"Mesh context fetch failed: {e}")
            return ""

    # ════════════════════════════════════════════════════════════════
    # BACKUP
    # ════════════════════════════════════════════════════════════════

    def _init_backup(self) -> BackupManager:
        """
        Create and configure the BackupManager for this agent.

        Configuration comes from config.backup (if present) with safe defaults:
          - enabled:          False (opt-in — no surprise disk writes)
          - path:             ~/.familiar/backups
          - encrypt:          False (opt-in — requires cryptography package)
          - retention_days:   30
          - include_history:  True
          - include_audit:    True

        The BackupManager is always created (never None) so agent.backup is safe
        to call even when backup is disabled — create_backup() will still work,
        it just won't be triggered automatically.

        Episodic memory directory is threaded through so the manager knows where
        per-user recalled conversation files live.
        """
        backup_cfg_raw = getattr(self.config, "backup", None)
        if backup_cfg_raw is not None and hasattr(backup_cfg_raw, "__dict__"):
            try:
                raw_dict = {k: v for k, v in vars(backup_cfg_raw).items() if not k.startswith("_")}
                # Convert path string -> Path (BackupConfig expects Path)
                raw_path = raw_dict.get("path", "")
                if not raw_path:
                    raw_dict["path"] = Path.home() / ".familiar" / "backups"
                else:
                    raw_dict["path"] = Path(raw_path)
                # Empty encryption_key -> None so Fernet isn't called with ''
                if not raw_dict.get("encryption_key", ""):
                    raw_dict["encryption_key"] = None
                # Empty s3_bucket -> None
                if not raw_dict.get("s3_bucket", ""):
                    raw_dict["s3_bucket"] = None
                # Only pass fields BackupConfig knows about
                valid_fields = set(BackupConfig.__dataclass_fields__)
                bc = BackupConfig(**{k: v for k, v in raw_dict.items() if k in valid_fields})
            except Exception as _bc_err:
                logger.debug(f"BackupConfig from agent config failed ({_bc_err}), using defaults")
                bc = BackupConfig()
        else:
            bc = BackupConfig()

        manager = BackupManager(config=bc)
        # Pass episodic dir so _backup_episodic knows where to look
        manager._episodic_dir = self._episodic_dir

        return manager

    def create_backup(
        self,
        backup_type: BackupType = BackupType.FULL,
        description: Optional[str] = None,
    ) -> "BackupManifest":
        """
        Create a backup of all agent data immediately.

        Captures:
          - memory.json          (semantic facts per user)
          - episodic/*.json(.enc) (per-user conversation history — highest priority)
          - history.json          (global conversation log)
          - users.db              (user database, if present)
          - analytics.db          (analytics, if present)
          - audit.db              (audit log, if present)
          - skills/               (skill data, if present)
          - config.yaml           (configuration, for FULL backup)

        Args:
            backup_type: FULL (default), DATA_ONLY, or CONFIG_ONLY
            description: Optional label stored in manifest

        Returns:
            BackupManifest with status, file list, checksum, and timing.

        Example::

            manifest = agent.create_backup()
            print(f"Backup {manifest.id}: {manifest.file_count} files, "
                  f"{manifest.total_size} bytes")
        """
        manifest = self.backup.create_backup(
            backup_type=backup_type,
            description=description,
        )
        # Register freshness with health checker
        if hasattr(self, "health"):
            self.health._cached_result = None  # bust cache so next check is fresh
        logger.info(
            f"Backup created: {manifest.id} "
            f"({manifest.file_count} files, {manifest.total_size} bytes, "
            f"status={manifest.status.value})"
        )
        return manifest

    def restore_backup(
        self,
        backup_id: str,
        target_dir: Optional[Path] = None,
        dry_run: bool = False,
    ) -> dict:
        """
        Restore agent data from a backup.

        WARNING: Overwrites current data files. Call create_backup() first to
        snapshot current state before restoring an older backup.

        Args:
            backup_id: Backup ID from list_backups() or create_backup().id
            target_dir: Optional directory to restore into. Default: original
                        data paths (~/.familiar/data, ~./familiar/episodic).
            dry_run:    If True, simulate and count files without writing.

        Returns:
            Dict with files_restored count, errors list, dry_run flag.
        """
        result = self.backup.restore(
            backup_id=backup_id,
            target_dir=target_dir,
            dry_run=dry_run,
        )
        if not dry_run:
            logger.warning(
                f"Backup restored: {backup_id} ({result.get('files_restored', 0)} files written)"
            )
        return result

    def list_backups(self) -> list:
        """
        Return all available backup manifests, newest first.

        Each manifest has:
            id, created_at, backup_type, status, file_count,
            total_size, encrypted, compressed, duration_seconds

        Example::

            for b in agent.list_backups():
                print(f"{b.id}  {b.backup_type.value}  "
                      f"{b.file_count} files  {b.status.value}")
        """
        return self.backup.list_backups()

    def verify_backup(self, backup_id: str) -> dict:
        """
        Verify the integrity of a backup archive.

        Checks that the archive exists on disk and the manifest is readable.
        Returns a result dict with ``valid`` bool, file count, size, and
        encryption flag.  On success, also updates the manifest status to
        VERIFIED so the health probe can report freshness correctly.

        Args:
            backup_id: ID from list_backups() or create_backup().id

        Returns:
            Dict with ``valid``, ``backup_id``, ``created_at``, ``size``,
            ``files``, ``encrypted``.  ``valid=False`` includes an ``error``
            key describing the problem.

        Example::

            result = agent.verify_backup("20260220_030000")
            if result["valid"]:
                print(f"OK — {result['files']} files, {result['size']} bytes")
            else:
                print(f"CORRUPT: {result['error']}")
        """
        return self.backup.verify_backup(backup_id)

    def cleanup_old_backups(self) -> int:
        """
        Delete backups older than config.backup.retention_days (default 30).

        Returns the number of backup archives deleted.  Safe to call at any
        time — only removes archives whose manifest ``created_at`` predates
        the retention window.

        Example::

            deleted = agent.cleanup_old_backups()
            print(f"Pruned {deleted} old backups")
        """
        return self.backup.cleanup_old_backups()

    def _schedule_backup(self) -> None:
        """
        Register a recurring backup task with the scheduler.

        Called from __init__ when both backup.enabled and scheduler are active.
        The schedule is driven by config.backup.schedule:
          daily   → runs at 03:00 each night (TaskFrequency.DAILY, time_of_day="03:00")
          hourly  → runs at the top of each hour (TaskFrequency.HOURLY)
          weekly  → runs at 03:00 on Sundays (TaskFrequency.DAILY every 7 days)

        A cleanup task is also registered to prune backups older than
        retention_days.

        Both tasks are silent if the scheduler has already registered tasks
        with the same name (idempotent on re-init).
        """
        if self.scheduler is None:
            return

        schedule = getattr(self.backup.config, "schedule", "daily")

        # Map schedule string to TaskFrequency
        if schedule == "hourly":
            freq = TaskFrequency.HOURLY
            kwargs: dict = {}
        else:
            # daily and weekly both use DAILY with time_of_day; weekly is
            # handled by cleanup running every day but retention_days controls
            # the window (weekly schedule is just daily with longer retention)
            freq = TaskFrequency.DAILY
            kwargs = {"time_of_day": "03:00"}

        _backup_ref = self.backup

        def _run_scheduled_backup():
            try:
                manifest = _backup_ref.create_backup(BackupType.FULL)
                logger.info(
                    f"Scheduled backup completed: {manifest.id} "
                    f"({manifest.file_count} files, {manifest.total_size} bytes)"
                )
                # Prune old backups after each successful backup
                _backup_ref.cleanup_old_backups()
            except Exception as e:
                logger.error(f"Scheduled backup failed: {e}")

        self.scheduler.add_task(
            name="familiar_auto_backup",
            description=f"Automatic {schedule} backup of all agent data",
            frequency=freq,
            callback=_run_scheduled_backup,
            **kwargs,
        )
        logger.info(f"Scheduled {schedule} backup registered (03:00 daily)")

    # ════════════════════════════════════════════════════════════════
    # HEALTH CHECKER
    # ════════════════════════════════════════════════════════════════

    def _init_health_checker(self) -> HealthChecker:
        """
        Create and configure the HealthChecker with agent-aware dependency checks.

        Built-in checks registered here:
          filesystem   — write/read /tmp, always registered by HealthChecker itself
          llm_provider — verifies provider has API key / is reachable
          memory_agent — background extraction thread liveness + last-error
          planner      — no plans stuck > 30 min
          bus          — SkillBus worker thread alive

        The version string is taken from the package __version__ if available,
        falling back to "unknown". Cache TTL is 10 s — cheap for polling probes.
        """
        try:
            from familiar import __version__ as _ver
        except Exception:
            _ver = "unknown"

        hc = HealthChecker(
            version=_ver,
            check_timeout_seconds=5.0,
            cache_ttl_seconds=10.0,
        )

        # ── LLM provider ─────────────────────────────────────────────────────
        # Checks that the configured provider has an API key set (or is local).
        provider_name = getattr(self.provider, "name", "unknown")

        def _check_provider() -> DependencyHealth:
            # Local providers (ollama, llamacpp) are always reachable when configured.
            local_names = ("ollama", "llamacpp", "local", "mock")
            if any(n in provider_name.lower() for n in local_names):
                return DependencyHealth(
                    name="llm_provider",
                    healthy=True,
                    message=f"Local provider '{provider_name}' configured",
                    dependency_type=DependencyType.LLM_PROVIDER,
                    details={"provider": provider_name, "local": True},
                )
            # Cloud providers need an API key
            key_env = {
                "anthropic": "ANTHROPIC_API_KEY",
                "openai": "OPENAI_API_KEY",
                "gemini": "GOOGLE_API_KEY",
            }.get(provider_name.lower(), "")
            key_set = bool(os.environ.get(key_env)) if key_env else True
            return DependencyHealth(
                name="llm_provider",
                healthy=key_set,
                message=(
                    f"Provider '{provider_name}' ready"
                    if key_set
                    else f"Provider '{provider_name}' missing {key_env}"
                ),
                dependency_type=DependencyType.LLM_PROVIDER,
                details={"provider": provider_name, "key_env": key_env, "key_set": key_set},
            )

        hc.register_dependency_check("llm_provider", _check_provider, DependencyType.LLM_PROVIDER)

        # ── SkillBus ─────────────────────────────────────────────────────────
        _bus = self.bus

        def _check_bus() -> DependencyHealth:
            running = getattr(_bus, "_running", False) or (
                hasattr(_bus, "_worker") and _bus._worker is not None and _bus._worker.is_alive()
            )
            return DependencyHealth(
                name="bus",
                healthy=running,
                message="SkillBus worker running" if running else "SkillBus worker not running",
                dependency_type=DependencyType.INTERNAL,
                details={"running": running},
            )

        hc.register_dependency_check("bus", _check_bus, DependencyType.INTERNAL)

        # ── Memory agent ─────────────────────────────────────────────────────
        if self.memory is not None:
            _mem_agent = getattr(self.memory, "_memory_agent", None) or getattr(
                self.memory, "_agent", None
            )
            if _mem_agent is not None:
                hc.register_dependency_check(
                    "memory_agent",
                    create_memory_agent_health_check(_mem_agent),
                    DependencyType.INTERNAL,
                )

        # ── Planner ──────────────────────────────────────────────────────────
        if self.planner is not None:
            hc.register_dependency_check(
                "planner",
                create_planner_health_check(self.planner),
                DependencyType.INTERNAL,
            )

        # ── Backup freshness ──────────────────────────────────────────────────
        # Healthy: at least one completed backup exists AND the most recent one
        # is within 2× the configured retention period (generous threshold —
        # we just want to surface "never backed up" and "backup very stale").
        # Reports "no backups yet" on a fresh install, which is informational
        # rather than critical: healthy=True so it doesn't break readiness.
        _backup_mgr = self.backup

        def _check_backup_freshness() -> DependencyHealth:
            try:
                backups = _backup_mgr.list_backups()
            except Exception as e:
                return DependencyHealth(
                    name="backup_freshness",
                    healthy=False,
                    message=f"Could not list backups: {e}",
                    dependency_type=DependencyType.INTERNAL,
                )
            if not backups:
                return DependencyHealth(
                    name="backup_freshness",
                    healthy=True,  # informational — fresh install
                    message="No backups yet. Call agent.create_backup() to protect data.",
                    dependency_type=DependencyType.INTERNAL,
                    details={"backup_count": 0},
                )
            latest = backups[0]
            age_days = (
                datetime.now(_dt_timezone.utc).replace(tzinfo=None) - latest.created_at
            ).total_seconds() / 86400
            retention = getattr(_backup_mgr.config, "retention_days", 30)
            stale_threshold_days = retention * 2
            healthy = age_days <= stale_threshold_days
            return DependencyHealth(
                name="backup_freshness",
                healthy=healthy,
                message=(
                    f"Latest backup: {latest.id} ({age_days:.1f} days ago, "
                    f"status={latest.status.value})"
                ),
                dependency_type=DependencyType.INTERNAL,
                details={
                    "backup_count": len(backups),
                    "latest_id": latest.id,
                    "latest_status": latest.status.value,
                    "age_days": round(age_days, 1),
                    "stale_threshold_days": stale_threshold_days,
                },
            )

        hc.register_dependency_check(
            "backup_freshness", _check_backup_freshness, DependencyType.INTERNAL
        )

        return hc

    # ════════════════════════════════════════════════════════════════
    # SEMANTIC RETRIEVER (RAG)
    # ════════════════════════════════════════════════════════════════

    def _init_retriever(self) -> SemanticRetriever:
        """
        Create a SemanticRetriever backed by the same embedding provider
        cascade that memory uses.

        The bridge adapter makes embeddings.EmbeddingProvider satisfy
        context.EmbeddingProvider's interface — they share the same abstract
        surface (embed, embed_batch) but have different ABC hierarchies and
        one uses .dimensions vs .dimension.
        """

        class _EmbeddingBridge(ContextEmbeddingProvider):
            """Adapt embeddings.EmbeddingProvider → context.EmbeddingProvider."""

            def __init__(self, provider: RawEmbeddingProvider):
                self._p = provider

            def embed(self, text: str):
                return self._p.embed(text)

            def embed_batch(self, texts):
                return self._p.embed_batch(texts)

            @property
            def dimension(self) -> int:
                # embeddings.py uses .dimensions; context.py uses .dimension
                if hasattr(self._p, "dimensions") and self._p.dimensions:
                    return self._p.dimensions
                if hasattr(self._p, "_dimensions") and self._p._dimensions:
                    return self._p._dimensions
                return 512  # safe default for TF-IDF and unknown providers

        # Reuse memory's already-initialized embedder when available
        # so we don't double-initialize (or double-probe Ollama).
        raw_provider = None
        if self.memory and getattr(self.memory, "_embedder", None):
            raw_provider = self.memory._embedder
        else:
            try:
                prefer_local = (
                    self.config.agent.get("prefer_local_embeddings", False)
                    if hasattr(self.config.agent, "get")
                    else False
                )
                raw_provider = get_embedding_provider(prefer_local=prefer_local)
            except Exception as e:
                logger.warning(
                    f"Retriever could not initialize embedding provider: {e}. Using TF-IDF."
                )
                raw_provider = TFIDFEmbeddings()

        bridge = _EmbeddingBridge(raw_provider)
        config = RetrievalConfig(
            top_k=5,
            min_score=0.05,  # Low threshold — TF-IDF scores are naturally lower than neural
            max_tokens=1500,
            chunk_size=400,
            chunk_overlap=40,
        )
        return SemanticRetriever(
            embedding_provider=bridge,
            vector_store=InMemoryVectorStore(),
            config=config,
        )

    def _retrieve_rag_context(self, query: str, max_tokens: int = 1500) -> str:
        """
        Query the RAG retriever with the current user message and return a
        formatted string suitable for injection into the LLM system prompt.

        Returns empty string when:
        - No documents have been indexed
        - Query is empty or too short
        - All results score below min_score threshold

        Always safe to call — exceptions are caught and logged.
        """
        if not query or len(query) < 3:
            return ""
        if self.retriever.count == 0:
            return ""
        try:
            ctx = self.retriever.get_context(
                query,
                max_tokens=max_tokens,
                format="numbered",
            )
            if ctx:
                return "## Retrieved Knowledge\n\n" + ctx
            return ""
        except Exception as e:
            logger.debug(f"RAG retrieval skipped: {e}")
            return ""

    def _handle_guardrail_violation(self, gv: GuardrailViolation, user_id: str) -> str:
        """
        Convert a GuardrailViolation into a clean user-facing message and
        log it to the audit trail.

        Never raises — always returns a string the agent can return directly.
        Each violation_type gets a specific, actionable message rather than
        leaking internal limit values to the user.
        """
        vtype = gv.violation_type

        user_messages = {
            ViolationType.RATE_LIMIT: (
                "⚠️ You're sending messages too quickly. Please wait a moment before trying again."
            ),
            ViolationType.COST_LIMIT: (
                "⚠️ You've reached your usage limit for this period. "
                "Limits reset hourly and daily — try again later or contact your admin."
            ),
            ViolationType.TOKEN_LIMIT: (
                "⚠️ This conversation has grown very long and hit a processing limit. "
                "Please start a new conversation to continue."
            ),
            ViolationType.ITERATION_LIMIT: (
                "⚠️ I got stuck in a loop trying to complete that request. "
                "Please try rephrasing or breaking it into smaller steps."
            ),
            ViolationType.TIMEOUT: (
                "⚠️ That request took too long to complete. "
                "Please try a simpler request or try again in a moment."
            ),
            ViolationType.CONTENT_SAFETY: (
                "⚠️ I can't process that message — it triggered a content safety check. "
                "Please rephrase your request."
            ),
            ViolationType.TOOL_BLOCKED: (
                "⚠️ That action requires administrator approval and can't be completed automatically. "
                "Please contact your admin."
            ),
        }

        msg = user_messages.get(vtype, f"⚠️ Request blocked: {vtype.value}")

        # Route to existing audit trail (not the duplicate AuditLogger in guardrails.py)
        try:
            from .audit import get_audit_logger as _get_audit

            _get_audit().log_guardrail_violation(
                user_id=str(user_id),
                violation_type=vtype.value,
                detail=str(gv),
            )
        except Exception:
            # audit.py may not have log_guardrail_violation — fall back to session audit
            logger.warning(f"Guardrail violation [{vtype.value}] for user {user_id}: {gv}")

        return msg

    def _validate_compliance(self):
        """Validate and apply compliance configuration."""
        if not COMPLIANCE_AVAILABLE:
            return

        # Get compliance mode from config
        compliance_config = getattr(self.config, "compliance", None)
        if not compliance_config:
            return

        mode_str = getattr(compliance_config, "mode", "none")
        if mode_str == "none":
            return

        try:
            self.compliance_mode = ComplianceMode(mode_str)
        except ValueError:
            logger.warning(f"Unknown compliance mode: {mode_str}")
            return

        # Store the compliance profile for later use (session timeout, etc.)
        try:
            from .compliance import get_compliance_config as _get_cc

            self._compliance_profile = _get_cc(self.compliance_mode)
        except ImportError:
            self._compliance_profile = None

        # Hard-fail: encryption key required for HIPAA/SOC2/PCI-DSS
        encryption_modes = {"hipaa", "soc2", "pci_dss"}
        if mode_str in encryption_modes:
            if not os.environ.get("FAMILIAR_ENCRYPTION_KEY"):
                raise SystemExit(
                    f"\n{'=' * 60}\n"
                    f"FATAL: Compliance mode '{mode_str}' requires encryption.\n"
                    f"Set FAMILIAR_ENCRYPTION_KEY before starting the agent.\n\n"
                    f"Generate a key with:\n"
                    f'  python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"\n\n'
                    f"Then export it:\n"
                    f"  export FAMILIAR_ENCRYPTION_KEY=<your-key>\n"
                    f"{'=' * 60}"
                )

        # Convert config to dict for validation
        config_dict = {
            "security": {
                "encrypt_sessions": getattr(self.config.security, "encrypt_sessions", False),
                "encrypt_memory": getattr(self.config.security, "encrypt_memory", False),
                "encrypt_history": getattr(self.config.security, "encrypt_history", False),
                "session_timeout_minutes": getattr(
                    self.config.security, "session_timeout_minutes", 60
                ),
            },
            "observability": {
                "audit_logging": getattr(self.config.observability, "tracing_enabled", False),
                "log_retention_days": 90,  # Default
            },
        }

        # Validate compliance requirements
        errors = validate_compliance(config_dict, self.compliance_mode)
        if errors:
            logger.warning(f"Compliance validation warnings for {self.compliance_mode.value}:")
            for error in errors:
                logger.warning(f"  - {error}")
        else:
            logger.info(f"Compliance mode {self.compliance_mode.value} requirements met")

        # Wire encrypted storage to audit subsystem
        encryption_modes = {"hipaa", "soc2", "pci_dss"}
        if mode_str in encryption_modes:
            try:
                from .encryption import EncryptedStorage

                es = EncryptedStorage()
                if es.is_available:
                    self._audit_encrypted_storage = es
            except Exception:
                pass

        # Initialize audit store with encryption if available
        from .audit import get_audit_store

        get_audit_store(encrypted_storage=self._audit_encrypted_storage)

        # Instantiate ComplianceAuditLogger
        try:
            from .compliance import ComplianceAuditLogger
            from .config import DATA_DIR

            self._compliance_audit_logger = ComplianceAuditLogger(
                config=self._compliance_profile,
                base_path=DATA_DIR,
                encrypted_storage=self._audit_encrypted_storage,
            )
        except Exception as e:
            logger.warning(f"ComplianceAuditLogger init failed: {e}")

    # _init_provider, _init_model_router, get_routing_config, set_routing_config,
    # switch_provider — moved to ProviderManager (provider_manager.py)
    # get_system_prompt — moved to PromptBuilder (prompt_builder.py)
    # Backward-compat delegates are defined above.

    def chat(
        self,
        message: str,
        user_id: Any = "default",
        channel: str = "default",
        include_history: bool = True,
        user_context: Optional["UserContext"] = None,
    ) -> str:
        """
        Process a message and return a response.

        Args:
            message: The user's message
            user_id: Unique identifier for the user/chat (ignored if user_context provided)
            channel: Which channel this came from (telegram, discord, cli)
            include_history: Whether to include conversation history
            user_context: Phase 1 - UserContext from authenticated channel user

        Returns:
            The agent's response text

        Raises:
            BudgetExceededError: If user's daily budget is exhausted
            ProviderError: If LLM provider fails
        """
        # _setup_request may raise before session/trace are returned.
        # _GuardrailBlock is caught here and returned as a plain string.
        try:
            session, message, user_id, trace, guardrail_tracker = self._setup_request(
                message, user_id, channel, user_context
            )
        except _GuardrailBlock as gb:
            return str(gb)

        gen = self._run_agent_loop(
            message=message,
            user_id=user_id,
            channel=channel,
            include_history=include_history,
            user_context=user_context,
            session=session,
            trace=trace,
            stream=False,
            emit_tool_events=False,
            guardrail_tracker=guardrail_tracker,
        )
        try:
            # stream=False: loop emits no events, returns result via StopIteration.value
            while True:
                next(gen)
        except StopIteration as done:
            return done.value

    def chat_with_results(
        self,
        message: str,
        user_id: Any = "default",
        channel: str = "default",
        include_history: bool = True,
        user_context: Optional["UserContext"] = None,
    ) -> AgentResponse:
        """Like chat(), but returns structured AgentResponse with tool results.

        Args:
            message: The user's message
            user_id: Unique identifier for the user/chat
            channel: Which channel this came from
            include_history: Whether to include conversation history
            user_context: UserContext from authenticated channel user

        Returns:
            AgentResponse with text and structured tool_results
        """
        collected_tool_results: list = []
        try:
            session, message, user_id, trace, guardrail_tracker = self._setup_request(
                message, user_id, channel, user_context
            )
        except _GuardrailBlock as gb:
            return AgentResponse(text=str(gb))

        gen = self._run_agent_loop(
            message=message,
            user_id=user_id,
            channel=channel,
            include_history=include_history,
            user_context=user_context,
            session=session,
            trace=trace,
            stream=False,
            emit_tool_events=False,
            guardrail_tracker=guardrail_tracker,
            collected_tool_results=collected_tool_results,
        )
        try:
            while True:
                next(gen)
        except StopIteration as done:
            return AgentResponse(
                text=done.value,
                tool_results=collected_tool_results,
            )
        except Exception as exc:
            logger.error("chat_with_results failed: %s", exc)
            return AgentResponse(
                text=f"I encountered an error processing your request: {exc}",
                tool_results=collected_tool_results,
            )

    def chat_stream(
        self,
        message: str,
        user_id: Any = "default",
        channel: str = "default",
        include_history: bool = True,
        emit_tool_events: bool = True,
        user_context: Optional["UserContext"] = None,
    ) -> Generator[StreamEvent, None, str]:
        """
        Process a message and stream the response.

        Tool calls are executed synchronously (buffered), but the final
        text response is streamed as it's generated.

        Args:
            message: The user's message
            user_id: Unique identifier for the user/chat
            channel: Which channel this came from
            include_history: Whether to include conversation history
            emit_tool_events: If True, emit TOOL_START/TOOL_END events
            user_context: Phase 1 - UserContext from authenticated channel user

        Yields:
            StreamEvent objects with text chunks and status updates

        Returns:
            The complete response text (accessible after iteration)

        Usage:
            final_text = ""
            for event in agent.chat_stream("Hello!"):
                if event.type == StreamEventType.TEXT:
                    print(event.content, end="", flush=True)
                    final_text += event.content
                elif event.type == StreamEventType.TOOL_START:
                    print(f"\\n[Running {event.tool_name}...]")
            print()  # Final newline
        """
        # Sanitize before checking streaming support (mirrors chat() path)
        sanitization_result = sanitize_user_input(
            message, level=SanitizationLevel.BASIC, max_length=50_000
        )
        if sanitization_result.was_modified:
            logger.debug(f"Input sanitized: {sanitization_result.warnings}")
        message = sanitization_result.sanitized

        # If provider doesn't support streaming, fall through to chat()
        # which has the full pipeline — this is the safe fallback path.
        if not self.provider.supports_streaming:
            result = self.chat(
                message, user_id, channel, include_history, user_context=user_context
            )
            yield StreamEvent(type=StreamEventType.TEXT, content=result)
            yield StreamEvent(type=StreamEventType.DONE)
            return result

        # Derive user_id first so _setup_request gets the right value
        if user_context and HAS_USER_MANAGEMENT:
            user_id = user_context.user.id

        try:
            session, message, user_id, trace, guardrail_tracker = self._setup_request(
                message,
                user_id,
                channel,
                user_context,
                already_sanitized=True,  # we sanitized above
            )
        except _GuardrailBlock as gb:
            yield StreamEvent(type=StreamEventType.TEXT, content=str(gb))
            yield StreamEvent(type=StreamEventType.DONE)
            return str(gb)

        yield from self._run_agent_loop(
            message=message,
            user_id=user_id,
            channel=channel,
            include_history=include_history,
            user_context=user_context,
            session=session,
            trace=trace,
            stream=True,
            emit_tool_events=emit_tool_events,
            guardrail_tracker=guardrail_tracker,
        )

    # ============================================================
    # CORE: UNIFIED AGENT LOOP
    # ============================================================

    def _setup_request(
        self,
        message: str,
        user_id: Any,
        channel: str,
        user_context: Optional["UserContext"],
        already_sanitized: bool = False,
    ):
        """
        Shared pre-flight for chat() and chat_stream().

        Handles: sanitization, user_id derivation, session creation,
        role→trust sync, budget check, interaction recording, trace creation.

        Returns: (session, sanitized_message, resolved_user_id, trace)
        """
        # Sanitize (skip if caller already did it, e.g. chat_stream fallback)
        if not already_sanitized:
            sanitization_result = sanitize_user_input(
                message, level=SanitizationLevel.BASIC, max_length=50_000
            )
            if sanitization_result.was_modified:
                logger.debug(f"Input sanitized: {sanitization_result.warnings}")
            message = sanitization_result.sanitized

        # Resolve user_id early — needed for guardrails before session exists
        resolved_uid = str(
            user_context.user.id if (user_context and HAS_USER_MANAGEMENT) else user_id
        )

        # ── Guardrails: rate limit + hourly/daily cost gate ──────────────────
        # Runs before session creation so blocked requests don't consume budget
        # or record an interaction.  Returns a UsageTracker that must be passed
        # to every subsequent guardrail call for this request.
        try:
            guardrail_tracker: Optional[UsageTracker] = self.guardrails.start_request(resolved_uid)
        except GuardrailViolation as gv:
            raise _GuardrailBlock(self._handle_guardrail_violation(gv, resolved_uid)) from gv

        # ── Guardrails: PII — redact before message reaches LLM ─────────────
        # process_with_pii returns (processed_text, findings, blocked).
        # Default PIIAction is REDACT so findings are replaced with <EMAIL> etc.
        # and the cleaned text is used as the message going forward.
        # If operator configured PIIAction.BLOCK this raises GuardrailViolation.
        try:
            message, pii_findings, _blocked = self.guardrails.process_with_pii(message)
            if pii_findings:
                logger.info(
                    f"PII redacted: {[f.entity_type for f in pii_findings]} for user {resolved_uid}"
                )
                session_pii_note = f"pii_redacted:{len(pii_findings)}_entities"
            else:
                session_pii_note = None
        except GuardrailViolation as gv:
            # PIIAction.BLOCK — content contains PII that must not be processed
            self.guardrails.end_request(resolved_uid, guardrail_tracker)
            raise _GuardrailBlock(self._handle_guardrail_violation(gv, resolved_uid)) from gv

        # Resolve user_id from context (use .id, not .email — email is mutable)
        if user_context and HAS_USER_MANAGEMENT:
            user_id = user_context.user.id
            logger.info(f"Chat from {user_context.user.email} ({user_context.user.role.value})")

        # Session setup
        session = self.sessions.get_or_create_session(str(user_id), channel)

        # Session timeout enforcement — invalidate expired sessions in compliance mode
        if self.compliance_mode and self._compliance_profile:
            timeout = self._compliance_profile.session_timeout_minutes
            if session.is_expired(timeout):
                self.sessions.invalidate_session(str(user_id), channel)
                raise SessionExpiredError(
                    f"Session expired (idle >{timeout} min). Please re-authenticate."
                )

        # Sync role → trust level (must happen before capability checks).
        # Always called: handles both the user_context path and the
        # preauth-only path (no user_context but persisted user_role).
        self._sync_user_to_session(user_context, session)

        # Budget gate — check before recording interaction so over-budget
        # users don't gain trust credit for a request that was blocked
        if session.remaining_budget <= 0:
            raise BudgetExceededError(
                "Daily budget exceeded. Try again tomorrow or contact admin.",
                budget_type="daily",
                spent=session.spent_today,
                budget=session.daily_budget,
            )

        # Record interaction after budget gate passes
        session.record_interaction(positive=True)

        # Interaction tracker (constitutional relationship stage)
        try:
            from .constitution import InteractionTracker

            if not hasattr(self, "_interaction_tracker"):
                self._interaction_tracker = InteractionTracker()
            self._interaction_tracker.record_interaction(str(user_id))
        except ImportError:
            pass

        # Trace
        trace = None
        if self.tracer:
            trace = self.tracer.create_trace(user_id=str(user_id), channel=channel)

        # Record PII redaction in session audit if it happened
        if session_pii_note:
            session.add_audit("pii_redacted", {"note": session_pii_note})

        return session, message, user_id, trace, guardrail_tracker

    def _finalize_response(
        self,
        message: str,
        final_text: str,
        user_id: Any,
        channel: str,
        user_context,
        session,
        trace,
        response,
        start_time: float,
        guardrail_tracker: Optional[UsageTracker] = None,
    ) -> str:
        """
        Shared post-processing after the agent produces its final text.

        Handles: history save, analytics, trace finish, session save,
        memory extraction, progressive milestone check, guardrail cleanup.

        Returns final_text (may be augmented with achievement banner).
        """
        # Save conversation history
        if user_context and HAS_USER_MANAGEMENT:
            self._save_user_history(user_context, channel, message, final_text)
        else:
            self.history.add_message(user_id, "user", message, channel)
            self.history.add_message(user_id, "assistant", final_text, channel)

        # Analytics
        self._record_analytics(
            user_id=str(user_id),
            channel=channel,
            response=response,
            start_time=start_time,
            user_context=user_context,
        )

        # Trace
        if trace:
            self.tracer.finish_trace(trace)

        # Session
        self.sessions.save_session(session)

        # Async memory extraction
        if self.memory_extractor:
            self.memory_extractor.extract_async(message, final_text, channel)

        # Guardrails: finalize request tracking.
        self.guardrails.end_request(str(user_id), guardrail_tracker)

        # EpisodicMemory: store this exchange as a conversation memory.
        # Runs after guardrails so we only store completed, non-blocked requests.
        # Stored as CONVERSATION type with both turns linked — the MemorySystem
        # associates adjacent turns so retrieval surfaces full exchanges.
        # Saved async to disk so it doesn't block response finalization.
        _episodic_ms = self._get_episodic_memory(str(user_id))
        if _episodic_ms is not None and final_text:
            try:
                _episodic_ms.store_conversation(
                    messages=[
                        {"role": "user", "content": message},
                        {"role": "assistant", "content": final_text},
                    ],
                    conversation_id=str(session.session_id),
                    importance=0.6,
                )
                self._save_episodic_memory_async(str(user_id))
            except Exception as e:
                logger.debug(f"EpisodicMemory store skipped: {e}")

        # Progressive capability milestones
        try:
            from familiar.core.progressive import on_interaction_complete

            if hasattr(self, "_interaction_tracker"):
                count = self._interaction_tracker.get_count(str(user_id))
                achievement = on_interaction_complete(str(user_id), count, self.skills)
                if achievement:
                    final_text = f"{final_text}\n\n---\n{achievement}"
        except (ImportError, AttributeError, Exception):
            pass

        return final_text

    def _run_agent_loop(
        self,
        message: str,
        user_id: Any,
        channel: str,
        include_history: bool,
        user_context,
        session,
        trace,
        stream: bool,
        emit_tool_events: bool,
        guardrail_tracker: Optional[UsageTracker] = None,
        collected_tool_results: Optional[list] = None,
    ):
        """
        Unified agent loop used by both chat() and chat_stream().

        In non-stream mode (stream=False): returns final response str.
        In stream mode (stream=True): is a generator that yields StreamEvents
        and returns the final text string via StopIteration.value / return.

        All security, capability filtering, planning, routing, PHI guard,
        self-correction, history, analytics, and memory extraction run
        through this single path regardless of streaming mode.

        guardrail_tracker: UsageTracker returned by guardrails.start_request()
            in _setup_request.  Passed explicitly (not stored on self) so
            concurrent requests don't share state.
        collected_tool_results: Optional request-scoped list for collecting
            structured ToolResult objects.  Passed explicitly (not stored on
            self) so concurrent requests don't share state.
        """
        try:
            fallback_notice = None  # Set if provider fallback occurs

            # ── History ──────────────────────────────────────────
            if include_history:
                if user_context and HAS_USER_MANAGEMENT:
                    history = self._get_user_history(user_context, channel)
                else:
                    history = self.history.get_history(user_id, channel)
            else:
                history = []

            history.append({"role": "user", "content": message})

            # ── Tool context ─────────────────────────────────────
            # EventEmitter: one per request, keyed to a short execution ID.
            # Injected into tool_context so any tool handler can emit structured
            # progress events without coupling to the agent or streaming layer.
            # The agent automatically emits TOOL_START / TOOL_COMPLETE / TOOL_ERROR
            # for every call — tools only need to emit TOOL_PROGRESS themselves.
            #
            # Clients subscribe a callback before the request:
            #   agent.set_tool_event_callback(user_id, handler)
            # See _get_tool_event_callback() for the registry.
            _request_emitter = EventEmitter(
                execution_id=f"{str(user_id)[:8]}-{str(uuid.uuid4())[:6]}"
            )
            _tool_cb = self._tool_executor.get_callback(str(user_id))
            if _tool_cb is not None:
                _request_emitter.subscribe(_tool_cb)

            # StructuredGenerator: lazy adapter wrapping the current provider.
            # Built as a lambda so provider swaps (fallback routing) are picked up
            # at call time rather than captured at request-start.
            # Tools call: tool_context["structured"].generate(MyModel, "prompt...")
            # Returns a GenerationResult — check .success before calling .unwrap().
            def _make_structured_provider(agent_ref):
                def _provider(messages, model=None, **kw):
                    # Bridge StructuredGenerator's (messages, model) call convention
                    # to provider.chat(messages, tools, system, max_tokens).
                    return agent_ref.provider.chat(
                        messages=messages,
                        tools=[],
                        system="You are a precise data extraction assistant. Output only valid JSON.",
                        max_tokens=kw.get("max_tokens", 2048),
                    )

                return _provider

            _structured_generator = StructuredGenerator(
                llm_provider=_make_structured_provider(self),
                max_retries=3,
                extraction_method=ExtractionMethod.FIRST_JSON,
                validation_strategy=ValidationStrategy.RETRY,
                repair_json=True,
            )

            tool_context = {
                "memory": self.memory,
                "user_id": user_id,
                "channel": channel,
                "session": session,
                "session_manager": self.sessions,
                "agent": AgentProxy(self),
                "user_context": user_context,
                "emitter": _request_emitter,
                "structured": _structured_generator,
                # sagas: shared SagaOrchestrator for multi-step atomic workflows.
                # Tools call tool_context["sagas"].run(saga, context) (sync bridge)
                # or tool_context["run_saga"](saga, context) for convenience.
                # Register saga definitions at skill load time via agent.sagas.register(saga).
                "sagas": self.sagas,
                "run_saga": self.run_saga,
                # experiments: feature flags and A/B testing.
                # Check flags: tool_context["experiments"].is_enabled("flag_name", str(user_id))
                # Get variant: tool_context["experiments"].get_variant("exp_name", str(user_id))
                "experiments": self.experiments,
                # mesh: MeshGateway (None if mesh not enabled/started).
                # Check agent.get_mesh_status() for runtime state.
                # Peer tool delegation: tool_context["orchestrator"].execute(task)
                "mesh": self.mesh_gateway,
                "orchestrator": self.orchestrator,
                # bus: SkillBus — inter-skill pub/sub, invocation, workflows.
                #
                # Publish an event (fire-and-forget, async):
                #   tool_context["bus"].publish("donor.intake.completed",
                #       {"donor_id": "123", "name": "Alice"})
                #
                # Invoke another skill synchronously (with retry):
                #   result = tool_context["bus"].invoke(
                #       "crm", "upsert_contact", {"email": "alice@example.com"})
                #
                # Shared state (cross-tool, in-process):
                #   tool_context["bus"].state.set("intake_lock", True)
                #   tool_context["bus"].state.get("intake_lock")
                #
                # Run a multi-skill workflow:
                #   wf = tool_context["bus"].create_workflow("donor_intake")
                #   wf.add_step("crm", "upsert", step_id="crm")
                #   wf.add_step("email", "send_welcome", depends_on=["crm"])
                #   tool_context["bus"].execute_workflow(wf, initial_context)
                "bus": self.bus,
                # retriever: SemanticRetriever — chunked document RAG.
                # Skills add org docs at load time or on demand:
                #   tool_context["retriever"].add_text(
                #       grant_guidelines, metadata={"source": "smith_foundation_2026"})
                # Query with natural language:
                #   results = tool_context["retriever"].retrieve("grant deadline", top_k=3)
                #   ctx = tool_context["retriever"].get_context("eligibility criteria")
                # The current user message is automatically used to retrieve
                # relevant chunks and inject them into the LLM system prompt.
                "retriever": self.retriever,
                # health: HealthChecker — liveness/readiness/dependency probes.
                # Skills register custom dependency checks:
                #   tool_context["health"].register_dependency_check(
                #       "crm", lambda: DependencyHealth(name="crm", healthy=ping_crm()))
                # Run probes:
                #   tool_context["health"].liveness_check()   → {"status": "ok"}
                #   tool_context["health"].readiness_check()  → {"ready": True, ...}
                #   tool_context["health"].check_health()     → HealthCheckResult
                "health": self.health,
                # metrics: MetricsCollector — per-call latency, success rates, cost.
                # Skills record custom metrics:
                #   tool_context["metrics"].increment_counter("donor_created")
                #   tool_context["metrics"].record_histogram("email_size_bytes", len(body))
                #   tool_context["metrics"].set_gauge("queue_depth", q.size())
                # Full report: agent.get_metrics_summary()
                "metrics": self.metrics,
                # backup: BackupManager — create/list/restore backups from skills.
                # tool_context["backup"].create_backup() triggers an on-demand backup.
                # tool_context["backup"].list_backups() returns manifest list.
                "backup": self.backup,
            }
            if user_context and HAS_USER_MANAGEMENT and get_data_store:
                try:
                    tool_context["data_store"] = get_data_store(
                        user_id=user_context.user.id,
                        user_role=user_context.user.role.value,
                    )
                except Exception as e:
                    logger.warning(f"Could not get user data store: {e}")

            # ── Capability-filtered tools (run first so skill names are available) ──
            tool_schemas, routed_skills = self._get_allowed_tools(session, message=message)

            # ── System prompt (with skill docs trimmed to routed skills) ─────────
            system = self.get_system_prompt(
                session,
                user_context,
                current_message=message,
                channel=channel,
                routed_skills=routed_skills,
            )

            # ── Context injection (episodic, RAG, mesh, planning) ───
            system = self._prompt_builder.inject_episodic_context(system, message, str(user_id))
            system = self._prompt_builder.inject_rag_context(system, message)
            system = self._prompt_builder.inject_mesh_context(system, message)
            system = self._prompt_builder.inject_planning_context(system, message, tool_schemas)

            # ── Provider selection + PHI guard ────────────────────
            try:
                active_provider, model_name = self._select_provider(message, session)
            except RuntimeError as phi_err:
                if "PHI_NO_LOCAL_FALLBACK" in str(phi_err):
                    session.add_audit("phi_blocked", {"reason": "PHI detected, no local fallback"})
                    self.sessions.save_session(session)
                    phi_msg = (
                        "⚠️ This message appears to contain protected health "
                        "information (PHI) and cannot be sent to a cloud provider "
                        "under current compliance settings. Please configure a "
                        "local Ollama model or rephrase your request."
                    )
                    if stream:
                        yield StreamEvent(type=StreamEventType.TEXT, content=phi_msg)
                        yield StreamEvent(type=StreamEventType.DONE)
                        return phi_msg
                    return phi_msg
                raise

            # ── Loop setup ────────────────────────────────────────
            security_config = SecurityConfig.from_mode(self.security_mode)
            max_iterations = min(15, security_config.max_iterations)
            start_time = time.time()

            from .self_correction import ToolRetryTracker

            retry_tracker = ToolRetryTracker()

            # ── Agent loop ────────────────────────────────────────
            for i in range(max_iterations):
                # ── Guardrails: iteration cap + timeout ───────────
                # Explicit tracker passed — thread-safe, no shared state.
                try:
                    self.guardrails.check_iteration(i, guardrail_tracker)
                except GuardrailViolation as gv:
                    guard_msg = self._handle_guardrail_violation(gv, user_id)
                    if stream:
                        yield StreamEvent(type=StreamEventType.TEXT, content=guard_msg)
                        yield StreamEvent(type=StreamEventType.DONE)
                        return guard_msg
                    return guard_msg

                # ── Proactive rate limit skip ─────────────────────
                provider_key = active_provider.name.split("(")[0].strip().lower()
                if self.capacity_tracker.is_rate_limited(provider_key):
                    original_name = active_provider.name
                    fallback = self._provider_fallback(
                        active_provider, RuntimeError("proactive_skip")
                    )
                    if fallback:
                        active_provider = fallback
                        fallback_notice = f"[{original_name} rate limited — using {fallback.name}]"
                        session.add_audit(
                            "proactive_fallback",
                            {"from": original_name, "to": fallback.name},
                        )
                        logger.info(f"Proactive fallback: {original_name} → {fallback.name}")

                # ── LLM call ──────────────────────────────────────
                response = None
                try:
                    if stream:
                        # Consume the stream, collecting text and the response object.
                        # Providers signal tool_calls via the returned LLMResponse,
                        # which is only accessible via StopIteration.value after the
                        # generator is exhausted (generator.value is not a valid attr).
                        stream_gen = active_provider.chat_stream(
                            messages=history,
                            tools=tool_schemas,
                            system=system,
                            max_tokens=self.config.llm.max_tokens,
                            trace=trace,
                        )
                        collected_text = ""
                        response = None
                        try:
                            while True:
                                event = next(stream_gen)
                                if event.type == StreamEventType.TEXT:
                                    collected_text += event.content
                                elif event.type == StreamEventType.ERROR:
                                    raise Exception(event.error)
                        except StopIteration as si:
                            response = si.value  # LLMResponse from generator return
                        if response is None:
                            # Provider didn't return an LLMResponse via StopIteration.
                            # Reconstruct from collected text (no tool calls possible).
                            logger.debug(
                                "Stream provider returned no LLMResponse; reconstructing from text"
                            )
                            response = LLMResponse(
                                text=collected_text or None,
                                tool_calls=[],
                                stop_reason="end_turn",
                            )
                    else:
                        _llm_start = time.time()
                        _llm_exc: Optional[str] = None
                        try:
                            response = active_provider.chat(
                                messages=history,
                                tools=tool_schemas,
                                system=system,
                                max_tokens=self.config.llm.max_tokens,
                                trace=trace,
                                retry_config=self.retry_config,
                            )
                        except Exception as _llm_e:
                            _llm_exc = str(_llm_e)
                            raise
                        finally:
                            try:
                                _usage = (response.usage if response else {}) or {}
                                self.metrics.record_llm_call(
                                    provider=active_provider.name,
                                    model=getattr(active_provider, "model_name", ""),
                                    duration_ms=(time.time() - _llm_start) * 1000,
                                    success=_llm_exc is None,
                                    prompt_tokens=_usage.get("input_tokens", 0),
                                    completion_tokens=_usage.get("output_tokens", 0),
                                    error_message=_llm_exc,
                                    user_id=str(current_user_id)
                                    if (current_user_id := locals().get("user_id"))
                                    else "",
                                )
                            except Exception as _m_err:
                                logger.debug(f"Metrics record_llm_call skipped: {_m_err}")

                except Exception as e:
                    logger.error(f"LLM call failed: {e}")
                    session.add_audit("llm_error", {"error": str(e)})

                    original_name = active_provider.name
                    fallback = self._provider_fallback(active_provider, e)
                    if fallback:
                        try:
                            session.add_audit(
                                "provider_fallback",
                                {
                                    "from": str(e)[:100],
                                    "to": fallback.name,
                                },
                            )
                            active_provider = fallback
                            fallback_notice = (
                                f"[{original_name} unavailable — using {fallback.name}]"
                            )
                            # Always use non-streaming for fallback (simpler, safer)
                            response = active_provider.chat(
                                messages=history,
                                tools=tool_schemas,
                                system=system,
                                max_tokens=self.config.llm.max_tokens,
                                trace=trace,
                                retry_config=self.retry_config,
                            )
                        except Exception as e2:
                            logger.error(f"Fallback provider also failed: {e2}")
                            error_msg = (
                                "I'm having trouble reaching my AI providers right now. "
                                "Please try again in a moment."
                            )
                            if trace:
                                self.tracer.finish_trace(trace)
                            self.sessions.save_session(session)
                            if stream:
                                yield StreamEvent(type=StreamEventType.ERROR, error=error_msg)
                                return error_msg
                            return error_msg
                    else:
                        logger.error(f"Provider error: {e}")
                        error_msg = (
                            f"I'm having trouble reaching {active_provider.name} right now. "
                            "Please try again in a moment."
                        )
                        if trace:
                            self.tracer.finish_trace(trace)
                        self.sessions.save_session(session)
                        if stream:
                            yield StreamEvent(type=StreamEventType.ERROR, error=error_msg)
                            return error_msg
                        return error_msg

                # ── Budget: charge actual token cost after LLM call ──────
                # Done here (after the call) so we use real token counts
                # rather than a flat estimate. Falls back to a conservative
                # per-iteration floor for providers that don't report usage
                # (e.g. Ollama) so the session budget still enforces limits.
                if response is not None:
                    actual_cost = 0.0
                    in_tokens = 0
                    out_tokens = 0
                    if response.usage:
                        in_tokens = response.usage.get("input_tokens", 0)
                        out_tokens = response.usage.get("output_tokens", 0)
                        actual_cost = calculate_cost(
                            active_provider.model_name,
                            in_tokens,
                            out_tokens,
                        )
                    if actual_cost == 0.0:
                        # Provider returned no usage (Ollama) or unknown model:
                        # charge a conservative floor so budgets still bind.
                        actual_cost = 0.005
                    if not session.charge(actual_cost):
                        self.sessions.save_session(session)
                        budget_msg = "⚠️ Budget limit reached for this request."
                        if stream:
                            yield StreamEvent(type=StreamEventType.TEXT, content=budget_msg)
                            yield StreamEvent(type=StreamEventType.DONE)
                            return budget_msg
                        return budget_msg

                    # Guardrails: track token + cost accumulation for this request.
                    # Raises GuardrailViolation if per-request token/cost cap hit.
                    # Uses same numbers as session.charge() — no second calculation.
                    try:
                        self.guardrails.track_llm_usage(
                            in_tokens, out_tokens, actual_cost, guardrail_tracker
                        )
                    except GuardrailViolation as gv:
                        guard_msg = self._handle_guardrail_violation(gv, user_id)
                        if stream:
                            yield StreamEvent(type=StreamEventType.TEXT, content=guard_msg)
                            yield StreamEvent(type=StreamEventType.DONE)
                            return guard_msg
                        return guard_msg

                # ── Tool calls ────────────────────────────────────
                if response.tool_calls:
                    history.append({"role": "assistant", "content": response})

                    tool_results, tool_events = self._execute_tools(
                        tool_calls=response.tool_calls,
                        session=session,
                        tool_context=tool_context,
                        retry_tracker=retry_tracker,
                        trace=trace,
                        stream=stream,
                        emit_tool_events=emit_tool_events,
                    )

                    # Yield tool events in stream mode
                    if stream:
                        for ev in tool_events:
                            yield ev

                    # Collect structured tool results for chat_with_results()
                    if collected_tool_results is not None:
                        from .agent_response import ToolResult as _ToolResult

                        for tr in tool_results:
                            if tr.get("tool_name"):
                                collected_tool_results.append(
                                    _ToolResult(
                                        tool_name=tr["tool_name"],
                                        tool_input=tr.get("tool_input", {}),
                                        output=str(tr.get("content", "")),
                                        tool_use_id=tr.get("tool_use_id", ""),
                                    )
                                )

                    # Confirmation pending — return sentinel directly,
                    # skip adding to history (no LLM needed for UI routing)
                    if (
                        tool_results
                        and isinstance(tool_results[0], dict)
                        and tool_results[0].get("type")
                        in ("confirmation_pending", "email_menu_pending", "calendar_menu_pending")
                    ):
                        sentinel = tool_results[0]["sentinel"]
                        if stream:
                            yield StreamEvent(type=StreamEventType.DONE)
                            return sentinel
                        return sentinel

                    history.append({"role": "user", "content": tool_results})
                    continue

                # ── Final response ────────────────────────────────
                else:
                    final_text = response.text or "(no response)"

                    final_text = self._finalize_response(
                        message=message,
                        final_text=final_text,
                        user_id=user_id,
                        channel=channel,
                        user_context=user_context,
                        session=session,
                        trace=trace,
                        response=response,
                        start_time=start_time,
                        guardrail_tracker=guardrail_tracker,
                    )

                    if fallback_notice:
                        final_text = f"{fallback_notice}\n\n{final_text}"

                    if stream:
                        yield StreamEvent(type=StreamEventType.TEXT, content=final_text)
                        yield StreamEvent(type=StreamEventType.DONE, usage=response.usage)
                        return final_text
                    return final_text

            # ── Max iterations ────────────────────────────────────
            if trace:
                self.tracer.finish_trace(trace)
            self.sessions.save_session(session)
            timeout_msg = "(max iterations reached - task may be too complex)"
            if stream:
                yield StreamEvent(type=StreamEventType.TEXT, content=timeout_msg)
                yield StreamEvent(type=StreamEventType.DONE)
                return timeout_msg
            return timeout_msg

        except Exception as e:
            # Inner error paths (provider fail, budget) already save session and
            # finish trace before returning. This catches anything unexpected that
            # bypasses those paths (e.g. an exception in history loading, planning,
            # or _execute_tools itself).
            if trace:
                try:
                    self.tracer.finish_trace(trace)
                except Exception:
                    pass
            self.sessions.save_session(session)
            if stream:
                yield StreamEvent(type=StreamEventType.ERROR, error=str(e))
            raise

    # PHASE 1: MULTI-USER HELPERS
    # ============================================================

    def _sync_user_to_session(self, user_context: "UserContext", session: SecureSession):
        """
        Sync UserContext role to SecureSession trust level.

        This bridges the new multi-user system with the existing
        progressive trust security model.

        Role → Trust mapping:
            ADMIN → OWNER (full access)
            STAFF → TRUSTED (most capabilities)
            READONLY → KNOWN (read-only capabilities)
        """
        if not HAS_USER_MANAGEMENT or not user_context:
            # Check if this user is the configured owner (OWNER_TELEGRAM_ID or equivalent).
            # The Telegram channel does this in _get_session(), but agent.chat() is called
            # directly by tests and non-Telegram callers, so we replicate the check here.
            owner_id = getattr(self.config.channels, "owner_telegram_id", None)
            if owner_id and str(session.user_id) == str(owner_id):
                if session.trust_level != TrustLevel.OWNER:
                    session.set_trust_level(TrustLevel.OWNER)
                    logger.info(
                        f"Auto-promoted owner {session.user_id} to OWNER trust (agent sync)"
                    )
                return

            # Fallback: restore trust from previously persisted role on session.
            # This keeps staff from reverting to STRANGER if users.py fails to
            # import at bot restart (e.g. sqlite3 lock, missing dep).
            if session.user_role:
                _role_to_trust = {
                    "admin": TrustLevel.OWNER,
                    "staff": TrustLevel.TRUSTED,
                    "readonly": TrustLevel.KNOWN,
                }
                restored = _role_to_trust.get(session.user_role)
                if restored and session.trust_level == TrustLevel.STRANGER:
                    session.trust_level = restored
                    logger.warning(
                        f"user_management unavailable; restored trust={restored.value} "
                        f"from persisted role={session.user_role} for user={session.user_id}"
                    )
            return

        role = user_context.user.role

        # Map role to trust level and merge capabilities into _explicit_grants.
        # Use |= (union) so manually granted capabilities are preserved.
        if role == UserRole.ADMIN:
            session.trust_level = TrustLevel.OWNER
            session._explicit_grants |= {c.value for c in Capability}
        elif role == UserRole.STAFF:
            session.trust_level = TrustLevel.TRUSTED
            session._explicit_grants |= {
                c.value
                for c in {
                    Capability.READ_TIME,
                    Capability.READ_WEATHER,
                    Capability.READ_MEMORY,
                    Capability.WRITE_MEMORY,
                    Capability.READ_CALENDAR,
                    Capability.WRITE_CALENDAR,
                    Capability.READ_TASKS,
                    Capability.WRITE_TASKS,
                    Capability.SEND_NOTIFICATIONS,
                    Capability.HTTP_REQUESTS,
                }
            }
        else:  # READONLY
            session.trust_level = TrustLevel.KNOWN
            session._explicit_grants |= {
                c.value
                for c in {
                    Capability.READ_TIME,
                    Capability.READ_WEATHER,
                    Capability.READ_MEMORY,
                    Capability.READ_CALENDAR,
                    Capability.READ_TASKS,
                }
            }

        logger.debug(
            f"Synced {user_context.user.email} role {role.value} to trust {session.trust_level.value}"
        )
        # Persist role to session so trust survives bot restarts even if
        # core/users.py is temporarily unavailable at session load time.
        session.user_role = role.value

    # ============================================================
    # PHASE 2: ANALYTICS HELPERS
    # ============================================================

    def _record_analytics(
        self,
        user_id: str,
        channel: str,
        response: "LLMResponse",
        start_time: float,
        user_context: Optional["UserContext"] = None,
        skill: str = None,
    ):
        """
        Record usage analytics for this request.

        Called after successful chat completion.
        """
        try:
            import time

            from .analytics import record_agent_usage

            # Calculate latency
            latency_ms = int((time.time() - start_time) * 1000)

            # Get token counts from response.usage dict (if available)
            usage = getattr(response, "usage", None) or {}
            tokens_in = usage.get("input_tokens", 0) or 0
            tokens_out = usage.get("output_tokens", 0) or 0

            # Estimate if not provided
            if tokens_in == 0 and tokens_out == 0 and response.text:
                tokens_out = len(response.text) // 4  # Rough estimate

            # Get user email if available
            user_email = None
            if user_context and HAS_USER_MANAGEMENT:
                user_email = user_context.user.email

            record_agent_usage(
                user_id=user_id,
                channel=channel,
                tokens_in=tokens_in,
                tokens_out=tokens_out,
                latency_ms=latency_ms,
                model=self.provider.name if self.provider else None,
                skill=skill,
                user_email=user_email,
                success=True,
            )
        except ImportError:
            # Analytics not available
            pass
        except Exception as e:
            logger.warning(f"Could not record analytics: {e}")

    # ============================================================
    # END PHASE 2 HELPERS
    # ============================================================

    def add_reminder(
        self,
        name: str,
        message: str,
        when: datetime,
        channel: Optional[str] = None,
        chat_id: Optional[str] = None,
        callback: Optional[Any] = None,
    ) -> str:
        """Add a one-time reminder."""
        if not self.scheduler:
            return "Scheduler not enabled"

        minutes_until = (when - datetime.now()).total_seconds() / 60

        def reminder_callback(task):
            if callback:
                callback(message, channel, chat_id)

        task_id = self.scheduler.add_task(
            name=name,
            description=message,
            frequency=TaskFrequency.ONCE,
            callback=reminder_callback,
            channel=channel,
            chat_id=chat_id,
            interval_minutes=int(minutes_until),
            payload={"message": message},
        )

        return task_id

    def start_scheduler(self):
        """Start the background scheduler and register default proactive tasks."""
        if self.scheduler:
            self._register_default_tasks()
            self.scheduler.start()

    def _register_default_tasks(self):
        """Register built-in proactive tasks if not already present."""
        if not self.scheduler:
            return

        from .scheduler import TaskFrequency, TaskType

        # Check what's already registered (by name, not ID)
        existing_names = {t.name for t in self.scheduler.tasks.values()}

        defaults = [
            {
                "name": "morning_briefing",
                "description": "Daily morning briefing with calendar, tasks, and weather",
                "frequency": TaskFrequency.DAILY,
                "task_type": TaskType.BRIEFING,
                "time_of_day": "08:00",
                "payload": {
                    "prompt": (
                        "Give me my morning briefing. Include: "
                        "today's calendar events, pending tasks and deadlines, "
                        "weather if available, and any unread messages or notifications."
                    )
                },
            },
            {
                "name": "heartbeat",
                "description": "System health check every 4 hours",
                "frequency": TaskFrequency.HOURLY,
                "task_type": TaskType.HEARTBEAT,
                "interval_minutes": 240,
            },
            {
                "name": "eod_summary",
                "description": "End-of-day summary of completed work",
                "frequency": TaskFrequency.DAILY,
                "task_type": TaskType.BRIEFING,
                "time_of_day": "18:00",
                "payload": {
                    "prompt": (
                        "Give me an end-of-day summary. Include: "
                        "what was accomplished today, tasks completed, "
                        "upcoming deadlines, and anything that needs attention tomorrow."
                    )
                },
            },
        ]

        registered = 0
        for task_def in defaults:
            if task_def["name"] not in existing_names:
                try:
                    self.scheduler.add_task(**task_def)
                    registered += 1
                except Exception as e:
                    logger.debug(f"Could not register {task_def['name']}: {e}")

        if registered:
            logger.info(f"Registered {registered} default proactive task(s)")

    def stop_scheduler(self):
        """Stop the scheduler."""
        if self.scheduler:
            self.scheduler.stop()

    def get_session(self, user_id: str, channel: str) -> SecureSession:
        """Get the secure session for a user."""
        return self.sessions.get_or_create_session(user_id, channel)

    def set_user_trust(self, user_id: str, channel: str, level: TrustLevel):
        """Manually set a user's trust level (admin action)."""
        session = self.sessions.get_or_create_session(user_id, channel)
        session.set_trust_level(level)
        self.sessions.save_session(session)

    def grant_capability(self, user_id: str, channel: str, cap: Capability):
        """Grant a specific capability to a user."""
        session = self.sessions.get_or_create_session(user_id, channel)
        session.grant_capability(cap)
        self.sessions.save_session(session)

    def deny_capability(self, user_id: str, channel: str, cap: Capability):
        """Deny a specific capability from a user."""
        session = self.sessions.get_or_create_session(user_id, channel)
        session.deny_capability(cap)
        self.sessions.save_session(session)


class AgentProxy:
    """Read-only proxy exposed to tool handlers via tool_context['agent'].

    Prevents privilege escalation by blocking access to set_user_trust,
    grant_capability, deny_capability, and internal managers.
    """

    __slots__ = ("_agent",)

    def __init__(self, agent: "Agent"):
        self._agent = agent

    # ── Allowed read-only attributes ──

    @property
    def sessions(self):
        return self._agent.sessions

    @property
    def memory(self):
        return self._agent.memory

    @property
    def tools(self):
        return self._agent.tools

    @property
    def metrics(self):
        return self._agent.metrics

    @property
    def skills(self):
        return self._agent.skills

    @property
    def guardrails(self):
        return self._agent.guardrails

    @property
    def config(self):
        return self._agent.config

    @property
    def provider(self):
        return self._agent.provider

    @property
    def name(self):
        agent_cfg = getattr(self._agent.config, "agent", None)
        return agent_cfg.name if agent_cfg else None

    def __repr__(self):
        name = self.name or "unknown"
        return f"<AgentProxy for {name!r}>"


# Convenience function
def create_agent(config: Optional[Config] = None) -> Agent:
    """Create and return a configured agent."""
    return Agent(config)
