"""Self-learning LLM call wrapper for netanel-core.

The single entry point for every LLM call. Wraps any model with file-based
memory and self-learning. Every call: retrieve memories -> build context ->
call -> evaluate -> retry if needed -> extract learnings -> store -> check
triggers -> return.

Two execution paths:
- Simple path (use_agent=False): direct LLM call with quality loop
- Agent path (use_agent=True): DeepAgent spawn with tool scoring

Flow: [DEFERRED EVOLUTION] -> RETRIEVE -> BUILD -> CALL -> EVALUATE ->
      RETRY -> EXTRACT -> STORE -> CHECK TRIGGERS -> RETURN

Architecture ref: ARCHITECTURE.md Section 6 (Complete Flow), Section 13 Steps 5+10.
"""

from __future__ import annotations

import json
import logging
import time
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Protocol

from filelock import FileLock
from langchain_core.tools import BaseTool
from pydantic import BaseModel, ConfigDict, Field

from netanel_core._atomic import atomic_write_json
from netanel_core.config import NathanConfig
from netanel_core.context_builder import Context, ContextBuilder
from netanel_core.deep_agent import DeepAgent
from netanel_core.evaluator import EvalResult, Evaluator
from netanel_core.hooks import HookManager
from netanel_core.memory_store import MemoryStore
from netanel_core.prompt_evolver import PromptEvolver
from netanel_core.tool_selector import ToolSelector

logger = logging.getLogger(__name__)

# ────────────────────────────────────────────
# Data Types
# ────────────────────────────────────────────


class CallResult(BaseModel):
    """Result of a LearningLLM.call() invocation.

    Returned to the caller with the response plus metadata about
    quality, retries, and learnings extracted.
    """

    model_config = ConfigDict(extra="forbid")

    response: str
    """The LLM output text."""

    score: float
    """Quality score from the evaluator (0.0 to 1.0)."""

    passed: bool
    """Whether the response met the quality threshold."""

    retries: int
    """Number of quality-loop retries used (0 = passed first try)."""

    learnings_stored: int
    """Number of pattern files written from this call."""

    metadata: dict[str, Any] = Field(default_factory=dict)
    """Extra info: tier_used, trimmed, estimated_tokens, etc."""

    agent_steps: int | None = None
    """Number of agent steps taken (None when use_agent=False)."""

    agent_compactions: int | None = None
    """Number of compaction cycles during agent execution (None when use_agent=False)."""

    usage: dict[str, int] = Field(
        default_factory=lambda: {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
    )
    """Token usage in OpenAI format: {prompt_tokens, completion_tokens, total_tokens}."""

    usage_tracked: bool = True
    """Whether usage was actually tracked. False for DeepAgent calls where usage is unavailable."""


def _strip_defaults_add_required(schema: dict) -> None:  # noqa: ANN001
    """Mutate *schema* in-place for OpenAI structured-output strict mode.

    OpenAI strict mode rejects schemas that have ``"default"`` on any property
    or omit properties from the ``"required"`` array (BUG-037).  Fields with
    ``default_factory`` in Pydantic generate ``"default": []`` and are NOT put
    in ``"required"`` — both violations.  This hook strips defaults and ensures
    every listed property is required.
    """
    props: dict = schema.get("properties", {})
    for prop in props.values():
        prop.pop("default", None)
    schema["required"] = list(props)


class LearningExtraction(BaseModel):
    """Structured output from the learning extraction LLM call.

    gpt-4o-mini extracts patterns, decisions, and improvements from
    successful LLM responses. Each field is a list of learnings to store.

    Used with LangChain's .with_structured_output() for reliable parsing.

    The ``json_schema_extra`` callable strips ``"default"`` from every property
    and adds all fields to ``"required"`` — required by OpenAI strict mode
    (BUG-037). Python-side defaults (``default_factory=list``) are preserved so
    callers can instantiate the model without arguments.
    """

    model_config = ConfigDict(
        extra="forbid",
        json_schema_extra=_strip_defaults_add_required,
    )

    patterns: list[str] = Field(
        default_factory=list,
        description=(
            "Reusable patterns discovered in the response. "
            "Each entry: a concise description of what worked and why."
        ),
    )
    decisions: list[str] = Field(
        default_factory=list,
        description=(
            "Key decisions made in the response that should be remembered. "
            "Each entry: what was decided and the rationale."
        ),
    )
    improvements: list[str] = Field(
        default_factory=list,
        description=(
            "Specific improvements or techniques that could be reused. "
            "Each entry: what the improvement is and when to apply it."
        ),
    )

    @property
    def all_learnings(self) -> list[tuple[str, str]]:
        """Flatten all learnings into (category, content) pairs."""
        result: list[tuple[str, str]] = []
        for p in self.patterns:
            result.append(("pattern", p))
        for d in self.decisions:
            result.append(("decision", d))
        for i in self.improvements:
            result.append(("improvement", i))
        return result

    @property
    def total_count(self) -> int:
        """Total number of learnings extracted."""
        return len(self.patterns) + len(self.decisions) + len(self.improvements)


@dataclass
class TriggerStatus:
    """Result of checking evolution triggers.

    Returned by LearningLLM.check_triggers(). Reports whether prompt
    evolution should fire and which trigger(s) activated.
    """

    should_evolve: bool
    """Whether any trigger condition is met."""

    reasons: list[str] = field(default_factory=list)
    """Human-readable descriptions of which triggers fired."""

    call_count: int = 0
    """Total calls since last evolution."""

    failure_rate: float = 0.0
    """Failure rate in the recent window."""

    new_patterns_count: int = 0
    """Number of new patterns since last evolution."""


class LearningState(BaseModel):
    """Persistent state for LearningLLM.

    Saved to learning_state.json in the namespace directory.
    Tracks call history for trigger checks and evolution scheduling.
    Matches EvalState pattern from Evaluator.
    """

    model_config = ConfigDict(extra="forbid")

    total_calls: int = Field(
        default=0,
        ge=0,
        description="Total number of LearningLLM.call() invocations",
    )
    calls_since_last_evolution: int = Field(
        default=0,
        ge=0,
        description="Calls since last prompt evolution cycle",
    )
    recent_results: list[bool] = Field(
        default_factory=list,
        description="Recent pass/fail results (True=passed) for failure rate window",
    )
    total_learnings_stored: int = Field(
        default=0,
        ge=0,
        description="Total number of learnings stored across all calls",
    )
    patterns_since_last_evolution: int = Field(
        default=0,
        ge=0,
        description="New patterns stored since last prompt evolution",
    )


# ────────────────────────────────────────────
# LLM Model Protocol
# ────────────────────────────────────────────


class LLMModel(Protocol):
    """Protocol for the main LLM model — enables dependency injection for testing.

    Matches EvalModel pattern from Evaluator. Any object with an invoke()
    method accepting a list of messages and returning a string response works.
    """

    def invoke(self, messages: list[Any]) -> str: ...


class ExtractorModel(Protocol):
    """Protocol for the learning extraction model — enables DI for testing.

    Returns LearningExtraction via structured output.
    """

    def invoke(self, messages: list[Any]) -> LearningExtraction: ...


# ────────────────────────────────────────────
# Extraction Prompt
# ────────────────────────────────────────────

_EXTRACTION_SYSTEM_PROMPT = """\
You are a learning extraction system. Your job is to analyze a successful \
LLM response and extract reusable learnings.

## What to Extract

1. **Patterns**: Reusable approaches or techniques that worked well.
   - "Used step-by-step decomposition for complex math problems"
   - "Provided examples before asking for output format"

2. **Decisions**: Key choices made that should be remembered.
   - "Chose recursive approach over iterative for tree traversal"
   - "Used markdown tables for structured comparison"

3. **Improvements**: Techniques that could be applied to future tasks.
   - "Adding confidence scores to uncertain answers improves trust"
   - "Breaking long outputs into sections with headers aids readability"

## Rules
1. Only extract from SUCCESSFUL, high-quality responses
2. Each learning must be concise (1-2 sentences)
3. Each learning must be REUSABLE — applicable to future tasks
4. Skip trivial or obvious observations
5. If nothing worth learning, return empty lists

START DIRECTLY with the extraction. No preambles."""


def _build_extraction_prompt(task: str, response: str) -> str:
    """Build the human message for learning extraction.

    Args:
        task: The original task that produced the response.
        response: The successful LLM output to extract learnings from.

    Returns:
        Formatted prompt string for the extraction model.
    """
    return f"## Original Task\n{task}\n\n## Successful Response\n{response}"


# ────────────────────────────────────────────
# Constants
# ────────────────────────────────────────────

# Internal safety bound: max consecutive evolution failures before suppressing.
# Not user-configurable — this is a bounded-safety mechanism, not user behaviour.
# Matches the JPL pattern used for MAX_PATTERNS_PER_EVOLUTION in PromptEvolver.
_MAX_EVOLUTION_RETRIES: int = 3

# ────────────────────────────────────────────
# LearningLLM
# ────────────────────────────────────────────


class LearningLLM:
    """Self-learning LLM call wrapper.

    The single entry point for the entire library. Wraps any LLM call with:
    - Memory retrieval and context building
    - Quality evaluation with retry loop
    - Learning extraction from successful responses
    - Evolution trigger checking

    Usage:
        llm = LearningLLM(config)
        result = llm.call("Write a sorting function in Python")
        print(result.response)   # The LLM output
        print(result.score)      # Quality score
        print(result.passed)     # Met threshold?
    """

    def __init__(
        self,
        config: NathanConfig,
        model: LLMModel | None = None,
        evaluator: Evaluator | None = None,
        extractor: ExtractorModel | None = None,
        hooks: HookManager | None = None,
        deep_agent: DeepAgent | None = None,
        prompt_evolver: PromptEvolver | None = None,
        tool_selector: ToolSelector | None = None,
    ) -> None:
        """Initialize the self-learning LLM wrapper.

        Auto-creates all components if not provided. All components read
        from the shared NathanConfig.

        Args:
            config: Nathan configuration (single source of truth).
            model: Override main LLM model (for testing).
            evaluator: Override evaluator (for testing).
            extractor: Override extraction model (for testing).
            hooks: Override HookManager (for testing). Auto-created from store.
            deep_agent: Override DeepAgent (for testing). Auto-created from config + hooks.
            prompt_evolver: Override PromptEvolver (for testing). Auto-created from config + store.
            tool_selector: Override ToolSelector (for testing). Auto-created from config.
        """
        self._config = config
        self._store = MemoryStore(config)
        self._context_builder = ContextBuilder(config, self._store)
        self._evaluator = evaluator or Evaluator(config)
        # BUG-079: track if state was reset due to corruption — set before _load_state()
        self.state_was_reset: bool = False
        self._state, _evolution_pending_from_disk = self._load_state()

        # Step 10: Wire remaining components
        self._hooks = hooks if hooks is not None else HookManager(self._store)
        self._prompt_evolver = (
            prompt_evolver if prompt_evolver is not None else PromptEvolver(config, self._store)
        )
        self._tool_selector = tool_selector if tool_selector is not None else ToolSelector(config)
        self._deep_agent = deep_agent if deep_agent is not None else DeepAgent(config, self._hooks)

        # Tool registry: maps tool names (strings in config) to actual tool objects.
        # Populated via register_tool(). Used in _agent_path() to pass objects to spawn().
        self._tool_registry: dict[str, BaseTool | Callable] = {}

        # BUG-080: Restore _evolution_pending from disk. Also check if triggers already
        # fire (e.g. evolution was due at startup), and set flag if so.
        _trigger_at_startup = self.check_triggers()
        self._evolution_pending: bool = _evolution_pending_from_disk or _trigger_at_startup.should_evolve

        # Consecutive evolution failure counter (in-memory only).
        # Resets to 0 on success. Suppresses further attempts after _MAX_EVOLUTION_RETRIES.
        # Restart gives evolution a fresh chance — intentional recovery behaviour.
        self._evolution_failure_streak: int = 0

        # Rollback tracking (BUG-030): accumulate recent scores to compare pre/post evolution.
        # _recent_scores: rolling window of the last `rollback_window` call scores.
        # _pre_evolution_scores: snapshot saved when evolution fires.
        # _post_evolution_scores: scores collected after evolution (same window size).
        # _collecting_post_evolution: True while gathering post-evo scores.
        self._recent_scores: list[float] = []
        self._pre_evolution_scores: list[float | None] = []
        self._post_evolution_scores: list[float | None] = []
        self._collecting_post_evolution: bool = False

        # Main LLM model
        if model is not None:
            self._model = model
        else:
            from langchain_openai import ChatOpenAI

            self._model = ChatOpenAI(  # type: ignore[assignment]
                model=config.models.primary_model,
                temperature=0.7,
                timeout=config.safety.model_timeout_seconds,
            )

        # Extraction model (gpt-4o-mini with structured output)
        if extractor is not None:
            self._extractor = extractor
        else:
            from langchain_openai import ChatOpenAI

            self._extractor = ChatOpenAI(  # type: ignore[assignment]
                model=config.models.extractor_model,
                temperature=0,
                timeout=config.safety.model_timeout_seconds,
            ).with_structured_output(LearningExtraction)

    # ────────────────────────────────────────
    # Public API
    # ────────────────────────────────────────

    def call(self, task: str, use_agent: bool = False) -> CallResult:
        """Execute the full self-learning LLM call.

        Two execution paths:
        - Simple path (use_agent=False): direct LLM call with quality loop and retries
        - Agent path (use_agent=True): DeepAgent spawn with tool scoring, no retry

        Both paths: deferred evolution → retrieve → build → call → evaluate →
                    extract → store → check triggers → return

        Args:
            task: The task/instruction for the LLM.
            use_agent: If True, route through DeepAgent. Default False.

        Returns:
            CallResult with response, quality score, and metadata.

        Raises:
            ValueError: If task is empty or whitespace-only.
        """
        if not task or not task.strip():
            raise ValueError("Task cannot be empty")

        # Step 0: Execute deferred evolution from previous call (if pending)
        self._execute_deferred_evolution()

        if use_agent:
            return self._agent_path(task)
        return self._simple_path(task)

    def _simple_path(self, task: str) -> CallResult:
        """Execute the simple LLM call path (no agent).

        Steps: RETRIEVE -> BUILD -> CALL -> EVALUATE -> RETRY ->
               EXTRACT -> STORE -> CHECK TRIGGERS -> ON_COMPLETE -> RETURN
        """
        # Steps 1-2: RETRIEVE + BUILD
        context = self._context_builder.build(task)

        # Steps 3-5: CALL + EVALUATE + RETRY (quality loop)
        response, eval_result, retries, usage = self._quality_loop(task, context)

        # Track score for post-evolution rollback check (BUG-030)
        self._track_score_for_rollback(eval_result.score)

        # Step 6-7: EXTRACT + STORE (only from passing responses)
        learnings_stored = 0
        learnings_for_trigger = 0  # BUG-081: counts all categories (not just "pattern")
        if eval_result.passed:
            learnings_stored, learnings_for_trigger = self._extract_and_store(task, response)

        # Step 8: CHECK TRIGGERS (update state, evaluate triggers)
        trigger_status = self._update_state(eval_result.passed, learnings_stored, learnings_for_trigger)

        # Fire on_complete hook (simple path). Agent path fires from _agent_path().
        # Only persist successful outputs to avoid seeding future contexts with low-quality completions.
        # BUG-022: skip hook write when _extract_and_store already stored learnings — avoid duplicate storage.
        if eval_result.passed:
            try:
                self._hooks.on_complete(
                    task=task, result=response, write_memory=(learnings_stored == 0)
                )
            except Exception:
                logger.warning("on_complete hook failed in simple path", exc_info=True)

        # Step 9: RETURN
        return CallResult(
            response=response,
            score=eval_result.score,
            passed=eval_result.passed,
            retries=retries,
            learnings_stored=learnings_stored,
            usage=usage,
            metadata={
                "model": self._config.models.primary_model,
                "tier_used": eval_result.tier_used,
                "threshold": eval_result.threshold,
                "trimmed": context.trimmed,
                "estimated_tokens": context.estimated_tokens,
                "memories_count": context.memories_count,
                "should_evolve": trigger_status.should_evolve,
                "evolution_reasons": trigger_status.reasons,
            },
        )

    def register_tool(self, tool: BaseTool | Callable) -> None:
        """Register a tool object for agent use.

        config.tools holds string names for ToolSelector scoring.
        This method maps those names to actual tool objects for spawn().

        Args:
            tool: A BaseTool instance (uses tool.name) or any callable
                  (uses tool.__name__).

        Raises:
            TypeError: If the argument is not a BaseTool instance or a callable
                       with a ``__name__`` attribute.
            ValueError: If the resolved name is empty or whitespace-only.
        """
        if isinstance(tool, BaseTool):
            name: str = tool.name
        elif callable(tool) and hasattr(tool, "__name__"):
            name = tool.__name__
        else:
            raise TypeError(
                f"register_tool expects a BaseTool instance or a named callable, "
                f"got {type(tool).__name__!r}"
            )
        if not name or not name.strip():
            raise ValueError(f"Tool name must be a non-empty string, got {name!r}")
        self._tool_registry[name] = tool

    def _agent_path(self, task: str) -> CallResult:
        """Execute the agent path via DeepAgent.

        Steps: SCORE TOOLS -> SPAWN -> EVALUATE (no retry) ->
               EXTRACT -> STORE -> ON_COMPLETE -> LOG TOOL CALL -> CHECK TRIGGERS -> RETURN

        No retry loop: the agent's internal ReAct loop already iterates until completion.
        A second spawn would restart the agent from scratch, not refine its output.
        This is an intentional architecture difference from the simple path.

        on_complete fires HERE (not in DeepAgent.spawn) after BOTH evaluation passes
        AND the agent completed. Incomplete results (silent recursion limit) are never
        persisted even if the evaluator scores them high (BUG-031 fix).
        """
        # Warn if no tools configured
        if not self._config.tools:
            logger.warning(
                "use_agent=True but no tools configured. "
                "DeepAgent will use 3 pillars only (bash, files, search)."
            )

        # BUG-012: Build context (role prompt + memories) and inject into agent.
        # _simple_path does this via _quality_loop; _agent_path previously skipped it.
        context = self._context_builder.build(task)

        # Score and order tools — ToolSelector works with string names.
        # Convert tuple → list to satisfy get_scored_tools() type signature (mypy).
        # BUG-030: guard against empty tools (list if truthy, else None).
        tools_list = list(self._config.tools) if self._config.tools else None
        scored_tools = self._tool_selector.get_scored_tools(tools_list)
        ordered_tool_names = [t.name for t in scored_tools]

        # Resolve string names to registered tool objects.
        # Unregistered names are silently skipped (graceful degradation).
        resolved_tools = [
            self._tool_registry[name] for name in ordered_tool_names if name in self._tool_registry
        ]
        if len(resolved_tools) < len(ordered_tool_names):
            skipped = [n for n in ordered_tool_names if n not in self._tool_registry]
            if not resolved_tools and self._config.tools:
                logger.warning(
                    "All %d configured tool(s) are unregistered — none passed to agent: %s. "
                    "Call register_tool() for each tool before invoking with use_agent=True.",
                    len(skipped),
                    skipped,
                )
            else:
                logger.debug("Skipping %d unregistered tool(s): %s", len(skipped), skipped)

        # BUG-077: warn if registered tools are not in config.tools (they will be silently ignored).
        # The agent only sees tools that appear in BOTH config.tools AND _tool_registry.
        # Tools registered via register_tool() with names absent from config.tools are dropped.
        config_tool_names = set(self._config.tools) if self._config.tools else set()
        missing_from_config = set(self._tool_registry) - config_tool_names
        if missing_from_config:
            logger.warning(
                "BUG-077: register_tool() called for %d tool(s) not in config.tools — "
                "they will be ignored by the agent: %s. "
                "Add them to config.tools at construction time to include them in agent calls.",
                len(missing_from_config),
                sorted(missing_from_config),
            )

        # Spawn the agent (timed for ToolSelector logging)
        start_ms = int(time.monotonic() * 1000)
        spawn_result = self._deep_agent.spawn(
            task,
            tools=resolved_tools,
            system_prompt=context.full_system_prompt,  # BUG-012: inject memories
        )
        elapsed_ms = int(time.monotonic() * 1000) - start_ms

        # Guard: skip evaluation on failed or empty agent output (BUG-045).
        # completed=False means agent hit recursion limit or tool loop without finishing.
        # Empty result means no usable output — evaluation would always fail and waste tokens.
        if not spawn_result.completed or not spawn_result.result.strip():
            logger.warning(
                "Skipping evaluation — agent %s (completed=%s, result_len=%d)",
                "returned empty output" if spawn_result.completed else "did not complete",
                spawn_result.completed,
                len(spawn_result.result),
            )
            self._tool_selector.log_tool_call(
                tool="deep_agent",
                success=False,
                task_type="agent_spawn",
                duration_ms=elapsed_ms,
            )
            trigger_status = self._update_state(False, 0, 0)
            return CallResult(
                response=spawn_result.result,
                score=0.0,
                passed=False,
                retries=0,
                learnings_stored=0,
                usage={"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
                usage_tracked=False,
                agent_steps=spawn_result.steps_taken,
                agent_compactions=spawn_result.compactions,
                metadata={
                    "model": self._config.models.primary_model,
                    "skipped_eval": True,
                    "reason": "agent_failed_or_empty",
                    "tier_used": 0,
                    "threshold": 0.0,
                    "agent_completed": spawn_result.completed,
                    "tool_order": [n for n in ordered_tool_names if n in self._tool_registry],
                    "should_evolve": trigger_status.should_evolve,
                    "evolution_reasons": trigger_status.reasons,
                    "trimmed": context.trimmed,
                    "memories_count": context.memories_count,
                },
            )

        # Evaluate agent output (no retry — agent has internal ReAct loop)
        eval_result = self._evaluator.evaluate(task, spawn_result.result)

        # Track score for post-evolution rollback check (BUG-030)
        self._track_score_for_rollback(eval_result.score)

        # Extract learnings and fire on_complete only when the agent fully completed
        # AND evaluation passed. Incomplete results (e.g. silent recursion limit) must
        # not be persisted even if the evaluator somehow scores them high.
        learnings_stored = 0
        learnings_for_trigger = 0  # BUG-081: counts all categories (not just "pattern")
        if eval_result.passed and spawn_result.completed:
            learnings_stored, learnings_for_trigger = self._extract_and_store(task, spawn_result.result)
            try:
                # BUG-022: skip hook write when _extract_and_store already stored learnings.
                self._hooks.on_complete(
                    task=task,
                    result=spawn_result.result,
                    write_memory=(learnings_stored == 0),
                )
            except Exception:
                logger.warning("on_complete hook failed in agent path", exc_info=True)

        # Log the agent invocation as a tool call.
        # BUG-043 fix: use eval_result.passed (quality) not spawn_result.completed
        # (completion). At this point spawn_result.completed is always True (we
        # returned early at line ~560 if it was False), so logging completion
        # inflated the score regardless of output quality.
        self._tool_selector.log_tool_call(
            tool="deep_agent",
            success=eval_result.passed,
            task_type="agent_spawn",
            duration_ms=elapsed_ms,
        )

        # Update state and check triggers
        trigger_status = self._update_state(eval_result.passed, learnings_stored, learnings_for_trigger)

        # TODO: Track usage from DeepAgent tool calls (future enhancement)
        usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}

        return CallResult(
            response=spawn_result.result,
            score=eval_result.score,
            passed=eval_result.passed,
            retries=0,  # No retry in agent path
            learnings_stored=learnings_stored,
            usage=usage,
            usage_tracked=False,  # DeepAgent usage not tracked yet
            agent_steps=spawn_result.steps_taken,
            agent_compactions=spawn_result.compactions,
            metadata={
                "model": self._config.models.primary_model,
                "tier_used": eval_result.tier_used,
                "threshold": eval_result.threshold,
                "should_evolve": trigger_status.should_evolve,
                "evolution_reasons": trigger_status.reasons,
                "agent_completed": spawn_result.completed,
                "tool_order": [n for n in ordered_tool_names if n in self._tool_registry],
                "trimmed": context.trimmed,
                "memories_count": context.memories_count,
            },
        )

    def check_triggers(self) -> TriggerStatus:
        """Check if evolution triggers are met.

        Reads current state and evaluates against config thresholds.
        Returns status — does NOT execute evolution (that's Step 8).

        Returns:
            TriggerStatus with should_evolve flag and reasons.
        """
        evo = self._config.evolution
        state = self._state
        reasons: list[str] = []

        # Trigger 1: Call count
        if state.calls_since_last_evolution >= evo.call_count_trigger:
            reasons.append(
                f"Call count trigger: {state.calls_since_last_evolution} "
                f"calls since last evolution (threshold: {evo.call_count_trigger})"
            )

        # Trigger 2: Failure rate
        failure_rate = 0.0
        window = evo.failure_window
        if len(state.recent_results) >= window:
            recent = state.recent_results[-window:]
            failures = sum(1 for r in recent if not r)
            failure_rate = failures / window
            if failure_rate >= evo.failure_rate_trigger:
                reasons.append(
                    f"Failure rate trigger: {failure_rate:.1%} failures "
                    f"in last {window} calls (threshold: {evo.failure_rate_trigger:.1%})"
                )

        # Trigger 3: Pattern accumulation
        if state.patterns_since_last_evolution >= evo.pattern_accumulation_trigger:
            reasons.append(
                f"Pattern accumulation trigger: {state.patterns_since_last_evolution} "
                f"new patterns (threshold: {evo.pattern_accumulation_trigger})"
            )

        return TriggerStatus(
            should_evolve=len(reasons) > 0,
            reasons=reasons,
            call_count=state.calls_since_last_evolution,
            failure_rate=failure_rate,
            new_patterns_count=state.patterns_since_last_evolution,
        )

    # ────────────────────────────────────────
    # Internal: Deferred Evolution
    # ────────────────────────────────────────

    def _execute_deferred_evolution(self) -> None:
        """Execute pending prompt evolution if flagged.

        Called at the start of every call(). Evolves the prompt using
        accumulated patterns, then resets counters on success.

        Note: _evolution_pending is in-memory only. If LearningLLM is
        recreated between calls (e.g., serverless), the flag is lost.
        This is documented as a known limitation.

        Failure streak: after _MAX_EVOLUTION_RETRIES consecutive failures,
        evolution is suppressed for the lifetime of this instance. A restart
        gives evolution a fresh chance (intentional recovery behaviour).
        """
        if not self._evolution_pending:
            return

        # Suppress after too many consecutive failures (BUG-011).
        # Prevents burning API budget on a permanently broken evolution path.
        if self._evolution_failure_streak >= _MAX_EVOLUTION_RETRIES:
            logger.warning(
                "Evolution suppressed after %d consecutive failures. Restart to retry.",
                _MAX_EVOLUTION_RETRIES,
            )
            self._evolution_pending = False
            return

        result = self._prompt_evolver.evolve(trigger="deferred")
        if result is not None:
            self._state.calls_since_last_evolution = 0
            self._state.patterns_since_last_evolution = 0
            self._save_state()
            self._evolution_failure_streak = 0  # Reset on success
            logger.info("Deferred evolution completed: version %d", result.version)

            # BUG-030: Snapshot pre-evolution scores; begin collecting post-evolution scores.
            window = self._config.evolution.rollback_window
            self._pre_evolution_scores = list(self._recent_scores[-window:])
            self._post_evolution_scores = []
            self._collecting_post_evolution = True
        else:
            self._evolution_failure_streak += 1
            logger.warning(
                "Deferred evolution failed (%d/%d), continuing",
                self._evolution_failure_streak,
                _MAX_EVOLUTION_RETRIES,
            )
        self._evolution_pending = False

    # ────────────────────────────────────────
    # Internal: Rollback Score Tracking (BUG-030)
    # ────────────────────────────────────────

    def _track_score_for_rollback(self, score: float) -> None:
        """Record a call score for post-evolution rollback analysis.

        Maintains a rolling window of recent scores (size=rollback_window).
        When collecting post-evolution scores, appends to that list and triggers
        rollback check once the window is full.

        Args:
            score: Quality score from the most recent evaluation (0.0–1.0).
        """
        window = self._config.evolution.rollback_window
        self._recent_scores.append(score)
        self._recent_scores = self._recent_scores[-window:]

        if self._collecting_post_evolution:
            self._post_evolution_scores.append(score)
            if len(self._post_evolution_scores) >= window:
                self._check_and_execute_rollback()

    def _check_and_execute_rollback(self) -> None:
        """Compare pre/post evolution scores and rollback if quality degraded.

        Called automatically when post-evolution score window fills.
        Dual condition (from PromptEvolver.check_rollback):
          - post_avg < pre_avg (quality decreased)
          - post_avg < rollback_threshold (below acceptable floor)

        Resets the collecting flag regardless of outcome.
        """
        self._collecting_post_evolution = False
        should_rollback, stats = self._prompt_evolver.check_rollback(
            pre_scores=self._pre_evolution_scores,
            post_scores=self._post_evolution_scores,
        )
        if should_rollback:
            result = self._prompt_evolver.rollback()
            if result is not None and result.rolled_back:
                logger.warning(
                    "Prompt rolled back to v%d: post-evolution quality degraded. "
                    "pre_avg=%.2f, post_avg=%.2f",
                    result.to_version,
                    stats.get("pre_avg", 0.0),
                    stats.get("post_avg", 0.0),
                )

    # ────────────────────────────────────────
    # Internal: Quality Loop (Steps 3-5)
    # ────────────────────────────────────────

    def _quality_loop(
        self,
        task: str,
        context: Context,
    ) -> tuple[str, EvalResult, int, dict[str, int]]:
        """Execute the LLM call with quality evaluation and retry loop.

        Calls the model, evaluates the output, retries with feedback if
        quality is below threshold. Keeps the best attempt across retries.

        Args:
            task: The original task.
            context: Built context from ContextBuilder.

        Returns:
            Tuple of (best_response, best_eval_result, retry_count, total_usage).
        """
        max_retries = self._config.safety.max_retries
        best_response: str | None = None
        best_eval: EvalResult | None = None
        last_eval: EvalResult | None = None  # BUG-048: track most-recent eval separately
        total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
        # Track the attempt number when an exception breaks the loop early (BUG-051 refinement).
        # None means we ran to completion (all retries exhausted, not an early exception break).
        _exception_break_at: int | None = None

        for attempt in range(max_retries + 1):  # 0 = first try, 1..N = retries
            try:
                # Step 3: CALL
                if attempt == 0:
                    response, usage = self._call_llm(context)
                else:
                    # BUG-026 fix: keep system prompt stable; feedback goes in user message.
                    # BUG-048 fix: feedback references last_eval (most recent), not best_eval.
                    assert last_eval is not None  # noqa: S101 — guaranteed by loop structure
                    feedback_task = (
                        f"{task}\n\n"
                        f"## Previous Attempt Feedback\n"
                        f"Score: {last_eval.score:.2f} (threshold: {last_eval.threshold:.2f})\n"
                        f"Feedback: {last_eval.feedback}\n\n"
                        f"Please improve your response based on this feedback."
                    )
                    response, usage = self._call_llm_raw(context.full_system_prompt, feedback_task)

                # Accumulate usage
                total_usage["prompt_tokens"] += usage["prompt_tokens"]
                total_usage["completion_tokens"] += usage["completion_tokens"]
                total_usage["total_tokens"] += usage["total_tokens"]

                # Step 4: EVALUATE
                # BUG-049: record_history=False — intermediate retry scores must NOT
                # enter the rolling average used for threshold auto-raise. Only the
                # final selected result is recorded below (via record_result).
                eval_result = self._evaluator.evaluate(task, response, record_history=False)

                # Track last evaluation (for feedback on next retry — BUG-048)
                last_eval = eval_result

                # Track best attempt (for selecting best response if all retries fail)
                if best_eval is None or eval_result.score > best_eval.score:
                    best_response = response
                    best_eval = eval_result

                # Step 5: PASS or RETRY
                if eval_result.passed:
                    # BUG-049: record the passing result now (record_history=False above).
                    self._evaluator.record_result(eval_result)
                    return response, eval_result, attempt, total_usage

            except Exception:
                # BUG-051 fix: transient errors on retry must not discard the best response
                # found so far. On the first attempt there is no prior work to fall back to,
                # so we re-raise. On retries, fall back to best_response if one exists.
                if attempt == 0 or best_response is None:
                    raise
                logger.warning(
                    "Retry %d failed with exception; returning best attempt (score=%.2f).",
                    attempt,
                    best_eval.score if best_eval else 0.0,
                    exc_info=True,
                )
                _exception_break_at = attempt
                break

        # Return the best response found across all attempts.
        assert best_response is not None  # noqa: S101 — at least one attempt ran
        assert best_eval is not None  # noqa: S101

        # BUG-049: Record only the final selected result into rolling history.
        # Intermediate attempts used record_history=False above to avoid skewing
        # the threshold auto-raise with repeated low-quality retry scores.
        self._evaluator.record_result(best_eval)

        if _exception_break_at is not None:
            # Loop broke early due to a retry exception — report the actual attempt count.
            # Only log max-retry exhaustion when we truly ran all retries.
            return best_response, best_eval, _exception_break_at, total_usage

        logger.warning(
            "Max retries (%d) exhausted for task. Best score: %.2f (threshold: %.2f)",
            max_retries,
            best_eval.score,
            best_eval.threshold,
        )

        return best_response, best_eval, max_retries, total_usage

    def _call_llm(self, context: Context) -> tuple[str, dict[str, int]]:
        """Call the main LLM model with the built context.

        Args:
            context: Complete context from ContextBuilder.

        Returns:
            Tuple of (response text, usage dict).
        """
        return self._call_llm_raw(context.full_system_prompt, context.task)

    def _call_llm_raw(self, system_prompt: str, task: str) -> tuple[str, dict[str, int]]:
        """Call the main LLM model with explicit system prompt and task.

        Args:
            system_prompt: The complete system prompt.
            task: The user's task.

        Returns:
            Tuple of (response text, usage dict in OpenAI format).
        """
        from langchain_core.messages import HumanMessage, SystemMessage

        messages = [
            SystemMessage(content=system_prompt),
            HumanMessage(content=task),
        ]
        result = self._model.invoke(messages)

        # Extract usage from response metadata
        usage = self._extract_usage(result)

        # LangChain models return AIMessage; extract .content for string
        response = str(result.content) if hasattr(result, "content") else str(result)
        return response, usage

    def _extract_usage(self, result: Any) -> dict[str, int]:
        """Extract token usage from LLM response.

        Args:
            result: AIMessage from LangChain model (langchain_core.messages.AIMessage).

        Returns:
            Usage dict in OpenAI format: {prompt_tokens, completion_tokens, total_tokens}.
            Values are clamped to non-negative integers.
        """
        default_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}

        if not hasattr(result, "response_metadata"):
            return default_usage

        metadata = result.response_metadata

        def _safe_int(value: Any) -> int:
            """Coerce to int and clamp negative to zero."""
            try:
                return max(0, int(value))
            except (ValueError, TypeError):
                return 0

        # Check both "usage" and "token_usage" (ChatOpenAI variants)
        usage_data = metadata.get("usage") or metadata.get("token_usage")
        if usage_data:
            prompt = _safe_int(usage_data.get("prompt_tokens", 0))
            completion = _safe_int(usage_data.get("completion_tokens", 0))
            total = _safe_int(usage_data.get("total_tokens", 0))
            # Compute total if missing
            if total == 0 and (prompt > 0 or completion > 0):
                total = prompt + completion
            return {
                "prompt_tokens": prompt,
                "completion_tokens": completion,
                "total_tokens": total,
            }

        # Anthropic format (input_tokens, output_tokens)
        if "input_tokens" in metadata or "output_tokens" in metadata:
            prompt = _safe_int(metadata.get("input_tokens", 0))
            completion = _safe_int(metadata.get("output_tokens", 0))
            return {
                "prompt_tokens": prompt,
                "completion_tokens": completion,
                "total_tokens": prompt + completion,
            }

        return default_usage

    # ────────────────────────────────────────
    # Internal: Learning Extraction (Steps 6-7)
    # ────────────────────────────────────────

    def _extract_and_store(self, task: str, response: str) -> tuple[int, int]:
        """Extract learnings from a successful response and store them.

        Only called when the response passed quality evaluation.
        Uses gpt-4o-mini to extract patterns, decisions, and improvements.

        Args:
            task: The original task.
            response: The successful LLM response.

        Returns:
            Tuple of (total_learnings_stored, learnings_stored_count).
            learnings_stored_count counts ALL categories (BUG-081 fix).
        """
        try:
            extraction = self._extract_learnings(task, response)
            if extraction is None:
                # with_structured_output() can return None on malformed JSON, empty tool call,
                # or model refusal — same risk as BUG-035 in Evaluator. No exception raised.
                logger.warning(
                    "Learning extraction returned None (structured output failure) — "
                    "skipping storage. Task: %.100s",
                    task,
                )
                return 0, 0
        except Exception:
            logger.warning(
                "Learning extraction failed for task, skipping. Task: %.100s",
                task,
                exc_info=True,
            )
            return 0, 0

        if extraction.total_count == 0:
            return 0, 0

        stored = 0
        # BUG-081: count ALL stored learnings regardless of category.
        # LLM extraction frequently returns "improvement" or "decision" categories,
        # never "pattern". Counting only "pattern" kept patterns_since_last_evolution
        # permanently at 0, making the pattern_accumulation_trigger permanently dead.
        # The state key name patterns_since_last_evolution is NOT renamed (backward compat).
        learnings_stored_count = 0
        max_learnings = self._config.safety.max_learnings_per_call
        all_learnings = list(extraction.all_learnings)
        if len(all_learnings) > max_learnings:
            logger.info(
                "Extraction returned %d learnings (cap=%d) — truncating to cap",
                len(all_learnings),
                max_learnings,
            )
            all_learnings = all_learnings[:max_learnings]
        for category, content in all_learnings:
            try:
                self._store.write_memory(
                    title=f"{category}: {content[:50]}",
                    content=content,
                    metadata={"category": category, "source": "auto_extraction"},
                    source_task=task[:200],
                )
                stored += 1
                # BUG-081: increment for ALL categories (pattern, improvement, decision, etc.)
                learnings_stored_count += 1
            except OSError:
                logger.warning(
                    "Failed to store learning: %.100s",
                    content,
                    exc_info=True,
                )

        return stored, learnings_stored_count

    def _extract_learnings(self, task: str, response: str) -> LearningExtraction:
        """Call the extraction model to pull learnings from a response.

        Args:
            task: The original task.
            response: The successful LLM response.

        Returns:
            LearningExtraction with patterns, decisions, improvements.
        """
        from langchain_core.messages import HumanMessage, SystemMessage

        messages = [
            SystemMessage(content=_EXTRACTION_SYSTEM_PROMPT),
            HumanMessage(content=_build_extraction_prompt(task, response)),
        ]
        return self._extractor.invoke(messages)

    # ────────────────────────────────────────
    # Internal: State Management (Step 8)
    # ────────────────────────────────────────

    def _update_state(
        self,
        passed: bool,
        learnings_stored: int,
        learnings_for_trigger: int = 0,
    ) -> TriggerStatus:
        """Update persistent state after a call completes.

        Also sets _evolution_pending flag if triggers fire (deferred evolution).
        Returns the TriggerStatus so callers can reuse it without calling
        check_triggers() a second time.

        Args:
            passed: Whether the call's output passed quality threshold.
            learnings_stored: Number of new learnings stored (all categories).
            learnings_for_trigger: Number of learnings to add to the evolution
                trigger counter (all categories — BUG-081). The state field is
                named patterns_since_last_evolution for backward compatibility.

        Returns:
            TriggerStatus from this call's trigger check.
        """
        self._state.total_calls += 1
        self._state.calls_since_last_evolution += 1
        self._state.total_learnings_stored += learnings_stored
        # BUG-081: count all categories for trigger (field name kept for backward compat)
        self._state.patterns_since_last_evolution += learnings_for_trigger

        # Keep recent_results bounded to failure_window * 3
        self._state.recent_results.append(passed)
        max_results = self._config.evolution.failure_window * 3
        if len(self._state.recent_results) > max_results:
            self._state.recent_results = self._state.recent_results[-max_results:]

        self._save_state()

        # Flag deferred evolution if triggers fire
        trigger_status = self.check_triggers()
        if trigger_status.should_evolve:
            self._evolution_pending = True

        return trigger_status

    def _load_state(self) -> tuple[LearningState, bool]:
        """Load learning state from file, or create fresh state.

        BUG-079: Sets self.state_was_reset = True when state is corrupted and
                 reset to zero, so callers can detect the reset happened.
        BUG-080: Also returns the persisted evolution_pending flag (bool) so
                 __init__() can restore it without re-reading the file.

        Returns:
            Tuple of (LearningState, evolution_pending_from_disk).
            evolution_pending_from_disk is False when file is absent or corrupted.
        """
        state_path = self._state_path
        if state_path.exists():
            try:
                raw = json.loads(state_path.read_text(encoding="utf-8"))
                # BUG-080: extract evolution_pending before model_validate (LearningState has
                # extra="forbid", so the key must be removed to avoid validation error)
                evolution_pending = raw.pop("evolution_pending", False)
                state = LearningState.model_validate(raw)
                return state, bool(evolution_pending)
            except (json.JSONDecodeError, KeyError, ValueError) as exc:
                # Corrupted state — start fresh (same pattern as Evaluator)
                logger.warning(
                    "Corrupted learning state at %s, starting fresh: %s",
                    state_path,
                    exc,
                )
                # BUG-079: signal to caller that state was reset
                self.state_was_reset = True
                return LearningState(), False
        return LearningState(), False

    def _save_state(self) -> None:
        """Persist learning state to file.

        BUG-019: file lock prevents concurrent-process data loss.
        BUG-033: atomic_write_json uses rename(2) to prevent partial writes.
        BUG-080: evolution_pending persisted alongside state so restarts don't lose it.
        """
        state_path = self._state_path
        lock_path = state_path.with_suffix(state_path.suffix + ".lock")
        with FileLock(lock_path, timeout=10):
            data = self._state.model_dump()
            # BUG-080: include _evolution_pending so deferred evolution survives restarts
            data["evolution_pending"] = self._evolution_pending
            atomic_write_json(state_path, data)

    @property
    def _state_path(self) -> Path:
        """Path to the learning state file."""
        return self._config.namespace_dir / "learning_state.json"

    # ────────────────────────────────────────
    # Properties (read-only access)
    # ────────────────────────────────────────

    @property
    def stats(self) -> dict[str, Any]:
        """Learning statistics for monitoring.

        Returns:
            Dict with total calls, learnings stored, trigger status, etc.
        """
        trigger = self.check_triggers()
        return {
            "total_calls": self._state.total_calls,
            "total_learnings_stored": self._state.total_learnings_stored,
            "calls_since_evolution": self._state.calls_since_last_evolution,
            "patterns_since_evolution": self._state.patterns_since_last_evolution,
            "should_evolve": trigger.should_evolve,
            "evolution_reasons": trigger.reasons,
            "evaluator_threshold": self._evaluator.threshold,
        }
