"""SOAR Orchestrator - Main Pipeline Coordination.

This module implements the simplified 7-phase SOAR (Sense-Orient-Adapt-Respond)
orchestrator that coordinates the entire reasoning pipeline from query assessment
to response formatting.

The 7 phases are (Task 5.7 - simplified from 9 phases):
1. Assess - Determine query complexity
2. Retrieve - Get relevant context from memory
3. Decompose - Break query into subgoals
4. Verify - Validate decomposition + assign agents (combined with routing)
5. Collect - Execute agents and gather results
6. Synthesize - Combine results into answer
7. Record - Cache reasoning patterns (lightweight)
8. Respond - Format final response

Key simplifications:
- Phase 4 (Verify) now uses verify_lite which combines validation + agent assignment
- Removed separate Route phase (agent assignment now part of verify_lite)
- Phase 7 (Record) uses record_pattern_lightweight for minimal overhead
- Total phases reduced from 9 to 7 for improved performance

The orchestrator manages:
- Phase execution and coordination
- Error handling and graceful degradation
- Budget tracking and enforcement
- Metadata aggregation
- Conversation logging
"""

from __future__ import annotations

import asyncio
import logging
import time
from typing import TYPE_CHECKING, Any, Callable

from aurora_core.budget import CostTracker
from aurora_core.exceptions import BudgetExceededError
from aurora_core.logging import ConversationLogger
from aurora_core.metrics.query_metrics import QueryMetrics
from aurora_soar import discovery_adapter
from aurora_soar.phases import (
    assess,
    collect,
    decompose,
    record,
    respond,
    retrieve,
    synthesize,
    verify,
)
from aurora_soar.phases.respond import Verbosity


if TYPE_CHECKING:
    from aurora_core.config.loader import Config
    from aurora_core.store.base import Store
    from aurora_reasoning.llm_client import LLMClient
    from aurora_soar.agent_registry import AgentRegistry


logger = logging.getLogger(__name__)


class SOAROrchestrator:
    """SOAR Pipeline Orchestrator.

    Coordinates the 9-phase reasoning pipeline with budget tracking,
    error handling, and conversation logging.

    Attributes:
        store: ACT-R memory store for context retrieval and pattern caching
        agent_registry: Registry for agent discovery and routing
        config: System configuration
        reasoning_llm: LLM client for reasoning tasks (assessment, decomposition, verification)
        solving_llm: LLM client for solving tasks (agent execution, synthesis)
    """

    def __init__(
        self,
        store: Store,
        config: Config,
        reasoning_llm: LLMClient,
        solving_llm: LLMClient,
        agent_registry: AgentRegistry | None = None,
        cost_tracker: CostTracker | None = None,
        conversation_logger: ConversationLogger | None = None,
        interactive_mode: bool = False,
        phase_callback: Callable[[str, str, dict[str, Any]], None] | None = None,
    ):
        """Initialize SOAR orchestrator.

        Args:
            store: ACT-R memory store instance
            config: System configuration
            reasoning_llm: LLM client for reasoning tasks (Tier 2 model: Sonnet/GPT-4)
            solving_llm: LLM client for solving tasks (Tier 1 model: Haiku/GPT-3.5)
            agent_registry: Optional agent registry instance. If None, uses discovery_adapter.
            cost_tracker: Optional cost tracker (creates default if not provided)
            conversation_logger: Optional conversation logger (creates default if not provided)
            interactive_mode: If True, prompt user for weak retrieval matches (CLI only, default: False)
            phase_callback: Optional callback invoked before/after each phase.
                Signature: callback(phase_name, status, result_summary)
                - phase_name: str - Name of the phase (e.g., "assess", "decompose")
                - status: str - Either "before" or "after"
                - result_summary: dict - Summary of phase result (empty for "before")
        """
        self.store = store
        self.agent_registry = agent_registry
        self.config = config
        self.interactive_mode = interactive_mode
        self.reasoning_llm = reasoning_llm
        self.solving_llm = solving_llm
        self.phase_callback = phase_callback

        # Initialize ManifestManager if agent_registry not provided
        self._use_discovery = agent_registry is None
        if self._use_discovery:
            self._manifest_manager = discovery_adapter.get_manifest_manager()

        # Initialize cost tracker
        if cost_tracker is None:
            monthly_limit = config.get("budget", {}).get("monthly_limit_usd", 100.0)
            cost_tracker = CostTracker(monthly_limit_usd=monthly_limit)
        self.cost_tracker = cost_tracker

        # Initialize conversation logger
        if conversation_logger is None:
            logging_enabled = config.get("logging", {}).get("conversation_logging_enabled", True)
            conversation_logger = ConversationLogger(enabled=logging_enabled)
        self.conversation_logger = conversation_logger

        # Initialize query metrics tracker (Task 4.5.3)
        self.query_metrics = QueryMetrics()

        # Initialize phase-level metadata tracking
        self._phase_metadata: dict[str, Any] = {}
        self._total_cost: float = 0.0
        self._token_usage: dict[str, int] = {"input": 0, "output": 0}
        self._start_time: float = 0.0
        self._query_id: str = ""
        self._query: str = ""

        logger.info("SOAR orchestrator initialized")

    def _list_agents(self):
        """List all available agents using registry or discovery adapter.

        Returns:
            List of AgentInfo objects (from either registry or discovery)
        """
        if self._use_discovery:
            return discovery_adapter.list_agents()
        else:
            return self.agent_registry.list_all()

    def _get_agent(self, agent_id: str):
        """Get agent by ID using registry or discovery adapter.

        Args:
            agent_id: Agent identifier

        Returns:
            AgentInfo object if found, None otherwise
        """
        if self._use_discovery:
            return discovery_adapter.get_agent(agent_id)
        else:
            return self.agent_registry.get(agent_id)

    def _get_or_create_fallback_agent(self):
        """Get or create a fallback agent when no suitable agent is found.

        Returns:
            AgentInfo object for fallback agent
        """
        if self._use_discovery:
            return discovery_adapter.create_fallback_agent()
        else:
            return self.agent_registry.create_fallback_agent()

    def _invoke_callback(
        self, phase_name: str, status: str, result_summary: dict[str, Any] | None = None
    ) -> None:
        """Invoke phase callback if configured.

        Exceptions from the callback are caught and logged, not propagated.

        Args:
            phase_name: Name of the phase (e.g., "assess", "decompose")
            status: Either "before" or "after"
            result_summary: Summary of phase result (empty dict for "before")
        """
        if self.phase_callback is None:
            return

        try:
            self.phase_callback(phase_name, status, result_summary or {})
        except Exception as e:
            logger.warning(f"Phase callback failed for {phase_name}/{status}: {e}")

    def execute(
        self,
        query: str,
        verbosity: str = "NORMAL",
        max_cost_usd: float | None = None,
        context_files: list[str] | None = None,
    ) -> dict[str, Any]:
        """Execute the full 9-phase SOAR pipeline.

        This is the main entry point for query processing. It coordinates all
        phases and handles errors gracefully.

        Args:
            query: User query string
            verbosity: Output verbosity level (QUIET, NORMAL, VERBOSE, JSON)
            max_cost_usd: Optional budget limit for this query (overrides config)

        Returns:
            Dict with keys:
                - answer: str (synthesized answer)
                - confidence: float (0-1)
                - overall_score: float (0-1)
                - reasoning_trace: dict (phase outputs)
                - metadata: dict (execution metadata)
                - cost_usd: float (actual cost)

        Raises:
            BudgetExceededError: If query would exceed budget limits
            ValidationError: If query is invalid or malformed
            StorageError: If memory operations fail critically
        """
        self._start_time = time.time()
        self._phase_metadata = {}
        self._total_cost = 0.0
        self._token_usage = {"input": 0, "output": 0}
        self._query = query
        self._query_id = f"soar-{int(time.time() * 1000)}"

        logger.info(f"Starting SOAR execution for query: {query[:100]}...")

        try:
            # Budget check before execution
            estimated_cost = self.cost_tracker.estimate_cost(
                model=self.reasoning_llm.default_model,
                prompt_length=len(query),
                max_output_tokens=4096,
            )

            # Check budget with raise_on_exceeded=False to get tuple return
            # We'll raise manually with proper error attributes
            can_proceed, budget_message = self.cost_tracker.check_budget(
                estimated_cost, raise_on_exceeded=False
            )

            if not can_proceed:
                # Hard limit exceeded - reject query
                status = self.cost_tracker.get_status()
                raise BudgetExceededError(
                    message=budget_message,
                    consumed_usd=status["consumed_usd"],
                    limit_usd=status["limit_usd"],
                    estimated_cost=estimated_cost,
                )

            if budget_message:
                # Soft limit warning
                logger.warning(budget_message)

            # Phase 1: Assess complexity
            phase1_result = self._phase1_assess(query)
            self._phase_metadata["phase1_assess"] = phase1_result

            # Phase 2: Retrieve context
            phase2_result = self._phase2_retrieve(query, phase1_result["complexity"])

            # Inject context files if provided (--context flag)
            if context_files:
                phase2_result = self._inject_context_files(phase2_result, context_files)

            self._phase_metadata["phase2_retrieve"] = phase2_result

            # Check for SIMPLE query early exit
            if phase1_result["complexity"] == "SIMPLE":
                # Skip decomposition, go directly to solving
                logger.info("SIMPLE query detected, bypassing decomposition")
                return self._execute_simple_path(query, phase2_result, verbosity)

            # Phase 3: Decompose query
            phase3_result = self._phase3_decompose(
                query, phase2_result, phase1_result["complexity"]
            )
            self._phase_metadata["phase3_decompose"] = phase3_result

            # Check if Phase 3 failed before proceeding
            if phase3_result.get("_error") is not None:
                logger.error(f"Phase 3 decomposition failed: {phase3_result['_error']}")
                # Return partial results with error indication
                return self._handle_decomposition_failure(query, phase3_result, verbosity)

            # Phase 4: Verify decomposition with verify_lite (Task 5.3)
            logger.info("Phase 4: Verifying decomposition")
            self._invoke_callback("verify", "before", {})

            # Extract decomposition dict and get available agents
            decomposition_dict = phase3_result.get("decomposition", phase3_result)
            available_agents = self._get_available_agents()

            # Call verify_lite which combines validation + agent assignment
            passed, agent_assignments, issues = verify.verify_lite(
                decomposition_dict, available_agents
            )

            # Build subgoal summary for display and output formatting
            subgoals = decomposition_dict.get("subgoals", [])
            subgoal_details = []
            for idx, agent in agent_assignments:
                sg = subgoals[idx] if idx < len(subgoals) else {}
                # Check if agent is a spawn (gap) - marked by verify_lite
                is_spawn = getattr(agent, "config", {}).get("is_spawn", False)
                subgoal_details.append(
                    {
                        "index": idx + 1,
                        "description": sg.get("description", ""),
                        "agent": agent.id,
                        "is_critical": sg.get("is_critical", False),
                        "depends_on": sg.get("depends_on", []),
                        "is_spawn": is_spawn,
                    }
                )

            # Invoke callback with result (including subgoal details for table display)
            self._invoke_callback(
                "verify",
                "after",
                {
                    "verdict": "PASS" if passed else "FAIL",
                    "overall_score": 1.0 if passed else 0.5,
                    "issues": issues,
                    "agents_assigned": len(agent_assignments),
                    "subgoals": subgoal_details,  # For table display
                },
            )

            phase4_result = {
                "final_verdict": "PASS" if passed else "FAIL",
                "agent_assignments": [
                    {"index": idx, "agent_id": agent.id} for idx, agent in agent_assignments
                ],
                "issues": issues,
                "subgoals_detailed": subgoal_details,  # Add detailed subgoal data
                "_timing_ms": 0,
                "_error": None,
            }
            self._phase_metadata["phase4_verify"] = phase4_result

            # Check verification verdict with auto-retry (Task 5.3)
            if not passed:
                logger.warning(f"Verification failed: {issues}. Retrying decomposition...")
                # Generate retry feedback from issues
                retry_feedback = "Please fix the following issues:\n" + "\n".join(
                    f"- {issue}" for issue in issues
                )

                # Retry decomposition with feedback
                phase3_result = self._phase3_decompose(
                    query, phase2_result, phase1_result["complexity"], retry_feedback=retry_feedback
                )
                self._phase_metadata["phase3_decompose_retry"] = phase3_result

                # Retry verification
                decomposition_dict = phase3_result.get("decomposition", phase3_result)
                passed, agent_assignments, issues = verify.verify_lite(
                    decomposition_dict, available_agents
                )

                # Rebuild subgoal details for retry
                subgoal_details = []
                for idx, agent in agent_assignments:
                    sg = subgoals[idx] if idx < len(subgoals) else {}
                    is_spawn = getattr(agent, "config", {}).get("is_spawn", False)
                    subgoal_details.append(
                        {
                            "index": idx + 1,
                            "description": sg.get("description", ""),
                            "agent": agent.id,
                            "is_critical": sg.get("is_critical", False),
                            "depends_on": sg.get("depends_on", []),
                            "is_spawn": is_spawn,
                        }
                    )

                phase4_result = {
                    "final_verdict": "PASS" if passed else "FAIL",
                    "agent_assignments": [
                        {"index": idx, "agent_id": agent.id} for idx, agent in agent_assignments
                    ],
                    "issues": issues,
                    "subgoals_detailed": subgoal_details,  # Add detailed subgoal data
                    "_timing_ms": 0,
                    "_error": None,
                }
                self._phase_metadata["phase4_verify_retry"] = phase4_result

                if not passed:
                    logger.error("Decomposition verification failed after retry")
                    return self._handle_verification_failure(query, phase4_result, verbosity)

            # Phase 5: Execute agents (Task 5.4 - removed route phase, pass agent_assignments directly)
            subgoals = decomposition_dict.get("subgoals", [])
            progress_callback = self._get_progress_callback()  # Task 5.5

            # Build context with original query for agent prompts
            collect_context = dict(phase2_result)
            collect_context["query"] = query  # Add original query for agent context

            phase5_result_obj = self._phase5_collect(
                agent_assignments, subgoals, collect_context, progress_callback
            )
            # Store dict version in metadata
            phase5_dict = phase5_result_obj.to_dict()
            phase5_dict["_timing_ms"] = 0  # Timing handled internally
            phase5_dict["_error"] = None
            phase5_dict["agents_executed"] = len(phase5_result_obj.agent_outputs)
            self._phase_metadata["phase5_collect"] = phase5_dict

            # Track spawned agents count for metrics (Task 4.5.1)
            spawned_agents_count = len(agent_assignments)

            # Track fallback to LLM count for metrics (Task 4.5.2)
            fallback_to_llm_count = len(phase5_result_obj.fallback_agents)

            # Phase 6: Synthesize results
            phase6_result_obj = self._phase6_synthesize(
                phase5_result_obj, query, decomposition_dict
            )
            # Store dict version in metadata
            phase6_dict = phase6_result_obj.to_dict()
            phase6_dict["_timing_ms"] = 0
            phase6_dict["_error"] = None
            self._phase_metadata["phase6_synthesize"] = phase6_dict

            # Phase 7: Record pattern with lightweight record (Task 5.6)
            import tempfile

            log_path = (
                self.conversation_logger.get_current_log_path()
                if hasattr(self.conversation_logger, "get_current_log_path")
                else tempfile.gettempdir() + "/soar.log"
            )
            phase7_result = self._phase7_record(
                query,
                phase6_result_obj,
                log_path,
            )
            phase7_dict = phase7_result.to_dict()
            phase7_dict["_timing_ms"] = 0
            phase7_dict["_error"] = None
            self._phase_metadata["phase7_record"] = phase7_dict

            # Record query metrics (Task 4.5.3)
            execution_duration_ms = (time.time() - self._start_time) * 1000
            self.query_metrics.record_query(
                query_id=self._query_id,
                query_type="soar",
                duration_ms=execution_duration_ms,
                query_text=query,
                complexity=phase1_result["complexity"],
                success=True,
                phase_count=7,  # Updated from 9 to 7 phases (Task 5.7)
                metadata={
                    "spawned_agents_count": spawned_agents_count,
                    "fallback_to_llm_count": fallback_to_llm_count,
                },
            )

            # Phase 8: Format response
            return self._phase8_respond(phase6_result_obj, phase7_result, verbosity)

        except BudgetExceededError:
            # Re-raise budget errors without handling - caller should handle budget limits
            raise
        except Exception as e:
            logger.exception(f"SOAR execution failed: {e}")
            return self._handle_execution_error(e, verbosity)

    def _get_available_agents(self) -> list[Any]:
        """Get list of available agents from registry or discovery system.

        Returns:
            List of available AgentInfo objects
        """
        if self._use_discovery:
            # Use discovery adapter to get agents from manifest
            manifest = self._manifest_manager.get_or_refresh()
            return manifest.agents if manifest else []
        else:
            # Use agent registry
            if not self.agent_registry:
                return []
            return self.agent_registry.list_all()

    def _get_progress_callback(self) -> callable:
        """Create progress callback for streaming agent execution updates (Task 5.5).

        Returns:
            Callback function that prints progress messages
        """

        def progress_callback(message: str) -> None:
            """Print progress message to stdout."""
            print(message, flush=True)

        return progress_callback

    def _phase1_assess(self, query: str) -> dict[str, Any]:
        """Execute Phase 1: Complexity Assessment."""
        logger.info("Phase 1: Assessing complexity")
        self._invoke_callback("assess", "before", {})
        start_time = time.time()
        try:
            result = assess.assess_complexity(query, llm_client=self.reasoning_llm)
            result["_timing_ms"] = (time.time() - start_time) * 1000
            result["_error"] = None
            self._invoke_callback(
                "assess", "after", {"complexity": result.get("complexity", "UNKNOWN")}
            )
            return result
        except Exception as e:
            logger.error(f"Phase 1 failed: {e}")
            result = {
                "complexity": "MEDIUM",
                "confidence": 0.0,
                "_timing_ms": (time.time() - start_time) * 1000,
                "_error": str(e),
            }
            self._invoke_callback("assess", "after", {"complexity": "MEDIUM", "error": str(e)})
            return result

    def _inject_context_files(
        self, phase2_result: dict[str, Any], context_files: list[str]
    ) -> dict[str, Any]:
        """Inject explicit context files into phase 2 results.

        Loads context files directly and merges them with retrieved context.
        Used for --context flag support.

        Args:
            phase2_result: Result from phase 2 (retrieve)
            context_files: List of file paths to load

        Returns:
            Modified phase2_result with injected context
        """
        from pathlib import Path

        try:
            from aurora_cli.memory.retrieval import MemoryRetriever

            retriever = MemoryRetriever()
            paths = [Path(f) for f in context_files]
            loaded_chunks = retriever.load_context_files(paths)

            if loaded_chunks:
                # Prepend to code_chunks (explicit context has priority)
                existing_chunks = phase2_result.get("code_chunks", [])
                phase2_result["code_chunks"] = loaded_chunks + existing_chunks
                logger.info(f"Injected {len(loaded_chunks)} context files into retrieval results")

        except Exception as e:
            logger.warning(f"Failed to inject context files: {e}")

        return phase2_result

    def _phase2_retrieve(self, query: str, complexity: str) -> dict[str, Any]:
        """Execute Phase 2: Context Retrieval."""
        logger.info("Phase 2: Retrieving context")
        self._invoke_callback("retrieve", "before", {})
        start_time = time.time()
        try:
            result = retrieve.retrieve_context(query, complexity, self.store)
            result["_timing_ms"] = (time.time() - start_time) * 1000
            result["_error"] = None
            chunks_count = len(result.get("code_chunks", [])) + len(
                result.get("reasoning_chunks", [])
            )
            self._invoke_callback("retrieve", "after", {"chunks_retrieved": chunks_count})
            return result
        except Exception as e:
            logger.error(f"Phase 2 failed: {e}")
            result = {
                "code_chunks": [],
                "reasoning_chunks": [],
                "_timing_ms": (time.time() - start_time) * 1000,
                "_error": str(e),
            }
            self._invoke_callback("retrieve", "after", {"chunks_retrieved": 0, "error": str(e)})
            return result

    def _phase3_decompose(
        self, query: str, context: dict[str, Any], complexity: str
    ) -> dict[str, Any]:
        """Execute Phase 3: Query Decomposition."""
        logger.info("Phase 3: Decomposing query")
        self._invoke_callback("decompose", "before", {})
        start_time = time.time()
        try:
            # Get available agents from registry or discovery
            agents = self._list_agents()
            available_agents = [agent.id for agent in agents]

            phase_result = decompose.decompose_query(
                query=query,
                context=context,
                complexity=complexity,
                llm_client=self.reasoning_llm,
                available_agents=available_agents,
            )
            result = phase_result.to_dict()
            # Add convenience fields for E2E tests
            result["subgoals_total"] = len(result["decomposition"]["subgoals"])
            result["_timing_ms"] = (time.time() - start_time) * 1000
            result["_error"] = None
            self._invoke_callback("decompose", "after", {"subgoal_count": result["subgoals_total"]})
            return result
        except Exception as e:
            logger.error(f"Phase 3 failed: {e}")
            result = {
                "goal": query,
                "subgoals": [],
                "_timing_ms": (time.time() - start_time) * 1000,
                "_error": str(e),
            }
            self._invoke_callback("decompose", "after", {"subgoal_count": 0, "error": str(e)})
            return result

    # NOTE: Old _phase4_verify and _phase5_route removed (Tasks 5.2, 6.3)
    # - Verification now uses verify_lite directly in execute() method
    # - Routing functionality integrated into verify_lite

    def _phase5_collect(
        self,
        agent_assignments: list[tuple[int, Any]],
        subgoals: list[dict[str, Any]],
        context: dict[str, Any],
        on_progress: callable = None,
    ) -> collect.CollectResult:
        """Execute Phase 5: Agent Execution (Task 5.4 - updated signature).

        Args:
            agent_assignments: List of (subgoal_index, AgentInfo) tuples
            subgoals: List of subgoal dictionaries
            context: Retrieved context from phase 2
            on_progress: Optional progress callback for streaming updates

        Returns:
            CollectResult object for use by phase 6
        """
        logger.info("Phase 5: Executing agents")
        self._invoke_callback("collect", "before", {})
        try:
            # Load recovery config from policies
            from aurora_cli.policies import PoliciesEngine

            try:
                policies = PoliciesEngine()
                recovery_config = policies.get_recovery_config()
                max_retries = recovery_config.max_retries
                fallback_to_llm = recovery_config.fallback_to_llm
                agent_timeout = recovery_config.timeout_seconds
            except Exception as e:
                logger.warning(f"Failed to load policies, using defaults: {e}")
                max_retries = 2
                fallback_to_llm = True
                agent_timeout = 300  # 5 minutes - matches collect.DEFAULT_AGENT_TIMEOUT

            # Execute agents asynchronously with recovery config
            result = asyncio.run(
                collect.execute_agents(
                    agent_assignments=agent_assignments,
                    subgoals=subgoals,
                    context=context,
                    on_progress=on_progress,
                    agent_timeout=agent_timeout,
                    max_retries=max_retries,
                    fallback_to_llm=fallback_to_llm,
                )
            )
            findings_count = len(result.agent_outputs) if result.agent_outputs else 0
            self._invoke_callback("collect", "after", {"findings_count": findings_count})
            return result
        except Exception as e:
            logger.error(f"Phase 5 collect failed: {e}")
            # Return empty CollectResult on failure
            from aurora_soar.phases.collect import CollectResult

            self._invoke_callback("collect", "after", {"findings_count": 0, "error": str(e)})
            return CollectResult(
                agent_outputs=[], execution_metadata={"error": str(e)}, user_interactions=[]
            )

    def _phase6_synthesize(
        self, collect_result: collect.CollectResult, query: str, decomposition: dict[str, Any]
    ) -> synthesize.SynthesisResult:
        """Execute Phase 6: Result Synthesis (Task 5.7 - renumbered from phase 7)."""
        logger.info("Phase 6: Synthesizing results")
        self._invoke_callback("synthesize", "before", {})
        start_time = time.time()
        try:
            result = synthesize.synthesize_results(
                llm_client=self.solving_llm,
                query=query,
                collect_result=collect_result,
                decomposition=decomposition,
            )
            # Track timing
            self._track_llm_cost(
                self.solving_llm.default_model,
                result.timing.get("input_tokens", 0),
                result.timing.get("output_tokens", 0),
                "synthesize",
            )
            self._invoke_callback("synthesize", "after", {"confidence": result.confidence})
            return result
        except Exception as e:
            logger.error(f"Phase 7 failed: {e}")
            # Return error synthesis result
            self._invoke_callback("synthesize", "after", {"confidence": 0.0, "error": str(e)})
            return synthesize.SynthesisResult(
                answer=f"Synthesis failed: {str(e)}",
                confidence=0.0,
                traceability=[],
                metadata={"error": str(e)},
                timing={"synthesis_ms": (time.time() - start_time) * 1000},
            )

    def _phase7_record(
        self,
        query: str,
        synthesis_result: synthesize.SynthesisResult,
        log_path: str,
    ) -> record.RecordResult:
        """Execute Phase 7: Pattern Recording with lightweight record (Task 5.6).

        Args:
            query: Original user query
            synthesis_result: Synthesis result from phase 6
            log_path: Path to conversation log file

        Returns:
            RecordResult with caching status
        """
        logger.info("Phase 7: Recording pattern (lightweight)")
        self._invoke_callback("record", "before", {})
        start_time = time.time()
        try:
            result = record.record_pattern_lightweight(
                store=self.store,
                query=query,
                synthesis_result=synthesis_result,
                log_path=log_path,
            )
            self._invoke_callback("record", "after", {"cached": result.cached})
            return result
        except Exception as e:
            logger.error(f"Phase 7 record failed: {e}")
            self._invoke_callback("record", "after", {"cached": False, "error": str(e)})
            return record.RecordResult(
                cached=False,
                reasoning_chunk_id=None,
                pattern_marked=False,
                activation_update=0.0,
                timing={"record_ms": (time.time() - start_time) * 1000, "error": str(e)},
            )

    def _phase8_respond(
        self,
        synthesis_result: synthesize.SynthesisResult,
        record_result: record.RecordResult,
        verbosity: str,
    ) -> dict[str, Any]:
        """Execute Phase 8: Response Formatting (Task 5.7 - renumbered from phase 9)."""
        logger.info("Phase 8: Formatting response")
        self._invoke_callback("respond", "before", {})

        # Add phase 8 metadata (Task 5.7 - updated phase number)
        self._phase_metadata["phase8_respond"] = {
            "verbosity": verbosity,
            "formatted": True,
        }

        metadata = self._build_metadata()
        response = respond.format_response(
            synthesis_result, record_result, metadata, Verbosity(verbosity.lower())
        )

        # Log conversation (async, non-blocking)
        execution_summary = {
            "duration_ms": metadata.get("total_duration_ms", 0),
            "overall_score": synthesis_result.confidence,
            "cached": record_result.cached,
            "cost_usd": metadata.get("total_cost_usd", 0.0),
            "tokens_used": metadata.get("tokens_used", {}),
        }

        log_path = self.conversation_logger.log_interaction(
            query=self._query,
            query_id=self._query_id,
            phase_data=metadata.get("phases", {}),
            execution_summary=execution_summary,
            metadata=metadata,
        )

        # Auto-index the conversation log as knowledge chunk
        if log_path and self.store:
            self._index_conversation_log(log_path)

        self._invoke_callback("respond", "after", {"formatted": True})
        return response.to_dict()

    def _execute_simple_path(
        self, query: str, context: dict[str, Any], verbosity: str
    ) -> dict[str, Any]:
        """Execute simplified path for SIMPLE queries (bypass decomposition).

        Args:
            query: User query
            context: Retrieved context
            verbosity: Output verbosity

        Returns:
            Formatted response
        """
        logger.info("Executing SIMPLE query path")
        start_time = time.time()

        # For SIMPLE queries, we skip decomposition and call solving LLM directly
        from aurora_soar.phases.record import RecordResult
        from aurora_soar.phases.synthesize import SynthesisResult

        try:
            # Build prompt with context
            prompt_parts = [f"Query: {query}"]

            # Add retrieved context if available
            code_chunks = context.get("code_chunks", [])
            reasoning_chunks = context.get("reasoning_chunks", [])

            if code_chunks:
                prompt_parts.append("\nRelevant Code:")
                for chunk in code_chunks[:5]:  # Limit to top 5
                    # Handle both CodeChunk objects and dict representations
                    if hasattr(chunk, "file_path"):
                        # CodeChunk object - build display from attributes
                        file_path = chunk.file_path
                        name = getattr(chunk, "name", "")
                        docstring = getattr(chunk, "docstring", "") or ""
                        signature = getattr(chunk, "signature", "") or ""
                        display = f"{file_path}: {name}"
                        if signature:
                            display += f" - {signature[:100]}"
                        if docstring:
                            display += f"\n  {docstring[:150]}"
                        prompt_parts.append(f"- {display}")
                    elif isinstance(chunk, dict):
                        # Dict representation
                        content = chunk.get("content", "")
                        if isinstance(content, dict):
                            # Nested content dict (from to_json format)
                            file_path = content.get("file", "")
                            name = content.get("function", "")
                            display = f"{file_path}: {name}"
                        else:
                            display = str(content)[:200]
                        prompt_parts.append(f"- {display}")
                    else:
                        # Fallback
                        prompt_parts.append(f"- {str(chunk)[:200]}")

            if reasoning_chunks:
                prompt_parts.append("\nRelevant Context:")
                for chunk in reasoning_chunks[:3]:  # Limit to top 3
                    # Handle both objects and dict representations
                    if hasattr(chunk, "pattern"):
                        pattern = chunk.pattern
                    elif isinstance(chunk, dict):
                        pattern = chunk.get("pattern", str(chunk)[:200])
                    else:
                        pattern = str(chunk)[:200]
                    prompt_parts.append(f"- {pattern}")

            prompt_parts.append("\nPlease provide a clear, concise answer:")
            prompt = "\n".join(prompt_parts)

            # Call solving LLM
            logger.info("Calling solving LLM for SIMPLE query")
            response = self.solving_llm.generate(
                prompt=prompt,
                max_tokens=1024,
                temperature=0.7,
            )

            # Track cost
            self._track_llm_cost(
                model=response.model,
                input_tokens=response.input_tokens,
                output_tokens=response.output_tokens,
                operation="simple_query_solving",
            )

            # Create synthesis result
            synthesis = SynthesisResult(
                answer=response.content,
                confidence=0.9,  # High confidence for direct LLM response
                traceability=[],
                metadata={
                    "simple_path": True,
                    "context_used": {
                        "code_chunks": len(code_chunks),
                        "reasoning_chunks": len(reasoning_chunks),
                    },
                },
                timing={
                    "synthesis_ms": (time.time() - start_time) * 1000,
                    "input_tokens": response.input_tokens,
                    "output_tokens": response.output_tokens,
                },
            )

        except Exception as e:
            logger.error(f"SIMPLE query path failed: {e}")
            synthesis = SynthesisResult(
                answer=f"Error processing query: {str(e)}",
                confidence=0.0,
                traceability=[],
                metadata={"error": str(e)},
                timing={"synthesis_ms": (time.time() - start_time) * 1000},
            )

        # Record pattern (lightweight for simple queries)
        record = RecordResult(
            cached=False,
            reasoning_chunk_id=None,
            pattern_marked=False,
            activation_update=0.0,
            timing={"record_ms": 0},
        )

        # Add simple path metadata to phase tracking
        self._phase_metadata["phase7_synthesize"] = synthesis.to_dict()
        self._phase_metadata["phase8_record"] = record.to_dict()

        # Record query metrics for simple path (Task 4.5.3)
        execution_duration_ms = (time.time() - self._start_time) * 1000
        self.query_metrics.record_query(
            query_id=self._query_id,
            query_type="simple",
            duration_ms=execution_duration_ms,
            query_text=query,
            complexity="SIMPLE",
            success=True,
            phase_count=2,  # Only assess + retrieve for simple queries
            metadata={
                "spawned_agents_count": 0,  # No agents spawned for simple queries
                "fallback_to_llm_count": 0,  # No fallback needed
            },
        )

        # Use phase 9 to format response properly
        return self._phase8_respond(synthesis, record, verbosity)

    def _handle_verification_failure(
        self, query: str, verification: dict[str, Any], verbosity: str
    ) -> dict[str, Any]:
        """Handle decomposition verification failure.

        Args:
            query: Original query
            verification: Verification result
            verbosity: Output verbosity

        Returns:
            Error response with partial results
        """
        logger.error("Returning partial results due to verification failure")
        from aurora_soar.phases.record import RecordResult
        from aurora_soar.phases.synthesize import SynthesisResult

        synthesis = SynthesisResult(
            answer="Unable to decompose query successfully. Please rephrase or simplify.",
            confidence=0.0,
            traceability=[],
            metadata={
                "error": "verification_failed",
                "feedback": verification.get("feedback", ""),
            },
            timing={"synthesis_ms": 0},
        )

        record = RecordResult(
            cached=False,
            reasoning_chunk_id=None,
            pattern_marked=False,
            activation_update=0.0,
            timing={"record_ms": 0},
        )

        # Add verification failure to phase metadata before response formatting
        self._phase_metadata["verification_failure"] = verification

        return self._phase8_respond(synthesis, record, verbosity)

    def _handle_decomposition_failure(
        self, query: str, decomposition_result: dict[str, Any], verbosity: str
    ) -> dict[str, Any]:
        """Handle decomposition failure (Phase 3 error).

        Args:
            query: Original query
            decomposition_result: Decomposition result with error
            verbosity: Output verbosity

        Returns:
            Error response with partial results
        """
        error_msg = decomposition_result.get("_error", "Unknown decomposition error")
        logger.error(f"Returning partial results due to decomposition failure: {error_msg}")
        from aurora_soar.phases.record import RecordResult
        from aurora_soar.phases.synthesize import SynthesisResult

        synthesis = SynthesisResult(
            answer="Unable to decompose query successfully. Please rephrase or simplify.",
            confidence=0.0,
            traceability=[],
            metadata={
                "error": "decomposition_failed",
                "details": error_msg,
            },
            timing={"synthesis_ms": 0},
        )

        record = RecordResult(
            cached=False,
            reasoning_chunk_id=None,
            pattern_marked=False,
            activation_update=0.0,
            timing={"record_ms": 0},
        )

        # Add decomposition failure to phase metadata before response formatting
        self._phase_metadata["decomposition_failure"] = decomposition_result

        return self._phase8_respond(synthesis, record, verbosity)

    def _handle_execution_error(self, error: Exception, verbosity: str) -> dict[str, Any]:
        """Handle execution errors with graceful degradation.

        Args:
            error: Exception that occurred
            verbosity: Output verbosity

        Returns:
            Error response
        """
        logger.error(f"Handling execution error: {error}")
        from aurora_soar.phases.record import RecordResult
        from aurora_soar.phases.synthesize import SynthesisResult

        synthesis = SynthesisResult(
            answer=f"An error occurred during query processing: {str(error)}",
            confidence=0.0,
            traceability=[],
            metadata={"error": error.__class__.__name__},
            timing={"synthesis_ms": 0},
        )

        record = RecordResult(
            cached=False,
            reasoning_chunk_id=None,
            pattern_marked=False,
            activation_update=0.0,
            timing={"record_ms": 0},
        )

        # Add error details to phase metadata
        self._phase_metadata["error_details"] = str(error)

        return self._phase8_respond(synthesis, record, verbosity)

    def _track_llm_cost(
        self,
        model: str,
        input_tokens: int,
        output_tokens: int,
        operation: str,
    ) -> float:
        """Track cost of an LLM call.

        Args:
            model: Model identifier
            input_tokens: Input tokens used
            output_tokens: Output tokens generated
            operation: Operation name (e.g., "assess", "decompose")

        Returns:
            Cost in USD
        """
        cost = self.cost_tracker.record_cost(
            model=model,
            input_tokens=input_tokens,
            output_tokens=output_tokens,
            operation=operation,
        )

        # Accumulate totals
        self._total_cost += cost
        self._token_usage["input"] += input_tokens
        self._token_usage["output"] += output_tokens

        logger.debug(
            f"LLM cost tracked: ${cost:.6f} for {operation} "
            f"({input_tokens} in, {output_tokens} out)"
        )

        return cost

    def _build_metadata(self) -> dict[str, Any]:
        """Build aggregated metadata from all phases.

        Returns:
            Dict with execution metadata
        """
        elapsed_time = time.time() - self._start_time

        return {
            "query_id": self._query_id,
            "query": self._query,
            "total_duration_ms": elapsed_time * 1000,
            "total_cost_usd": self._total_cost,
            "tokens_used": self._token_usage,
            "budget_status": self.cost_tracker.get_status(),
            "phases": self._phase_metadata,
            "timestamp": time.time(),
        }

    def _split_large_chunk_by_sections(self, chunk, max_chars: int = 2048):
        """Split a large chunk by H2 markdown sections.

        Args:
            chunk: CodeChunk to split
            max_chars: Maximum characters per chunk

        Returns:
            List of smaller chunks split by sections
        """
        from aurora_core.chunks import CodeChunk

        text = chunk.docstring or ""
        if len(text) <= max_chars:
            return [chunk]

        # Split by H2 headers (##)
        sections = []
        current_section = []
        current_header = None

        for line in text.split("\n"):
            if line.strip().startswith("## "):
                # Save previous section
                if current_section:
                    sections.append((current_header, "\n".join(current_section)))
                # Start new section
                current_header = line.strip()[3:]
                current_section = [line]
            else:
                current_section.append(line)

        # Save last section
        if current_section:
            sections.append((current_header, "\n".join(current_section)))

        # Create chunks from sections
        split_chunks = []
        truncation_suffix = "\n\n[... content truncated ...]"
        for i, (header, content) in enumerate(sections, 1):
            if len(content) > max_chars:
                # Section still too large, truncate (debug level - this is expected behavior)
                original_len = len(content)
                content = content[: max_chars - len(truncation_suffix)] + truncation_suffix
                logger.debug(
                    f"Section '{header}' truncated from {original_len} to {len(content)} chars"
                )

            # Create new chunk with section suffix
            section_chunk = CodeChunk(
                chunk_id=f"{chunk.id}_section_{i}",
                file_path=chunk.file_path,
                element_type=chunk.element_type,
                name=f"{chunk.name} - {header or 'Section ' + str(i)}",
                line_start=chunk.line_start,
                line_end=chunk.line_end,
                signature=chunk.signature,
                docstring=content,
                language=chunk.language,
                metadata={
                    **(chunk.metadata if chunk.metadata else {}),
                    "section_index": i,
                    "section_header": header,
                    "is_split_section": True,
                    "parent_chunk_id": chunk.id,
                },
            )
            split_chunks.append(section_chunk)

        logger.info(f"Split chunk {chunk.id} into {len(split_chunks)} sections")
        return split_chunks

    def _index_conversation_log(self, log_path) -> None:
        """Index conversation log as knowledge chunk for future retrieval.

        Args:
            log_path: Path to the conversation log markdown file
        """
        try:
            from pathlib import Path

            from aurora_context_code.languages.markdown import MarkdownParser
            from aurora_context_code.semantic import EmbeddingProvider

            # Parse markdown log into chunks (ensure absolute path)
            parser = MarkdownParser()
            chunks = parser.parse(Path(log_path).resolve())

            if not chunks:
                logger.debug(f"No chunks extracted from {log_path}")
                return

            # Generate embeddings and store chunks
            embedding_provider = EmbeddingProvider()
            indexed_count = 0
            split_count = 0

            for chunk in chunks:
                try:
                    # Check if chunk is too large and needs splitting
                    if len(chunk.docstring or "") > 2048:
                        # Split by H2 sections
                        section_chunks = self._split_large_chunk_by_sections(chunk)
                        split_count += len(section_chunks) - 1

                        # Index each section
                        for section_chunk in section_chunks:
                            try:
                                # Final safety check: ensure docstring is under limit
                                docstring = section_chunk.docstring or ""
                                if len(docstring) > 2048:
                                    logger.warning(
                                        f"Section {section_chunk.id} still too large after splitting "
                                        f"({len(docstring)} chars), truncating to 2048"
                                    )
                                    docstring = docstring[:2019] + "\n\n[... truncated ...]"
                                    section_chunk.docstring = docstring

                                embedding = embedding_provider.embed_chunk(docstring)
                                section_chunk.embeddings = embedding
                                self.store.save_chunk(section_chunk)
                                indexed_count += 1
                                logger.debug(f"Indexed section chunk: {section_chunk.id}")
                            except Exception as e:
                                logger.warning(f"Failed to index section {section_chunk.id}: {e}")
                                continue
                    else:
                        # Chunk is small enough, index directly
                        docstring = chunk.docstring or ""
                        # Safety check (should never trigger since we check > 2048 above)
                        if len(docstring) > 2048:
                            logger.warning(
                                f"Chunk {chunk.id} unexpectedly too large ({len(docstring)} chars), truncating"
                            )
                            docstring = docstring[:2019] + "\n\n[... truncated ...]"
                            chunk.docstring = docstring

                        embedding = embedding_provider.embed_chunk(docstring)
                        chunk.embeddings = embedding
                        self.store.save_chunk(chunk)
                        indexed_count += 1
                        logger.debug(f"Indexed conversation chunk: {chunk.id}")

                except Exception as e:
                    logger.warning(f"Failed to index chunk {chunk.id}: {e}")
                    continue

            logger.info(
                f"Indexed conversation log: {log_path} "
                f"({indexed_count} chunks, {split_count} sections created from large chunks)"
            )

        except Exception as e:
            logger.warning(f"Failed to auto-index conversation log: {e}")
            # Don't fail the query if indexing fails
