"""
MCP Strategy using ReAct pattern (Reasoning + Acting).

This strategy makes LDR behave more like Claude - thinking about requests,
calling tools (MCP and web search), analyzing results, and iterating
until the query is properly answered.

Unlike traditional strategies with fixed pipelines, the LLM decides
what to do at each step based on what it has learned.
"""

import json
import re
from datetime import UTC, datetime
from typing import Any, Dict, List, Optional

from langchain_core.language_models import BaseChatModel
from loguru import logger

from ...citation_handler import CitationHandler
from ...utilities.json_utils import extract_json
from ...utilities.search_utilities import (
    extract_links_from_search_results,
    format_links_to_markdown,
)
from .base_strategy import BaseSearchStrategy

# ReAct prompt template - simplified for better model compatibility
# Note: {current_date} placeholder is replaced at runtime
REACT_SYSTEM_PROMPT = """You are a research assistant. Today's date: {current_date}

You MUST search before answering. Do NOT answer from memory. Do NOT ask clarifying questions.

## Tool Selection — pick the right tool for the domain:

| Domain | First step (quick) | Deep dive |
|--------|-------------------|-----------|
| Medical/biomedical | **search_pubmed** | focused_research with search_engine="pubmed" |
| Scientific papers (physics, math, CS) | **search_arxiv** | focused_research with search_engine="arxiv" |
| Academic papers (all fields) | **search_semantic_scholar** or **search_openalex** | focused_research with search_engine="semantic_scholar" |
| Background knowledge | **search_wikipedia** | — |
| Current news | **search_wikinews** or **web_search** | — |
| General/mixed topics | **web_search** | focused_research |

**Strategy**: Start with a quick direct search (web_search or search_[engine]) to learn about the topic. Only use focused_research after initial exploration — it runs many search rounds and is slow.

## Tools:

- **web_search** — General web search. You control the query — use exact phrases, date ranges, site filters.
- **search_[engine]** — Single query against a specialized database (arxiv, pubmed, wikipedia, etc.). Fast and targeted.
- **focused_research** — Deep iterative research with multiple automatic search rounds. Optional parameters:
  - **iterations** (1-25): Search rounds. Default 8. Use 15-20+ for exhaustive research. More iterations = deeper.
  - **search_engine**: Override which engine to use (e.g. "pubmed", "arxiv"). Powerful for domain-specific deep dives.
- **download_content** — Fetch full text from a URL (papers, articles, web pages).

## Response format:

THOUGHT: [your reasoning]
ACTION: [tool_name]
ARGUMENTS: {{"query": "search terms"}}

After searching, give your final answer:
THOUGHT: [summary of what you found]
ANSWER: [comprehensive answer citing sources as [1], [2], etc.]

## Example:

THOUGHT: I need to find recent papers on protein folding.
ACTION: search_arxiv
ARGUMENTS: {{"query": "protein structure prediction diffusion models 2024"}}

[Results: [1] AlphaFold3 paper, [2] RFdiffusion...]

THOUGHT: I found key papers. Let me search deeper on PubMed for clinical applications.
ACTION: search_pubmed
ARGUMENTS: {{"query": "protein structure prediction clinical applications"}}

[More results...]

THOUGHT: I have enough information from both sources.
ANSWER: Recent breakthroughs in protein structure prediction include... [1][2]. Clinical applications show... [3][4].

## Rules:
1. ALWAYS search before answering
2. Use THOUGHT: then ACTION: then ARGUMENTS: format
3. Cite sources as [1], [2], etc. in your ANSWER
4. Start with quick searches, escalate to focused_research only if needed
5. Match the search engine to the domain (arxiv for science, pubmed for medical, etc.)
"""


def _to_bool(value) -> bool:
    """Convert a value to bool, handling string 'false'/'0' from settings."""
    if isinstance(value, str):
        return value.strip().lower() not in ("false", "0", "no", "")
    return bool(value)


REACT_USER_PROMPT = """## Query
{query}

## Available Tools
{tool_descriptions}

## Research History
{history}

## Your Turn
Think about what you know and what you still need. Then either call a tool or provide your final answer."""


class MCPSearchStrategy(BaseSearchStrategy):
    """
    Agentic research strategy using the ReAct pattern.

    The LLM thinks step-by-step, decides which tools to call,
    analyzes results, and continues until the query is answered.
    """

    # -- Content limits --
    # NOTE: The MCP/agentic strategy is designed for large-context LLMs (32k+).
    # These high limits ensure the agent sees full sub-research output for better
    # synthesis. For small-context models (4k-8k), use focused-iteration or
    # source-based strategies instead — they don't accumulate history this way.
    OBSERVATION_MAX_LENGTH = 50000  # Max chars kept from a single tool result
    MAX_ARG_LENGTH = 10000
    DOWNLOAD_DEFAULT_MAX_LENGTH = 10000
    HISTORY_OBSERVATION_MAX_LENGTH = (
        10000  # Max chars per observation in LLM prompt history
    )
    DIRECT_ANSWER_MIN_LENGTH = 300

    # -- Display limits --
    THOUGHT_PREVIEW_LENGTH = 200
    OBSERVATION_PREVIEW_LENGTH = 500
    RAW_RESPONSE_PREVIEW_LENGTH = 500
    ENGINE_STRENGTHS_DISPLAY_LIMIT = 2
    SOURCE_REFS_DISPLAY_LIMIT = 5

    # -- Timeouts --
    MCP_DISCOVERY_TIMEOUT = 30.0
    CONTENT_FETCHER_TIMEOUT = 30
    MCP_TOOL_CALL_TIMEOUT = 60.0

    def __init__(
        self,
        model: BaseChatModel,
        search: Any,  # Search engine instance
        citation_handler=None,
        mcp_servers: Optional[List[Dict[str, Any]]] = None,
        max_iterations: int = 10,
        include_web_search: bool = True,
        include_sub_research: bool = True,
        depth: int = 0,
        max_depth: int = 2,
        all_links_of_system: Optional[List] = None,
        settings_snapshot: Optional[Dict] = None,
        **kwargs,
    ):
        """
        Initialize the MCP strategy.

        Args:
            model: Language model for reasoning and decisions
            search: Web search engine instance
            citation_handler: Citation handler for formatting sources
            mcp_servers: List of MCP server configurations
            max_iterations: Maximum ReAct iterations (safety limit)
            include_web_search: Whether to include web_search as a tool
            include_sub_research: Whether to include sub_research tool
            depth: Current recursion depth (0 = top level)
            max_depth: Maximum recursion depth for sub-research
            all_links_of_system: Shared links list
            settings_snapshot: Settings configuration
        """
        super().__init__(
            all_links_of_system=all_links_of_system,
            settings_snapshot=settings_snapshot,
            **kwargs,
        )

        self.model = model
        self.search = search
        self.mcp_servers = mcp_servers or []
        self.max_iterations = int(
            max_iterations
        )  # Ensure int (settings may pass string)
        self.include_web_search = _to_bool(include_web_search)
        self.include_sub_research = _to_bool(include_sub_research)
        self.depth = int(depth)
        self.max_depth = int(max_depth)

        # Citation handler for proper source formatting
        self.citation_handler = citation_handler or CitationHandler(
            self.model,
            handler_type="standard",
            settings_snapshot=settings_snapshot,
        )

        # Track reasoning history
        self._history: List[Dict[str, str]] = []
        self._sources: List[str] = []
        self._findings: List[Dict[str, Any]] = []

        # Track all search results for citation handling
        self.all_search_results: List[Dict[str, Any]] = []

        # Cache for MCP tools (avoid rediscovery on every call)
        self._mcp_tools_cache: Optional[List[Dict[str, Any]]] = None

    def analyze_topic(self, query: str) -> Dict:
        """
        Analyze a topic using the ReAct pattern.

        The LLM reasons about what to do, calls tools, observes results,
        and continues until it has enough information to answer.

        Args:
            query: The research query

        Returns:
            Dict with findings, sources, iterations, etc.
        """
        logger.info(f"Starting MCP ReAct research for: {query[:100]}...")
        self._update_progress(
            f'STARTING ReAct agent for: "{query[:80]}..."',
            5,
            {"phase": "init", "type": "milestone", "query": query[:100]},
        )

        # Reset state
        self._history = []
        self._sources = []
        self._findings = []
        self.all_search_results = []

        # Build tool descriptions
        tools = self._build_tool_descriptions()

        if not tools:
            return self._create_error_response(
                "No tools available for research"
            )

        tool_descriptions = self._format_tool_descriptions(tools)

        # ReAct loop
        iteration = 0
        final_answer = None

        while iteration < self.max_iterations:
            iteration += 1
            progress = 10 + int((iteration / self.max_iterations) * 80)

            self._update_progress(
                f"CYCLE {iteration}/{self.max_iterations}: Agent is reasoning about what to do next...",
                progress,
                {
                    "phase": "react",
                    "iteration": iteration,
                    "max": self.max_iterations,
                },
            )

            # Get LLM response
            try:
                response = self._get_llm_response(
                    query, tool_descriptions, tools
                )
            except Exception as e:
                logger.exception("LLM call failed")
                return self._create_error_response(f"LLM error: {e}")

            # Parse the response
            parsed = self._parse_response(response)

            if parsed["type"] == "answer":
                # LLM decided it has enough information
                final_answer = parsed["content"]
                self._history.append(
                    {
                        "role": "assistant",
                        "thought": parsed.get("thought", ""),
                        "answer": final_answer,
                    }
                )
                logger.info(
                    f"ReAct completed with answer after {iteration} iterations"
                )
                break

            if parsed["type"] == "action":
                # LLM wants to call a tool
                tool_name = parsed.get("tool")
                arguments = parsed.get("arguments", {})
                thought = parsed.get("thought", "")

                # Validate required action fields
                if not tool_name:
                    logger.warning(
                        "Action missing 'tool' field, treating as error"
                    )
                    self._history.append(
                        {
                            "role": "system",
                            "error": "Invalid action: missing tool name. Please specify which tool to use.",
                        }
                    )
                    continue

                # Show detailed progress with thought and action
                thought_preview = (
                    thought[: self.THOUGHT_PREVIEW_LENGTH]
                    if thought
                    else "Analyzing..."
                )
                search_query = arguments.get("query", str(arguments))

                self._update_progress(
                    f"THINKING: {thought_preview}",
                    progress,
                    {"phase": "thought", "thought": thought},
                )
                self._update_progress(
                    f'ACTION: Using {tool_name} - "{search_query}"',
                    progress + 2,
                    {
                        "phase": "tool_call",
                        "tool": tool_name,
                        "arguments": arguments,
                    },
                )

                # Execute the tool
                try:
                    result = self._execute_tool(tool_name, arguments, tools)
                    if not isinstance(result, dict):
                        result = (
                            {"content": str(result)}
                            if result
                            else {"content": "No result returned"}
                        )
                    observation = result.get("content", str(result))

                    # Truncate very long observations
                    if len(observation) > self.OBSERVATION_MAX_LENGTH:
                        observation = (
                            observation[: self.OBSERVATION_MAX_LENGTH]
                            + "\n... [truncated]"
                        )

                    # Show observation summary with actual content
                    obs_preview = observation[
                        : self.OBSERVATION_PREVIEW_LENGTH
                    ].replace("\n", " ")
                    self._update_progress(
                        f"RESULT: {obs_preview}",
                        progress + 4,
                        {
                            "phase": "observation",
                            "tool": tool_name,
                            "result_length": len(observation),
                            "content": obs_preview,
                        },
                    )

                except Exception as e:
                    logger.exception("Tool execution failed")
                    observation = f"Error: Tool '{tool_name}' failed: {e}"
                    self._update_progress(
                        f"ERROR: {tool_name} failed - {str(e)[:100]}",
                        progress + 4,
                        {"phase": "error", "tool": tool_name, "error": str(e)},
                    )

                # Add to history
                self._history.append(
                    {
                        "role": "assistant",
                        "thought": thought,
                        "action": tool_name,
                        "arguments": arguments,
                    }
                )
                self._history.append(
                    {
                        "role": "tool",
                        "tool": tool_name,
                        "observation": observation,
                    }
                )

            elif parsed["type"] == "error":
                # Parsing error - show what the LLM said anyway
                raw_response = parsed.get("raw", "")[
                    : self.RAW_RESPONSE_PREVIEW_LENGTH
                ]
                logger.warning(
                    f"Failed to parse LLM response: {parsed['message']}"
                )

                # Show the raw LLM output to the user so they can see what it's thinking
                self._update_progress(
                    f"LLM Response (unparsed): {raw_response}",
                    progress,
                    {
                        "phase": "thought",
                        "thought": raw_response,
                        "parse_error": True,
                    },
                )

                self._history.append(
                    {
                        "role": "system",
                        "error": f"Invalid response format: {parsed['message']}. Please use the correct format.",
                    }
                )

        # If we hit max iterations without an answer, synthesize one
        if final_answer is None:
            logger.warning(
                f"Max iterations ({self.max_iterations}) reached, synthesizing answer"
            )
            final_answer = self._synthesize_answer(query)

        # Use citation handler for final synthesis with proper source formatting
        self._update_progress(
            f"Synthesizing {len(self.all_search_results)} sources with citations...",
            90,
            {"phase": "synthesis", "type": "milestone"},
        )

        # If we have search results, use citation handler for proper formatting
        synthesized_content = final_answer
        documents = []
        if self.all_search_results:
            try:
                citation_result = self.citation_handler.analyze_followup(
                    query,
                    self.all_search_results,
                    previous_knowledge=final_answer,
                    nr_of_links=0,
                )
                # Handle case where citation_result is None or not a dict
                if citation_result is None or not isinstance(
                    citation_result, dict
                ):
                    logger.warning(
                        "Citation handler returned None or non-dict, using raw answer"
                    )
                    synthesized_content = final_answer
                else:
                    synthesized_content = citation_result.get(
                        "content", citation_result.get("response", final_answer)
                    )
                    documents = citation_result.get("documents", [])
                    logger.info(
                        f"Citation handler produced {len(documents)} documents"
                    )
            except Exception:
                logger.warning("Citation handler failed, using raw answer")
                synthesized_content = final_answer

        # Format sources as markdown bibliography and append to output
        formatted_output = synthesized_content
        logger.info(
            f"MCP Strategy: all_search_results has {len(self.all_search_results)} items"
        )
        if self.all_search_results:
            try:
                all_links = extract_links_from_search_results(
                    self.all_search_results
                )
                logger.info(
                    f"MCP Strategy: extracted {len(all_links)} links from search results"
                )
                if all_links:
                    sources_markdown = format_links_to_markdown(all_links)
                    logger.info(
                        f"MCP Strategy: sources_markdown length={len(sources_markdown) if sources_markdown else 0}"
                    )
                    if sources_markdown:
                        formatted_output = f"{synthesized_content}\n\n## Sources\n\n{sources_markdown}"
                        logger.info(
                            f"MCP Strategy: Appended {len(all_links)} sources to output"
                        )
                    else:
                        logger.warning(
                            "MCP Strategy: sources_markdown is empty despite having links"
                        )
                else:
                    logger.warning(
                        "MCP Strategy: No links extracted from search results"
                    )
            except Exception:
                logger.exception("MCP Strategy: Failed to format source links")
        else:
            logger.warning("MCP Strategy: all_search_results is empty")

        # Add final finding WITH search_results for research_service.py to extract
        # This is critical for sources to be saved to database and displayed
        final_finding = {
            "content": synthesized_content,
            "question": query,
            "search_results": self.all_search_results,
            "documents": documents,
        }
        self._findings.append(final_finding)

        # Build result
        self._update_progress(
            "Research complete",
            100,
            {"phase": "complete", "iterations": iteration},
        )

        # Extract actual search queries from history (not just tool names)
        questions = {}
        q_idx = 0
        for h in self._history:
            if h.get("action") and h.get("arguments"):
                action_query = h["arguments"].get(
                    "query", h["arguments"].get("question", "")
                )
                if action_query:
                    questions[q_idx] = [action_query]
                    q_idx += 1

        return {
            "findings": self._findings,
            "iterations": iteration,
            "questions": questions,
            "formatted_findings": formatted_output,
            "current_knowledge": synthesized_content,
            "sources": list({s for s in self._sources if isinstance(s, str)}),
            "search_results": self.all_search_results,
            "documents": documents,
            "reasoning_trace": self._history,
        }

    def _build_tool_descriptions(self) -> List[Dict[str, Any]]:
        """Build list of available tools with descriptions."""
        tools = []

        # Add web search if enabled
        if self.include_web_search and self.search:
            tools.append(
                {
                    "name": "web_search",
                    "description": "Search the web for information. Use this for quick searches to find current information, facts, news, or any web content. Returns search result snippets.",
                    "parameters": {
                        "query": {
                            "type": "string",
                            "description": "The search query",
                            "required": True,
                        }
                    },
                    "executor": self._execute_web_search,
                }
            )

        # Add strategy-based research tools if enabled and not at max depth
        logger.info(
            f"Sub-research check: enabled={self.include_sub_research}, depth={self.depth}, max_depth={self.max_depth}"
        )
        if self.include_sub_research and self.depth < self.max_depth:
            _sub_research_params = {
                "query": {
                    "type": "string",
                    "description": "The specific research question to investigate",
                    "required": True,
                },
                "iterations": {
                    "type": "integer",
                    "description": "Number of search iterations (1-25). More iterations = deeper research. Use 3-5 for quick checks, 10-15 for thorough, 20+ for exhaustive. Prefer more iterations over broader queries.",
                    "required": False,
                },
                "search_engine": {
                    "type": "string",
                    "description": "Override the search engine for this research (e.g. 'arxiv', 'pubmed', 'wikipedia', 'searxng'). By default uses the main configured engine.",
                    "required": False,
                },
            }

            # Focused iteration - deep iterative research
            tools.append(
                {
                    "name": "focused_research",
                    "description": "Deep iterative research that runs multiple search rounds automatically. Best for complex topics needing thorough investigation. Use 'search_engine' to target specific databases (e.g. 'pubmed' for medical, 'arxiv' for scientific). Use 'iterations' to control depth — default 8, but 15-20+ is better for exhaustive research.",
                    "parameters": _sub_research_params,
                    "executor": lambda args: self._execute_strategy_research(
                        args, "focused-iteration"
                    ),
                }
            )

        # Add specialized search engine tools dynamically from available engines
        available_engines = self._get_available_search_engines()
        for engine_name, engine_config in available_engines.items():
            # Skip the currently selected engine (already available via web_search)
            # Skip 'auto' and meta engines
            if (
                engine_name in ("auto", "meta")
                or engine_name == self._get_current_engine_name()
            ):
                continue

            # Create tool for this engine
            description = engine_config.get(
                "description", f"Search using {engine_name}"
            )
            strengths = engine_config.get("strengths", [])
            if strengths:
                description += f" Best for: {', '.join(strengths[: self.ENGINE_STRENGTHS_DISPLAY_LIMIT])}."

            tools.append(
                {
                    "name": f"search_{engine_name}",
                    "description": description,
                    "parameters": {
                        "query": {
                            "type": "string",
                            "description": "The search query",
                            "required": True,
                        }
                    },
                    "executor": lambda args, eng=engine_name: (
                        self._execute_specialized_search(args, eng)
                    ),
                }
            )

        # Add content download tool for fetching full papers/articles
        tools.append(
            {
                "name": "download_content",
                "description": "Download and extract full text content from a URL. Works with academic papers (arXiv, PubMed, Semantic Scholar, bioRxiv) and web pages. Use this when you need the complete text of a paper or article found in search results, not just the snippet.",
                "parameters": {
                    "url": {
                        "type": "string",
                        "description": "The URL to download content from",
                        "required": True,
                    },
                    "max_length": {
                        "type": "integer",
                        "description": f"Maximum characters to return (default {self.DOWNLOAD_DEFAULT_MAX_LENGTH})",
                        "required": False,
                    },
                },
                "executor": self._execute_download_content,
            }
        )

        # Add MCP tools (if configured)
        mcp_tools = self._discover_mcp_tools()
        tools.extend(mcp_tools)

        return tools

    def _discover_mcp_tools(self) -> List[Dict[str, Any]]:
        """Discover tools from configured MCP servers.

        Results are cached only on successful discovery to allow retry on failure.
        """
        # Return cached tools if available
        if self._mcp_tools_cache is not None:
            return self._mcp_tools_cache

        tools = []

        if not self.mcp_servers:
            return tools

        try:
            from local_deep_research.mcp.client import (
                MCPClientManager,
                run_async,
            )

            async def discover():
                manager = MCPClientManager(self.mcp_servers)
                async with manager.connect_all():
                    return await manager.list_all_tools()

            # Use shorter timeout for discovery instead of default 300s
            all_tools = run_async(
                discover(), timeout=self.MCP_DISCOVERY_TIMEOUT
            )

            for server_name, server_tools in all_tools.items():
                for tool in server_tools:
                    tools.append(
                        {
                            "name": f"{server_name}.{tool['name']}",
                            "description": tool.get(
                                "description", "No description"
                            ),
                            "parameters": tool.get("input_schema", {}).get(
                                "properties", {}
                            ),
                            "mcp_server": server_name,
                            "mcp_tool": tool["name"],
                        }
                    )

            # Only cache successful discoveries (when tools were found)
            if tools:
                self._mcp_tools_cache = tools
                logger.info(f"Discovered and cached {len(tools)} MCP tools")

        except Exception:
            # Don't cache failures - allow retry on next call
            logger.warning("Failed to discover MCP tools (will retry)")

        return tools

    def _format_tool_descriptions(self, tools: List[Dict[str, Any]]) -> str:
        """Format tool descriptions for the LLM."""
        lines = []
        for tool in tools:
            params = tool.get("parameters", {})
            param_str = ", ".join(
                f"{k}: {v.get('type', 'any')}" for k, v in params.items()
            )
            lines.append(f"- **{tool['name']}**({param_str})")
            lines.append(f"  {tool['description']}")
            lines.append("")
        return "\n".join(lines)

    def _format_history(self) -> str:
        """Format the reasoning history for the LLM."""
        if not self._history:
            return "No research done yet."

        lines = []
        for entry in self._history:
            role = entry.get("role", "")

            if role == "assistant":
                if "thought" in entry:
                    lines.append(f"THOUGHT: {entry['thought']}")
                if "action" in entry:
                    lines.append(f"ACTION: {entry['action']}")
                    lines.append(
                        f"ARGUMENTS: {json.dumps(entry.get('arguments', {}))}"
                    )
                if "answer" in entry:
                    lines.append(f"ANSWER: {entry['answer'][:500]}...")

            elif role == "tool":
                obs = entry.get("observation", "")
                if len(obs) > self.HISTORY_OBSERVATION_MAX_LENGTH:
                    obs = obs[: self.HISTORY_OBSERVATION_MAX_LENGTH] + "..."
                lines.append(
                    f"OBSERVATION ({entry.get('tool', 'unknown')}): {obs}"
                )

            elif role == "system":
                if "error" in entry:
                    lines.append(f"SYSTEM: {entry['error']}")

            lines.append("")

        return "\n".join(lines)

    def _get_llm_response(
        self, query: str, tool_descriptions: str, tools: List[Dict[str, Any]]
    ) -> str:
        """Get a response from the LLM with tools bound for native tool calling."""
        history = self._format_history()

        prompt = REACT_USER_PROMPT.format(
            query=query,
            tool_descriptions=tool_descriptions,
            history=history,
        )

        # Format system prompt with current date
        current_date = datetime.now(UTC).strftime("%Y-%m-%d")
        system_prompt = REACT_SYSTEM_PROMPT.format(current_date=current_date)

        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": prompt},
        ]

        # Build tool schemas for native tool calling
        tool_schemas = self._build_tool_schemas(tools)
        logger.info(
            f"Built {len(tool_schemas)} tool schemas: {[s['function']['name'] for s in tool_schemas]}"
        )

        # Bind tools to model if we have schemas
        if tool_schemas:
            try:
                model_with_tools = self.model.bind_tools(tool_schemas)
                response = model_with_tools.invoke(messages)
            except Exception:
                logger.warning("Failed to bind tools, using text-based")
                response = self.model.invoke(messages)
        else:
            response = self.model.invoke(messages)

        # Return the full response object so we can handle both text and tool calls
        return response

    def _build_tool_schemas(
        self, tools: List[Dict[str, Any]]
    ) -> List[Dict[str, Any]]:
        """Build tool schemas for native LLM tool calling."""
        schemas = []
        for tool in tools:
            # Build JSON schema for the tool
            properties = {}
            required = []
            for param_name, param_info in tool.get("parameters", {}).items():
                properties[param_name] = {
                    "type": param_info.get("type", "string"),
                    "description": param_info.get("description", ""),
                }
                if param_info.get("required", False):
                    required.append(param_name)

            schema = {
                "type": "function",
                "function": {
                    "name": tool["name"],
                    "description": tool["description"],
                    "parameters": {
                        "type": "object",
                        "properties": properties,
                        "required": required,
                    },
                },
            }
            schemas.append(schema)

        return schemas

    def _parse_response(self, response) -> Dict[str, Any]:
        """Parse the LLM's response to extract thought/action/answer.

        Handles both native tool calls and text-based responses.
        """
        # Check for native tool calls first (preferred)
        if (
            hasattr(response, "tool_calls")
            and response.tool_calls
            and len(response.tool_calls) > 0
        ):
            tool_call = response.tool_calls[0]
            tool_name = tool_call.get("name", "unknown")
            tool_args = tool_call.get("args", {})

            # Get any text content as the "thought"
            thought = ""
            if hasattr(response, "content") and response.content:
                thought = response.content
            else:
                thought = f"Calling {tool_name} to find information"

            logger.info(f"Native tool call: {tool_name} with args {tool_args}")

            return {
                "type": "action",
                "thought": thought,
                "tool": tool_name,
                "arguments": tool_args,
            }

        # Extract text content for text-based parsing
        if hasattr(response, "content"):
            response = response.content
        response = str(response).strip()

        # Log raw response at INFO level so we can see what the LLM generates
        logger.info(
            f"Raw LLM response ({len(response)} chars): {response[:800]}..."
        )

        # Try to extract THOUGHT
        thought = ""
        thought_match = re.search(
            r"THOUGHT:\s*(.+?)(?=(?:ACTION:|ANSWER:|$))",
            response,
            re.DOTALL | re.IGNORECASE,
        )
        if thought_match:
            thought = thought_match.group(1).strip()

        # Check for ANSWER
        answer_match = re.search(
            r"ANSWER:\s*(.+)", response, re.DOTALL | re.IGNORECASE
        )
        if answer_match:
            return {
                "type": "answer",
                "thought": thought,
                "content": answer_match.group(1).strip(),
            }

        # Check for ACTION
        action_match = re.search(r"ACTION:\s*(\S+)", response, re.IGNORECASE)
        if action_match:
            tool_name = action_match.group(1).strip()

            # Try to extract ARGUMENTS using centralized JSON parser
            arguments = {}
            args_section = re.search(
                r"ARGUMENTS:\s*(.+?)(?=(?:THOUGHT:|ACTION:|ANSWER:|$))",
                response,
                re.DOTALL | re.IGNORECASE,
            )
            if args_section:
                parsed_args = extract_json(
                    args_section.group(1), expected_type=dict
                )
                if parsed_args is not None:
                    arguments = parsed_args
                else:
                    logger.warning(
                        f"Failed to parse ARGUMENTS JSON: {args_section.group(1)[:200]}"
                    )

            # If no JSON arguments, try to find query parameter
            if not arguments:
                query_match = re.search(
                    r'["\']?query["\']?\s*[:=]\s*["\'](.+?)["\']',
                    response,
                    re.IGNORECASE,
                )
                if query_match:
                    arguments = {"query": query_match.group(1)}

            return {
                "type": "action",
                "thought": thought,
                "tool": tool_name,
                "arguments": arguments,
            }

        # FALLBACK: Try to detect intent from response even without proper format
        response_lower = response.lower()

        # Check if model wants to search (common patterns)
        search_patterns = [
            r"(?:let me |i will |i should |i need to |going to )(?:search|look up|find|research)",
            r"search(?:ing)? for",
            r"look(?:ing)? up",
            r"find(?:ing)? information",
        ]
        for pattern in search_patterns:
            if re.search(pattern, response_lower):
                # Extract what they want to search for
                # Try to find quoted text or text after "for" or "about"
                search_query = None
                quoted = re.search(r'["\']([^"\']+)["\']', response)
                if quoted:
                    search_query = quoted.group(1)
                else:
                    about_match = re.search(
                        r"(?:for|about|on)\s+(.+?)(?:\.|$)",
                        response,
                        re.IGNORECASE,
                    )
                    if about_match:
                        search_query = about_match.group(1).strip()[:100]

                if search_query:
                    logger.info(
                        f"Fallback parser detected search intent: {search_query}"
                    )
                    return {
                        "type": "action",
                        "thought": response[: self.THOUGHT_PREVIEW_LENGTH],
                        "tool": "web_search",
                        "arguments": {"query": search_query},
                    }

        # If response is long and looks like a direct answer, treat as answer
        if len(response) > self.DIRECT_ANSWER_MIN_LENGTH:
            logger.info("Fallback parser treating long response as answer")
            return {
                "type": "answer",
                "thought": "Model provided direct answer without following format",
                "content": response,
            }

        # If we can't parse, return error with the full raw response for display
        logger.warning(f"Could not parse LLM response: {response[:300]}...")
        return {
            "type": "error",
            "message": "Could not parse response. Expected THOUGHT/ACTION or THOUGHT/ANSWER format.",
            "raw": response,  # Include full response so user can see what LLM said
        }

    def _validate_tool_arguments(
        self, arguments: Dict[str, Any]
    ) -> Dict[str, Any]:
        """
        Validate and sanitize tool arguments.

        Args:
            arguments: Raw arguments from LLM

        Returns:
            Sanitized arguments dictionary
        """
        if not isinstance(arguments, dict):
            return {}

        sanitized = {}
        for key, value in arguments.items():
            # Ensure key is a string
            if not isinstance(key, str):
                continue
            # Truncate very long string values
            if isinstance(value, str) and len(value) > self.MAX_ARG_LENGTH:
                value = value[: self.MAX_ARG_LENGTH]
                logger.warning(
                    f"Truncated argument '{key}' to {self.MAX_ARG_LENGTH} chars"
                )
            sanitized[key] = value

        return sanitized

    def _execute_tool(
        self,
        tool_name: str,
        arguments: Dict[str, Any],
        tools: List[Dict[str, Any]],
    ) -> Dict[str, Any]:
        """Execute a tool and return the result."""
        # Validate and sanitize arguments
        arguments = self._validate_tool_arguments(arguments)

        # Find the tool
        tool = None
        for t in tools:
            if t["name"] == tool_name:
                tool = t
                break

        if not tool:
            return {"status": "error", "content": f"Unknown tool: {tool_name}"}

        # Execute based on tool type
        if "executor" in tool:
            # Built-in tool with executor function
            return tool["executor"](arguments)
        if "mcp_server" in tool:
            # MCP tool
            return self._execute_mcp_tool(
                tool["mcp_server"],
                tool["mcp_tool"],
                arguments,
            )
        return {
            "status": "error",
            "content": f"Tool {tool_name} has no executor",
        }

    def _execute_web_search(self, arguments: Dict[str, Any]) -> Dict[str, Any]:
        """Execute web search tool."""
        query = arguments.get("query", "")
        if not query:
            return {"status": "error", "content": "No query provided"}

        try:
            results = self.search.run(query)

            # Track sources and accumulate search results for citation handling
            if isinstance(results, list):
                # Handle empty results
                if not results:
                    logger.info(f"Web search returned no results for: {query}")
                    return {
                        "status": "success",
                        "content": f"No search results found for '{query}'. Try rephrasing the query or using a different search approach.",
                    }

                for r in results:
                    if isinstance(r, dict):
                        # Track source URLs
                        if "link" in r:
                            self._sources.append(r["link"])

                        # Add to all_search_results for citation handler
                        # Assign index based on current count
                        result_with_index = dict(r)
                        result_with_index["index"] = str(
                            len(self.all_search_results) + 1
                        )
                        self.all_search_results.append(result_with_index)

                        # Also add to shared links
                        if self.all_links_of_system is not None:
                            self.all_links_of_system.append(result_with_index)

                # Format with index, title, URL, and snippet for LLM
                content = "\n\n".join(
                    f"[{len(self.all_search_results) - len(results) + i + 1}] {r.get('title', 'No title')} ({r.get('link', 'no url')})\n{r.get('snippet', r.get('body', ''))}"
                    for i, r in enumerate(results)
                    if isinstance(r, dict)
                )
            else:
                content = str(results)

            # Add to findings
            self._findings.append(
                {
                    "type": "web_search",
                    "query": query,
                    "result_count": len(results)
                    if isinstance(results, list)
                    else 1,
                }
            )

            return {"status": "success", "content": content}

        except Exception as e:
            logger.exception("Web search failed")
            return {"status": "error", "content": f"Search failed: {e}"}

    def _execute_download_content(
        self, arguments: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Execute content download tool.

        Downloads full text content from URLs including academic papers
        (arXiv, PubMed, Semantic Scholar) and web pages.
        """
        url = arguments.get("url", "")
        if not url:
            return {"status": "error", "content": "No URL provided"}

        max_length = arguments.get(
            "max_length", self.DOWNLOAD_DEFAULT_MAX_LENGTH
        )

        try:
            from local_deep_research.content_fetcher import ContentFetcher

            with ContentFetcher(
                timeout=self.CONTENT_FETCHER_TIMEOUT
            ) as fetcher:
                result = fetcher.fetch(url, max_length=max_length)

                if result.get("status") == "success":
                    content = result.get("content", "")
                    source_type = result.get("source_type", "unknown")
                    title = result.get("title", "")

                    # Format output
                    output_parts = []
                    if title:
                        output_parts.append(f"Title: {title}")
                    output_parts.append(f"Source: {source_type}")
                    output_parts.append(f"URL: {url}")
                    output_parts.append("")
                    output_parts.append(content)

                    formatted_content = "\n".join(output_parts)

                    # Track as a finding
                    self._findings.append(
                        {
                            "type": "content_download",
                            "url": url,
                            "source_type": source_type,
                            "title": title,
                            "content_length": len(content),
                        }
                    )

                    # Track source
                    self._sources.append(url)

                    logger.info(
                        f"Downloaded {len(content)} chars from {source_type}: {url}"
                    )

                    return {"status": "success", "content": formatted_content}

                error_msg = result.get("error", "Download failed")
                logger.warning(f"Content download failed: {error_msg}")
                return {"status": "error", "content": error_msg}

        except ImportError:
            logger.warning("ContentFetcher not available")
            return {
                "status": "error",
                "content": "Content fetcher module not available",
            }
        except Exception as e:
            logger.exception(f"Content download failed for {url}")
            return {"status": "error", "content": f"Download failed: {e}"}

    def _execute_strategy_research(
        self, arguments: Dict[str, Any], strategy_name: str
    ) -> Dict[str, Any]:
        """Execute research using a specific strategy.

        This delegates to other research strategies (focused-iteration,
        source-based, comprehensive) for deeper investigation.

        The agent can control sub-research via optional arguments:
        - iterations: number of search rounds (1-25)
        - search_engine: override the search engine (e.g. "pubmed", "arxiv")
        """
        sub_query = arguments.get("query", "")
        if not sub_query:
            return {"status": "error", "content": "No query provided"}

        # Check recursion depth
        if self.depth >= self.max_depth:
            return {
                "status": "error",
                "content": f"Maximum research depth ({self.max_depth}) reached",
            }

        # Extract optional parameters from agent arguments
        iterations = arguments.get("iterations")
        search_engine_name = arguments.get("search_engine")

        # Clamp iterations to safe range
        if iterations is not None:
            try:
                iterations = max(1, min(25, int(iterations)))
            except (ValueError, TypeError):
                iterations = None

        try:
            # Import factory to create strategy
            from local_deep_research.search_system_factory import (
                create_strategy,
            )

            # Resolve the search engine override if requested
            search_instance = self.search
            actual_engine_name = None
            if search_engine_name:
                try:
                    from local_deep_research.web_search_engines.search_engine_factory import (
                        create_search_engine,
                    )

                    override_engine = create_search_engine(
                        engine_name=search_engine_name,
                        llm=self.model,
                        settings_snapshot=self.settings_snapshot,
                    )
                    if override_engine:
                        search_instance = override_engine
                        actual_engine_name = search_engine_name
                        logger.info(
                            f"Sub-research using overridden engine: {search_engine_name}"
                        )
                    else:
                        logger.warning(
                            f"Could not create engine '{search_engine_name}', "
                            f"falling back to default"
                        )
                except Exception:
                    logger.warning(
                        f"Failed to create engine '{search_engine_name}'"
                        f"falling back to default"
                    )

            # Log and emit progress after engine resolution so we report the actual engine
            info_parts = [
                f"Delegating to {strategy_name} strategy: {sub_query[:100]}"
            ]
            if iterations:
                info_parts.append(f"iterations={iterations}")
            if actual_engine_name:
                info_parts.append(f"engine={actual_engine_name}")
            logger.info(", ".join(info_parts))

            progress_msg = f"DELEGATING to {strategy_name}: {sub_query[:80]}..."
            if actual_engine_name:
                progress_msg += f" (using {actual_engine_name})"
            self._update_progress(
                progress_msg,
                50,
                {
                    "phase": "sub_research",
                    "strategy": strategy_name,
                    "depth": self.depth + 1,
                    "query": sub_query[:100],
                },
            )

            # Build kwargs for the strategy factory
            strategy_kwargs: Dict[str, Any] = {
                "depth": self.depth + 1,
                "max_depth": self.max_depth,
            }
            if iterations is not None:
                strategy_kwargs["max_iterations"] = iterations

            # Create the delegated strategy with incremented depth
            child_strategy = create_strategy(
                strategy_name=strategy_name,
                model=self.model,
                search=search_instance,
                all_links_of_system=self.all_links_of_system,
                settings_snapshot=self.settings_snapshot,
                **strategy_kwargs,
            )

            # Run the research
            result = child_strategy.analyze_topic(sub_query)

            # Merge sources from child into parent
            child_sources = []
            child_findings = result.get("findings", [])
            logger.info(
                f"MCP Strategy: Merging sources from {strategy_name}, got {len(child_findings)} findings"
            )
            for finding in child_findings:
                finding_sources = finding.get("search_results", [])
                logger.info(
                    f"MCP Strategy: Finding has {len(finding_sources)} search_results"
                )
                for source in finding_sources:
                    # Re-index sources to continue from parent's count
                    source_copy = dict(source)
                    source_copy["index"] = str(len(self.all_search_results) + 1)
                    self.all_search_results.append(source_copy)
                    child_sources.append(source_copy)

                    if self.all_links_of_system is not None:
                        self.all_links_of_system.append(source_copy)

            logger.info(
                f"MCP Strategy: After merge, all_search_results has {len(self.all_search_results)} items"
            )

            # Add findings to parent
            for finding in child_findings:
                finding["from_strategy"] = strategy_name
                finding["sub_query"] = sub_query
                self._findings.append(finding)

            # Track sources (only strings)
            for src in result.get("sources", []):
                if isinstance(src, str):
                    self._sources.append(src)
                elif isinstance(src, dict) and "link" in src:
                    self._sources.append(src["link"])

            # Return the synthesized content
            content = result.get("formatted_findings", "")
            if not content:
                content = result.get("current_knowledge", "No results found")

            # Add source references to the content
            if child_sources:
                source_refs = ", ".join(
                    f"[{s.get('index', '?')}]"
                    for s in child_sources[-self.SOURCE_REFS_DISPLAY_LIMIT :]
                )
                content = f"{content}\n\nSources: {source_refs}"

            logger.info(
                f"{strategy_name} research completed with {len(child_sources)} sources"
            )

            return {"status": "success", "content": content}

        except Exception as e:
            logger.exception(f"{strategy_name} research failed")
            return {
                "status": "error",
                "content": f"{strategy_name} research failed: {e}",
            }
        finally:
            # Close the overridden engine if we created one
            if search_engine_name and search_instance is not self.search:
                try:
                    from local_deep_research.utilities.resource_utils import (
                        safe_close,
                    )

                    safe_close(
                        search_instance,
                        f"sub-research engine ({search_engine_name})",
                    )
                except Exception:
                    logger.debug(
                        f"Failed to close sub-research engine {search_engine_name}",
                        exc_info=True,
                    )

    def _get_available_search_engines(self) -> Dict[str, Any]:
        """Get search engines that are actually usable (enabled for auto-search
        and with valid API keys)."""
        try:
            from local_deep_research.web_search_engines.search_engines_config import (
                get_available_engines,
            )

            return get_available_engines(
                settings_snapshot=self.settings_snapshot,
            )
        except Exception:
            logger.warning("Failed to get available search engines")
            return {}

    def _get_current_engine_name(self) -> str:
        """Get the name of the currently selected search engine."""
        try:
            if hasattr(self.search, "__class__"):
                # Extract engine name from class name
                class_name = self.search.__class__.__name__
                # Convert "ArXivSearchEngine" -> "arxiv"
                return class_name.replace("SearchEngine", "").lower()
        except Exception:
            logger.debug("best-effort class name extraction", exc_info=True)
        return ""

    def _execute_specialized_search(
        self, arguments: Dict[str, Any], engine_name: str
    ) -> Dict[str, Any]:
        """Execute search using a specific search engine."""
        query = arguments.get("query", "")
        if not query:
            return {"status": "error", "content": "No query provided"}

        try:
            from local_deep_research.web_search_engines.search_engine_factory import (
                create_search_engine,
            )

            logger.info(f"Specialized search with {engine_name}: {query[:100]}")

            # Create the specialized search engine
            engine = create_search_engine(
                engine_name=engine_name,
                llm=self.model,
                settings_snapshot=self.settings_snapshot,
            )

            if not engine:
                return {
                    "status": "error",
                    "content": f"Failed to create {engine_name} search engine",
                }

            try:
                # Run the search
                results = engine.run(query)

                # Track sources and accumulate search results
                if isinstance(results, list):
                    for r in results:
                        if isinstance(r, dict):
                            if "link" in r:
                                self._sources.append(r["link"])

                            result_with_index = dict(r)
                            result_with_index["index"] = str(
                                len(self.all_search_results) + 1
                            )
                            result_with_index["source_engine"] = engine_name
                            self.all_search_results.append(result_with_index)

                            if self.all_links_of_system is not None:
                                self.all_links_of_system.append(
                                    result_with_index
                                )

                    # Format results for LLM
                    content = "\n\n".join(
                        f"[{len(self.all_search_results) - len(results) + i + 1}] {r.get('title', 'No title')} ({r.get('link', 'no url')})\n{r.get('snippet', r.get('body', ''))}"
                        for i, r in enumerate(results)
                        if isinstance(r, dict)
                    )
                else:
                    content = str(results)

                self._findings.append(
                    {
                        "type": "specialized_search",
                        "engine": engine_name,
                        "query": query,
                        "result_count": len(results)
                        if isinstance(results, list)
                        else 1,
                    }
                )

                return {"status": "success", "content": content}
            finally:
                from local_deep_research.utilities.resource_utils import (
                    safe_close,
                )

                safe_close(engine, "specialized search engine")

        except Exception as e:
            logger.exception(f"Specialized search with {engine_name} failed")
            return {
                "status": "error",
                "content": f"{engine_name} search failed: {e}",
            }

    def _execute_mcp_tool(
        self,
        server_name: str,
        tool_name: str,
        arguments: Dict[str, Any],
        timeout: Optional[float] = None,
    ) -> Dict[str, Any]:
        """Execute an MCP tool.

        Args:
            server_name: Name of the MCP server
            tool_name: Name of the tool to call
            arguments: Arguments to pass to the tool
            timeout: Timeout in seconds for the tool call
                     (defaults to MCP_TOOL_CALL_TIMEOUT)
        """
        if timeout is None:
            timeout = self.MCP_TOOL_CALL_TIMEOUT
        try:
            from local_deep_research.mcp.client import (
                MCPClientManager,
                run_async,
            )

            # Find the server config
            server_config = None
            for config in self.mcp_servers:
                if config.get("name") == server_name:
                    server_config = config
                    break

            if not server_config:
                return {
                    "status": "error",
                    "content": f"Unknown server: {server_name}",
                }

            async def call():
                manager = MCPClientManager([server_config])
                async with manager.connect_all():
                    return await manager.call_tool(
                        server_name, tool_name, arguments
                    )

            result = run_async(call(), timeout=timeout)

            # Ensure result is a dict
            if not isinstance(result, dict):
                result = {
                    "status": "success",
                    "content": str(result) if result else "",
                }

            # Track findings
            self._findings.append(
                {
                    "type": "mcp_tool",
                    "server": server_name,
                    "tool": tool_name,
                    "arguments": arguments,
                }
            )

            return result

        except Exception as e:
            logger.exception("MCP tool call failed")
            return {"status": "error", "content": f"MCP tool failed: {e}"}

    def _synthesize_answer(self, query: str) -> str:
        """Synthesize an answer from the gathered information when max iterations reached."""
        history = self._format_history()

        prompt = f"""Based on the research conducted, provide a comprehensive answer to the query.

Query: {query}

Research History:
{history}

Please synthesize all the information gathered into a clear, comprehensive answer.
If some information is missing, note what couldn't be found."""

        try:
            response = self.model.invoke(prompt)
            if hasattr(response, "content"):
                return response.content
            return str(response)
        except Exception as e:
            logger.exception("Failed to synthesize answer")
            return f"Research completed but synthesis failed: {e}"

    def _create_error_response(self, error: str) -> Dict:
        """Create an error response."""
        return {
            "findings": [],
            "iterations": 0,
            "questions": {},
            "formatted_findings": f"Error: {error}",
            "current_knowledge": "",
            "error": error,
        }
