"""Researcher agent for executing tool macros and collecting evidence.

The Researcher executes tool macros to gather evidence for research questions.
It produces:
- EvidenceItem list (machine-verifiable)
- Claim list (grounded statements)
- Gap list (explicit unknowns)

Team values enforced:
- V1: Prefer "unknown" over guesses
- V2: All evidence is reproducible (tool calls documented)
- V4: Cost awareness (uses budget limits)

Enhanced with GitHub MCP:
- Code search across GitHub repositories
- Rich PR analysis with diffs and comments
- File content retrieval from GitHub
"""

import json
from typing import Optional

from rich.console import Console

from emdash.agent.toolkit import AgentToolkit
from emdash.agent.providers import get_provider
from emdash.agent.providers.factory import DEFAULT_MODEL
from emdash.core.config import get_config
from emdash.agent.events import AgentEventEmitter, EventType, NullEmitter
from emdash.agent.research.state import (
    EvidenceItem,
    Claim,
    Gap,
    ResearchQuestion,
    ResearchPlan,
)
from emdash.agent.research.macros import (
    MacroExecutor,
    suggest_macros,
    get_macro,
    TOOL_MACROS,
)
from emdash.agent.compaction import LLMCompactor


RESEARCHER_SYSTEM_PROMPT = """You are a research analyst that extracts claims from collected evidence.

CRITICAL RULES:
1. EVERY piece of evidence contains findings - extract them!
2. NEVER say "X results were found but not examined" - READ the evidence details
3. Each claim MUST cite evidence IDs (e.g., "based on E1, E3")
4. Report WHAT was found, not just that something was found

You will receive:
1. A research goal
2. Questions to answer  
3. DETAILED evidence with actual entities, names, and content

YOUR JOB - Extract claims from evidence:
1. READ each evidence item carefully - it contains actual names and details
2. MAKE CLAIMS about what the evidence shows (entities, patterns, relationships)
3. If the exact topic wasn't found, claim what WAS found (it's still valuable)
4. Identify GAPS only for questions with truly no relevant evidence

CLAIM FORMAT:
{
  "id": "C1",
  "statement": "The codebase contains a WebScrapeService class that handles web scraping operations",
  "evidence_ids": ["E1", "E3"],
  "confidence": 2,
  "assumptions": []
}

Example good claims from evidence:
- "The AgentToolkit class in emdash/agent/toolkit.py is the main entry point for LLM tools (E1)"
- "Recent development activity shows @johnsmith worked on auth features in PRs #123, #145 (E2)"
- "The codebase has 5 distinct code communities, largest being mastra-server with 45 modules (E3)"
- "The most central code entity by PageRank is createHttpClient in picasso-http-client (E4)"

Confidence levels:
- 1: Single source, may have assumptions
- 2: Multiple sources corroborate (requires 2+ evidence_ids)
- 3: Strong evidence, no assumptions

GAP FORMAT (only when evidence truly missing):
{
  "question": "What tests cover the toolkit?",
  "reason": "No test files found in any search results",
  "suggested_tools": ["text_search"]
}

OUTPUT FORMAT (JSON only, no markdown):
{
  "claims": [...],
  "gaps": [...],
  "summary": "Brief summary of key findings"
}

IMPORTANT: Evidence contains ACTUAL entity names and details. Extract them into claims.
Do NOT say "10 results found" - say WHAT those results ARE."""


class ResearcherAgent:
    """Executes tool macros, collects evidence, proposes claims.
    
    The Researcher is the "hands" of the research process. It:
    1. Runs appropriate tool macros for each question
    2. Collects evidence with unique IDs
    3. Proposes claims grounded in evidence
    4. Identifies gaps where evidence is missing
    
    Enhanced with GitHub MCP for:
    - Code search across GitHub repositories
    - Rich PR analysis with diffs and comments
    - File content retrieval directly from GitHub
    """
    
    def __init__(
        self,
        model: str = DEFAULT_MODEL,
        verbose: bool = True,
        emitter: Optional[AgentEventEmitter] = None,
    ):
        """Initialize the researcher agent.
        
        Args:
            model: LLM model to use for claim generation
            verbose: Whether to print progress
            emitter: Event emitter for unified message stream
        """
        self.provider = get_provider(model)
        self.model = model
        self.verbose = verbose
        self.console = Console()
        self.toolkit = AgentToolkit(enable_session=True)
        self.macro_executor = MacroExecutor(self.toolkit)
        self.claim_counter = 0
        self.emitter = emitter or NullEmitter(agent_name="ResearcherAgent")
        self.compactor = LLMCompactor(self.provider)
        
        # Check if GitHub MCP is available
        config = get_config()
        self._mcp_available = config.mcp.is_available
        if self._mcp_available and verbose:
            self.console.print("[dim]GitHub MCP tools available for enhanced research[/dim]")
        
        # Get repository context - priority: config > graph
        self._repo_context = self._get_repo_context(config)
        if self._repo_context and verbose:
            owner = self._repo_context.get("owner", "unknown")
            repo = self._repo_context.get("repo", "unknown")
            source = self._repo_context.get("source", "unknown")
            self.console.print(f"[dim]Repository context: {owner}/{repo} (from {source})[/dim]")
    
    def _get_repo_context(self, config=None) -> dict:
        """Get repository context from config or graph database.
        
        Priority:
        1. GITHUB_REPO environment variable (via config.mcp)
        2. Neo4j graph Repository node
        
        Args:
            config: EmDashConfig instance (optional)
            
        Returns:
            Dict with owner, repo, github_url, source
        """
        # Priority 1: Use GITHUB_REPO from config (most reliable)
        if config and config.mcp.repo_owner and config.mcp.repo_name:
            return {
                "owner": config.mcp.repo_owner,
                "repo": config.mcp.repo_name,
                "github_url": f"https://github.com/{config.mcp.repo_owner}/{config.mcp.repo_name}",
                "source": "config",
            }
        
        # Priority 2: Query from graph database
        try:
            from emdash.graph.connection import get_connection
            connection = get_connection()
            
            with connection.session() as session:
                result = session.run("""
                    MATCH (r:Repository)
                    WHERE r.owner IS NOT NULL AND r.name IS NOT NULL
                    RETURN r.owner as owner, r.name as name, 
                           r.url as url, r.default_branch as branch
                    LIMIT 1
                """)
                record = result.single()
                
                if record and record["owner"] and record["name"]:
                    return {
                        "owner": record["owner"],
                        "repo": record["name"],
                        "github_url": f"https://github.com/{record['owner']}/{record['name']}",
                        "default_branch": record.get("branch", "main"),
                        "source": "graph",
                    }
        except Exception as e:
            if self.verbose:
                self.console.print(f"[dim]Could not get repo context from graph: {e}[/dim]")
        
        return {}
    
    def _execute_tool(self, tool_name: str, **kwargs):
        """Execute a tool with event emission.
        
        Args:
            tool_name: Name of the tool to execute
            **kwargs: Tool arguments
            
        Returns:
            Tool execution result
        """
        # Emit tool start
        self.emitter.emit_tool_start(tool_name, kwargs)
        
        # Execute tool
        result = self.toolkit.execute(tool_name, **kwargs)
        
        # Build summary
        summary = None
        if result.success and result.data:
            if "results" in result.data:
                summary = f"{len(result.data['results'])} results"
            elif "prs" in result.data:
                summary = f"{len(result.data['prs'])} PRs"
        elif not result.success:
            summary = result.error
        
        # Emit tool result
        self.emitter.emit_tool_result(
            name=tool_name,
            success=result.success,
            summary=summary,
        )
        
        return result

    def run_macros(
        self,
        questions: list[ResearchQuestion],
        context: dict,
        budget: dict,
    ) -> tuple[list[EvidenceItem], dict]:
        """Execute appropriate macros for questions.
        
        Args:
            questions: Research questions to investigate
            context: Prior context from previous iterations
            budget: Remaining budget {tool_calls, tokens, time_s}
            
        Returns:
            Tuple of (evidence_items, updated_context)
        """
        all_evidence: list[EvidenceItem] = []
        updated_context = dict(context)
        budget_remaining = budget.get("tool_calls", 50)
        
        # Always include repo context - the agent should know which repo it's researching
        if self._repo_context:
            updated_context["repo_owner"] = self._repo_context.get("owner")
            updated_context["repo_name"] = self._repo_context.get("repo")
            updated_context["github_url"] = self._repo_context.get("github_url")
        
        for question in questions:
            if budget_remaining <= 0:
                if self.verbose:
                    self.console.print("[yellow]Budget exhausted, stopping macro execution[/yellow]")
                break
            
            if self.verbose:
                self.console.print(f"[dim]Investigating: {question.question}[/dim]")
            
            # Extract topic from question
            topic = self._extract_topic(question.question)
            
            # First, try direct tool calls to bootstrap context
            # This is more robust than macros when we don't have prior context
            if not updated_context.get("last_search_results"):
                evidence, ctx_updates, budget_remaining = self._bootstrap_search(
                    topic=topic,
                    budget_remaining=budget_remaining,
                )
                all_evidence.extend(evidence)
                updated_context.update(ctx_updates)
                
                if evidence and self.verbose:
                    self.console.print(f"  [green]✓[/green] Bootstrap search: {len(evidence)} evidence items")
            
            # CRITICAL: Drill down into search results to actually read content
            if budget_remaining > 0 and updated_context.get("last_search_results"):
                drill_evidence, ctx_updates, budget_remaining = self._drill_into_results(
                    results=updated_context["last_search_results"],
                    topic=topic,
                    budget_remaining=budget_remaining,
                )
                all_evidence.extend(drill_evidence)
                updated_context.update(ctx_updates)
                
                if drill_evidence and self.verbose:
                    self.console.print(f"  [green]✓[/green] Drill-down: {len(drill_evidence)} evidence items")
            
            # Then try macros if we have context
            if budget_remaining > 0 and updated_context.get("last_search_results"):
                macros_to_run = question.suggested_tools or suggest_macros(
                    question.question, 
                    include_github=self._mcp_available
                )
                
                for macro_name in macros_to_run:
                    if budget_remaining <= 0:
                        break
                    
                    if macro_name not in TOOL_MACROS:
                        if self.verbose:
                            self.console.print(f"    [dim]Skipping unknown tool: {macro_name}[/dim]")
                        continue
                    
                    params = {"topic": topic, "symbol": topic}
                    
                    # Add context-based params from bootstrap
                    if "last_search_top" in updated_context:
                        top_result = updated_context["last_search_top"]
                        params["identifier"] = top_result.get("qualified_name", topic)
                    
                    try:
                        evidence, ctx_updates = self.macro_executor.execute_macro(
                            macro_name=macro_name,
                            params=params,
                            budget_remaining=budget_remaining,
                            prior_context=updated_context,  # Pass context for dependency resolution
                        )
                        
                        all_evidence.extend(evidence)
                        updated_context.update(ctx_updates)
                        budget_remaining -= len(evidence)
                        
                        if self.verbose:
                            self.console.print(f"  [green]✓[/green] {macro_name}: {len(evidence)} evidence items")
                        
                    except Exception as e:
                        if self.verbose:
                            self.console.print(f"  [red]✗[/red] {macro_name}: {e}")
            
            # Fallback: direct exploration if still no evidence
            if not all_evidence and budget_remaining > 0:
                evidence, ctx_updates, budget_remaining = self._fallback_exploration(
                    topic=topic,
                    question=question,
                    budget_remaining=budget_remaining,
                )
                all_evidence.extend(evidence)
                updated_context.update(ctx_updates)
                
                if evidence and self.verbose:
                    self.console.print(f"  [green]✓[/green] Fallback exploration: {len(evidence)} evidence items")
        
        if self.verbose and not all_evidence:
            self.console.print("[yellow]⚠ No evidence collected. Check if the topic exists in the graph.[/yellow]")
        
        return all_evidence, updated_context
    
    def _bootstrap_search(
        self,
        topic: str,
        budget_remaining: int,
    ) -> tuple[list[EvidenceItem], dict, int]:
        """Bootstrap context with direct search calls.
        
        Tries multiple search strategies to find relevant code.
        
        Args:
            topic: The topic to search for
            budget_remaining: Remaining tool call budget
            
        Returns:
            Tuple of (evidence, context_updates, remaining_budget)
        """
        evidence: list[EvidenceItem] = []
        context: dict = {}
        
        # Generate search variations
        search_variations = self._generate_search_variations(topic)
        
        # Strategy 1: Try semantic search with variations (use lower min_score)
        for query in search_variations[:5]:  # Try up to 5 variations
            if budget_remaining <= 0 or context.get("last_search_results"):
                break
            try:
                # Use lower min_score (0.3) to catch more potential matches
                result = self.toolkit.execute(
                    "semantic_search", 
                    query=query, 
                    limit=15,
                    min_score=0.3,  # Lower threshold for better recall
                )
                budget_remaining -= 1
                
                if result.success and result.data.get("results"):
                    ev = self._create_evidence(
                        tool="semantic_search",
                        input_args={"query": query, "limit": 15, "min_score": 0.3},
                        result=result,
                        description=f"Semantic search for '{query}'",
                    )
                    evidence.append(ev)
                    
                    # Update context
                    context["last_search_results"] = result.data["results"]
                    context["last_search_top"] = result.data["results"][0]
                    context["successful_query"] = query
                    
                    if self.verbose:
                        count = len(result.data["results"])
                        self.console.print(f"    [dim]semantic_search '{query}': {count} results[/dim]")
                    break
                elif self.verbose:
                    self.console.print(f"    [dim]semantic_search '{query}': no results[/dim]")
                    
            except Exception as e:
                if self.verbose:
                    self.console.print(f"    [dim]semantic_search '{query}' failed: {e}[/dim]")
        
        # Strategy 2: Try text search with MORE variations (if semantic failed)
        if not context.get("last_search_results"):
            # Text search is exact match, so try all variations
            for query in search_variations[:10]:  # Try more variations for text search
                if budget_remaining <= 0 or context.get("last_search_results"):
                    break
                try:
                    result = self.toolkit.execute("text_search", query=query, limit=15)
                    budget_remaining -= 1
                    
                    if result.success and result.data.get("results"):
                        ev = self._create_evidence(
                            tool="text_search",
                            input_args={"query": query, "limit": 15},
                            result=result,
                            description=f"Text search for '{query}'",
                        )
                        evidence.append(ev)
                        
                        context["last_search_results"] = result.data["results"]
                        context["last_search_top"] = result.data["results"][0]
                        context["successful_query"] = query
                        
                        if self.verbose:
                            count = len(result.data["results"])
                            self.console.print(f"    [dim]text_search '{query}': {count} results[/dim]")
                        break
                    elif self.verbose:
                        self.console.print(f"    [dim]text_search '{query}': no results[/dim]")
                        
                except Exception as e:
                    if self.verbose:
                        self.console.print(f"    [dim]text_search '{query}' failed: {e}[/dim]")
        
        # Strategy 3: Try individual keywords
        if budget_remaining > 0 and not context.get("last_search_results"):
            keywords = self._extract_keywords(topic)
            for word in keywords[:5]:  # Try up to 5 keywords
                if budget_remaining <= 0:
                    break
                if len(word) < 3:
                    continue
                try:
                    result = self.toolkit.execute("text_search", query=word, limit=5)
                    budget_remaining -= 1
                    
                    if result.success and result.data.get("results"):
                        ev = self._create_evidence(
                            tool="text_search",
                            input_args={"query": word, "limit": 5},
                            result=result,
                            description=f"Text search for keyword '{word}'",
                        )
                        evidence.append(ev)
                        
                        context["last_search_results"] = result.data["results"]
                        context["last_search_top"] = result.data["results"][0]
                        context["successful_query"] = word
                        
                        if self.verbose:
                            count = len(result.data["results"])
                            self.console.print(f"    [dim]text_search '{word}': {count} results[/dim]")
                        break
                        
                except Exception:
                    pass
        
        return evidence, context, budget_remaining
    
    def _generate_search_variations(self, topic: str) -> list[str]:
        """Generate search query variations for a topic.
        
        Creates multiple variations to increase chances of finding relevant code.
        Handles various naming conventions: CamelCase, snake_case, kebab-case.
        
        Args:
            topic: The original topic
            
        Returns:
            List of search query variations
        """
        variations = []
        
        # Clean the topic first
        cleaned = topic.lower().strip()
        for word in ["the", "a", "an", "is", "are", "was", "how", "does", "what", "work", "works"]:
            cleaned = cleaned.replace(f" {word} ", " ")
            if cleaned.startswith(f"{word} "):
                cleaned = cleaned[len(word)+1:]
            if cleaned.endswith(f" {word}"):
                cleaned = cleaned[:-len(word)-1]
        cleaned = " ".join(cleaned.split())  # Normalize whitespace
        
        # Get words from cleaned topic
        words = cleaned.split()
        
        # 1. Original and cleaned
        variations.append(topic)
        if cleaned != topic.lower():
            variations.append(cleaned)
        
        # 2. CamelCase: "home page improver" -> "HomePageImprover"
        camel = "".join(word.capitalize() for word in words)
        variations.append(camel)
        
        # 3. lowerCamelCase: "home page improver" -> "homePageImprover"  
        if len(words) > 1:
            lower_camel = words[0].lower() + "".join(w.capitalize() for w in words[1:])
            variations.append(lower_camel)
        
        # 4. snake_case: "home page improver" -> "home_page_improver"
        snake = "_".join(words).lower()
        variations.append(snake)
        
        # 5. kebab-case: "home page improver" -> "home-page-improver"
        kebab = "-".join(words).lower()
        variations.append(kebab)
        
        # 6. No separators: "homepageimprover"
        nospace = "".join(words).lower()
        variations.append(nospace)
        
        # 7. Partial matches - last two words (often the core concept)
        if len(words) >= 2:
            last_two = "".join(w.capitalize() for w in words[-2:])
            variations.append(last_two)
            last_two_lower = "".join(words[-2:]).lower()
            variations.append(last_two_lower)
        
        # 8. Just the last word (the noun/object)
        if len(words) >= 1:
            variations.append(words[-1])
            variations.append(words[-1].capitalize())
        
        # 9. Common function prefixes: get*, handle*, create*, etc.
        if len(words) >= 1:
            core = "".join(w.capitalize() for w in words)
            for prefix in ["get", "handle", "create", "update", "delete", "fetch", "process"]:
                variations.append(f"{prefix}{core}")
        
        # 10. Common suffixes: *Prompt, *Handler, *Service, etc.
        if len(words) >= 1:
            core = "".join(w.capitalize() for w in words)
            for suffix in ["Prompt", "Handler", "Service", "Component", "Agent"]:
                variations.append(f"{core}{suffix}")
        
        # Deduplicate while preserving order
        seen = set()
        unique = []
        for v in variations:
            v_lower = v.lower()
            if v_lower not in seen:
                seen.add(v_lower)
                unique.append(v)
        
        return unique
    
    def _drill_into_results(
        self,
        results: list[dict],
        topic: str,
        budget_remaining: int,
    ) -> tuple[list[EvidenceItem], dict, int]:
        """Drill into search results to read actual content.
        
        This is CRITICAL - we don't just note "10 results found",
        we actually expand and read the top results.
        
        Args:
            results: Search results to drill into
            topic: Original topic (for context)
            budget_remaining: Remaining tool call budget
            
        Returns:
            Tuple of (evidence, context_updates, remaining_budget)
        """
        evidence: list[EvidenceItem] = []
        context: dict = {}
        
        # Drill into top 3 results
        for i, result in enumerate(results[:3]):
            if budget_remaining <= 0:
                break
            
            qualified_name = result.get("qualified_name")
            node_type_raw = result.get("node_type") or result.get("type") or "unknown"
            file_path = result.get("file_path")
            
            # Normalize node_type to capitalized form (Function, Class, File)
            node_type = node_type_raw.capitalize() if node_type_raw else "Function"
            if node_type.lower() in ["method"]:
                node_type = "Function"  # Methods are functions in Neo4j
            
            if not qualified_name and not file_path:
                continue
            
            # Expand the node to get full details
            try:
                expand_result = self.toolkit.execute(
                    "expand_node",
                    node_type=node_type,
                    identifier=qualified_name or file_path,
                )
                budget_remaining -= 1
                
                if expand_result.success:
                    ev = self._create_evidence(
                        tool="expand_node",
                        input_args={"node_type": node_type, "identifier": qualified_name or file_path},
                        result=expand_result,
                        description=f"Expanded search result #{i+1}: {qualified_name or file_path}",
                    )
                    evidence.append(ev)
                    
                    # Store for further reference
                    if i == 0:
                        context["last_expanded_node"] = expand_result.data.get("root_node", {})
                    
                    if self.verbose:
                        self.console.print(f"    [dim]Expanded: {qualified_name or file_path}[/dim]")
                        
            except Exception as e:
                if self.verbose:
                    self.console.print(f"    [dim]Failed to expand {qualified_name}: {e}[/dim]")
            
            # Also get callers/callees for functions (only if we have a valid qualified_name)
            if budget_remaining > 0 and qualified_name and node_type_raw.lower() in ["function", "method", "class"]:
                try:
                    callers_result = self.toolkit.execute(
                        "get_callers",
                        qualified_name=qualified_name,
                    )
                    budget_remaining -= 1
                    
                    if callers_result.success:
                        callers = callers_result.data.get("callers", [])
                        if callers:
                            ev = self._create_evidence(
                                tool="get_callers",
                                input_args={"qualified_name": qualified_name},
                                result=callers_result,
                                description=f"Who calls {qualified_name}?",
                            )
                            evidence.append(ev)
                            
                except Exception as e:
                    if self.verbose:
                        self.console.print(f"    [dim]get_callers failed: {e}[/dim]")
        
        return evidence, context, budget_remaining
    
    def _fallback_exploration(
        self,
        topic: str,
        question: ResearchQuestion,
        budget_remaining: int,
    ) -> tuple[list[EvidenceItem], dict, int]:
        """Fallback exploration when macros fail.
        
        Tries high-level exploration tools to gather context,
        then drills into the results for actual content.
        Now includes GitHub MCP tools for enhanced research.
        
        Args:
            topic: The topic to explore
            question: The research question
            budget_remaining: Remaining tool call budget
            
        Returns:
            Tuple of (evidence, context_updates, remaining_budget)
        """
        evidence: list[EvidenceItem] = []
        context: dict = {}
        
        # Phase 1: High-level exploration tools (local graph)
        fallback_tools = [
            ("get_top_pagerank", {"limit": 15}, "Most important code entities by centrality"),
            ("get_recent_prs", {"limit": 15}, "Recent development activity"),
            ("get_communities", {"limit": 8, "include_members": True}, "Code module clusters"),
            ("get_top_commit_importance", {"limit": 15}, "Most actively developed files"),
        ]
        
        top_entities = []
        
        for tool_name, args, description in fallback_tools:
            if budget_remaining <= 0:
                break
                
            try:
                result = self.toolkit.execute(tool_name, **args)
                budget_remaining -= 1
                
                if result.success:
                    ev = self._create_evidence(
                        tool=tool_name,
                        input_args=args,
                        result=result,
                        description=description,
                    )
                    evidence.append(ev)
                    
                    # Collect entities for drill-down
                    if "results" in result.data:
                        context[f"{tool_name}_results"] = result.data["results"][:5]
                        for r in result.data["results"][:5]:
                            if r.get("qualified_name"):
                                top_entities.append(r)
                    
                    if self.verbose:
                        if "results" in result.data:
                            self.console.print(f"    [dim]{tool_name}: {len(result.data['results'])} results[/dim]")
                        else:
                            self.console.print(f"    [dim]{tool_name}: success[/dim]")
                            
            except Exception as e:
                if self.verbose:
                    self.console.print(f"    [dim]{tool_name} failed: {e}[/dim]")
        
        # Phase 2: Drill into top entities from PageRank
        if budget_remaining > 0 and top_entities:
            drill_evidence, ctx_updates, budget_remaining = self._drill_into_results(
                results=top_entities[:3],
                topic=topic,
                budget_remaining=budget_remaining,
            )
            evidence.extend(drill_evidence)
            context.update(ctx_updates)
            
            if drill_evidence and self.verbose:
                self.console.print(f"    [dim]Drill-down: {len(drill_evidence)} evidence items[/dim]")
        
        # Phase 3: GitHub MCP code search (if available and topic specific)
        if budget_remaining > 0 and self._mcp_available:
            github_evidence, github_context, budget_remaining = self._github_code_search(
                topic=topic,
                budget_remaining=budget_remaining,
            )
            evidence.extend(github_evidence)
            context.update(github_context)
        
        return evidence, context, budget_remaining
    
    def _github_code_search(
        self,
        topic: str,
        budget_remaining: int,
    ) -> tuple[list[EvidenceItem], dict, int]:
        """Search GitHub for code patterns using MCP.
        
        This provides additional context from GitHub when:
        - Local graph search didn't find relevant results
        - Topic seems like a code pattern or implementation
        
        Args:
            topic: The topic to search for
            budget_remaining: Remaining tool call budget
            
        Returns:
            Tuple of (evidence, context_updates, remaining_budget)
        """
        evidence: list[EvidenceItem] = []
        context: dict = {}
        
        if budget_remaining <= 0:
            return evidence, context, budget_remaining
        
        # Try GitHub code search
        try:
            result = self.toolkit.execute(
                "github_search_code",
                query=topic,
                per_page=5,
            )
            budget_remaining -= 1
            
            if result.success and result.data.get("results"):
                ev = self._create_evidence(
                    tool="github_search_code",
                    input_args={"query": topic, "per_page": 5},
                    result=result,
                    description=f"GitHub code search for '{topic}'",
                )
                evidence.append(ev)
                
                context["github_search_results"] = result.data["results"]
                
                if self.verbose:
                    count = len(result.data["results"])
                    self.console.print(f"    [dim]github_search_code: {count} results[/dim]")
                    
        except Exception as e:
            if self.verbose:
                self.console.print(f"    [dim]github_search_code failed: {e}[/dim]")
        
        return evidence, context, budget_remaining
    
    def _create_evidence(
        self,
        tool: str,
        input_args: dict,
        result,
        description: str,
    ) -> EvidenceItem:
        """Create an evidence item from a tool result.
        
        Args:
            tool: Tool name
            input_args: Arguments passed to the tool
            result: Tool execution result
            description: Step description
            
        Returns:
            EvidenceItem with unique ID
        """
        self.macro_executor.evidence_counter += 1
        evidence_id = f"E{self.macro_executor.evidence_counter}"
        
        # Extract entities from result
        entities = self._extract_entities_from_result(result)
        
        # Create DETAILED summary that includes actual findings
        if result.success:
            summary = self._create_detailed_summary(tool, result.data, description, entities)
        else:
            summary = f"Error: {result.error}"
        
        return EvidenceItem(
            id=evidence_id,
            tool=tool,
            input=input_args,
            output_ref=f"result_{evidence_id}",
            summary=summary,
            entities=entities,
        )
    
    def _create_detailed_summary(
        self,
        tool: str,
        data: dict,
        description: str,
        entities: list[str],
    ) -> str:
        """Create a detailed summary that includes actual content.
        
        This is crucial - we don't just say "10 results found",
        we say WHAT was found.
        
        Args:
            tool: Tool name
            data: Result data
            description: Step description
            entities: Extracted entities
            
        Returns:
            Detailed summary string
        """
        lines = [description]
        
        # Add entity details
        if entities:
            lines.append(f"Found: {', '.join(entities[:8])}")
            if len(entities) > 8:
                lines.append(f"  (+{len(entities) - 8} more)")
        
        # Tool-specific details
        if tool in ["semantic_search", "text_search"]:
            results = data.get("results", [])
            if results:
                lines.append(f"Top results ({len(results)} total):")
                for r in results[:5]:
                    name = r.get("qualified_name") or r.get("identifier") or r.get("file_path", "?")
                    node_type = r.get("node_type", "")
                    score = r.get("score", r.get("relevance", ""))
                    score_str = f" (score: {score:.2f})" if isinstance(score, float) else ""
                    lines.append(f"  - {name} [{node_type}]{score_str}")
        
        elif tool == "expand_node":
            root = data.get("root_node", {})
            summary_data = data.get("summary", {})
            if root:
                name = root.get("qualified_name") or root.get("file_path", "?")
                node_type = root.get("node_type", "")
                docstring = root.get("docstring", "")
                lines.append(f"Expanded: {name} [{node_type}]")
                if docstring:
                    lines.append(f"  Doc: {docstring[:150]}...")
            if summary_data:
                funcs = summary_data.get("function_count", 0)
                classes = summary_data.get("class_count", 0)
                if funcs or classes:
                    lines.append(f"  Contains: {funcs} functions, {classes} classes")
        
        elif tool == "get_callers":
            callers = data.get("callers", [])
            count = data.get("count", len(callers))
            if callers:
                lines.append(f"Found {count} callers:")
                for c in callers[:5]:
                    name = c.get("qualified_name") or c.get("file_path", "?")
                    lines.append(f"  - {name}")
        
        elif tool == "get_callees":
            callees = data.get("callees", [])
            count = data.get("count", len(callees))
            if callees:
                lines.append(f"Found {count} callees:")
                for c in callees[:5]:
                    name = c.get("qualified_name") or c.get("file_path", "?")
                    lines.append(f"  - {name}")
        
        elif tool == "get_communities":
            communities = data.get("communities", [])
            if communities:
                lines.append(f"Found {len(communities)} code communities:")
                for i, comm in enumerate(communities[:3]):
                    members = comm.get("sample_members", comm.get("members", []))[:3]
                    size = comm.get("size", len(members))
                    lines.append(f"  Community {i+1} (size {size}): {', '.join(members)}")
        
        elif tool == "get_top_pagerank":
            results = data.get("results", [])
            if results:
                lines.append(f"Top {len(results)} entities by PageRank:")
                for r in results[:7]:
                    name = r.get("qualified_name") or r.get("identifier", "?")
                    score = r.get("pagerank", r.get("score", 0))
                    lines.append(f"  - {name} (rank: {score:.4f})" if isinstance(score, float) else f"  - {name}")
        
        elif tool == "get_recent_prs":
            results = data.get("results", [])
            if results:
                lines.append(f"Recent PRs ({len(results)} found):")
                for pr in results[:5]:
                    pr_num = pr.get("pr_number", "?")
                    title = pr.get("title", "")[:50]
                    author = pr.get("author", "?")
                    lines.append(f"  - PR #{pr_num}: {title} (by {author})")
        
        elif tool == "get_file_history":
            results = data.get("results", data.get("commits", []))
            if results:
                lines.append(f"File history ({len(results)} commits):")
                for c in results[:5]:
                    author = c.get("author", "?")
                    msg = c.get("message", "")[:40]
                    lines.append(f"  - {author}: {msg}")
        
        elif tool == "get_top_commit_importance":
            results = data.get("results", [])
            if results:
                lines.append(f"Top files by commit importance:")
                for r in results[:5]:
                    path = r.get("file_path", r.get("path", "?"))
                    score = r.get("importance", r.get("score", 0))
                    lines.append(f"  - {path}")
        
        elif tool == "get_impact_analysis":
            impact = data.get("impact", data)
            if impact:
                affected = impact.get("affected_files", [])
                risk = impact.get("risk_level", "unknown")
                lines.append(f"Impact analysis: {len(affected)} affected files, risk: {risk}")
                if affected[:3]:
                    lines.append(f"  Affected: {', '.join(affected[:3])}")
        
        elif tool == "detect_knowledge_silos":
            silos = data.get("silos", [])
            if silos:
                lines.append(f"Knowledge silos detected ({len(silos)}):")
                for s in silos[:3]:
                    owner = s.get("owner", "?")
                    files = s.get("files", [])
                    lines.append(f"  - {owner}: owns {len(files)} files exclusively")
        
        return "\n".join(lines)
    
    def _extract_entities_from_result(self, result) -> list[str]:
        """Extract entity identifiers from a tool result."""
        entities = []
        
        if not result.success:
            return entities
        
        data = result.data
        
        # Extract from results list
        if "results" in data:
            for item in data["results"][:10]:
                if isinstance(item, dict):
                    for key in ["qualified_name", "file_path", "identifier", "pr_number", "name"]:
                        if key in item and item[key]:
                            entities.append(str(item[key]))
                            break
        
        # Extract from root_node
        if "root_node" in data:
            root = data["root_node"]
            for key in ["qualified_name", "file_path"]:
                if key in root and root[key]:
                    entities.append(str(root[key]))
        
        # Extract from files
        if "files" in data:
            for f in data["files"][:10]:
                if isinstance(f, dict) and "path" in f:
                    entities.append(f["path"])
                elif isinstance(f, str):
                    entities.append(f)
        
        # Extract from communities
        if "communities" in data:
            for comm in data["communities"][:5]:
                if isinstance(comm, dict) and "sample_members" in comm:
                    entities.extend(comm["sample_members"][:3])
        
        return list(set(entities))[:20]  # Dedupe and limit
    
    def _summarize_tool_result(self, tool: str, data: dict, description: str) -> str:
        """Create a human-readable summary of a tool result."""
        count = None
        
        if "results" in data:
            count = len(data["results"])
        elif "callers" in data:
            count = data.get("count", len(data["callers"]))
        elif "callees" in data:
            count = data.get("count", len(data["callees"]))
        elif "communities" in data:
            count = len(data["communities"])
        elif "files" in data:
            count = len(data["files"])
        
        if count is not None:
            return f"{description}: Found {count} results"
        elif "summary" in data:
            s = data["summary"]
            return f"{description}: {s.get('function_count', 0)} functions, {s.get('class_count', 0)} classes"
        else:
            return description
    
    def propose_claims(
        self,
        goal: str,
        questions: list[ResearchQuestion],
        evidence: list[EvidenceItem],
        prior_claims: list[Claim] = None,
    ) -> list[Claim]:
        """Generate claims grounded in evidence.
        
        Uses LLM to analyze evidence and propose claims.
        
        Args:
            goal: Research goal
            questions: Questions being answered
            evidence: Evidence collected
            prior_claims: Claims from previous iterations
            
        Returns:
            List of new Claims
        """
        if not evidence:
            return []
        
        # Build detailed evidence summary for LLM
        evidence_summary = self._format_evidence_for_llm(evidence)
        questions_text = "\n".join([f"- [{q.priority}] {q.question}" for q in questions])
        
        prior_claims_text = ""
        if prior_claims:
            prior_claims_text = "\n\nPRIOR CLAIMS (don't repeat):\n" + "\n".join([
                f"- {c.statement}" for c in prior_claims
            ])
        
        # Build a list of entities found for context
        all_entities = []
        for e in evidence:
            all_entities.extend(e.entities)
        entities_text = ""
        if all_entities:
            unique_entities = list(set(all_entities))[:20]
            entities_text = f"\n\nENTITIES DISCOVERED:\n" + "\n".join([f"- {e}" for e in unique_entities])
        
        payload = {
            "questions": questions_text,
            "evidence": evidence_summary,
            "entities": entities_text,
            "prior_claims": prior_claims_text,
        }
        compacted = self.compactor.compact_payload(payload, goal)
        questions_text = compacted.get("questions", questions_text)
        evidence_summary = compacted.get("evidence", evidence_summary)
        entities_text = compacted.get("entities", entities_text)
        prior_claims_text = compacted.get("prior_claims", prior_claims_text)

        user_message = f"""Extract claims from the evidence below.

RESEARCH GOAL: {goal}

QUESTIONS:
{questions_text}

EVIDENCE (read carefully - contains actual names and details):
{evidence_summary}
{entities_text}
{prior_claims_text}

INSTRUCTIONS:
1. READ each evidence item - it lists actual entity names, file paths, PR numbers, authors
2. CREATE A CLAIM for each significant finding:
   - "X class/function does Y" (from expand_node evidence)
   - "Top code entities are: A, B, C" (from PageRank evidence)
   - "Recent PRs include #123 by author (topic)" (from PR evidence)
   - "Main code communities are: X, Y, Z" (from communities evidence)
3. If "{goal}" wasn't found exactly, claim what WAS found - still valuable
4. Reference evidence IDs in every claim (e.g., [E1, E2])
5. Only create Gaps for questions with ZERO relevant evidence

Example output:
{{
  "claims": [
    {{"id": "C1", "statement": "The codebase's most central component is createHttpClient in picasso-http-client, indicating heavy external API usage", "evidence_ids": ["E4"], "confidence": 2, "assumptions": []}},
    {{"id": "C2", "statement": "Recent development focused on streaming coder agent (PR #89) and designer integration (PR #91)", "evidence_ids": ["E2"], "confidence": 2, "assumptions": []}}
  ],
  "gaps": [],
  "summary": "Identified core components and recent development patterns"
}}

Return JSON only."""

        messages = [
            {"role": "user", "content": user_message},
        ]
        
        response = self.provider.chat(messages, system=RESEARCHER_SYSTEM_PROMPT)
        content = response.content or ""
        
        claims = self._parse_claims(content, evidence)
        
        # If LLM didn't generate claims, try to auto-generate some basic ones
        if not claims and evidence:
            claims = self._auto_generate_claims(evidence)
        
        return claims
    
    def _auto_generate_claims(self, evidence: list[EvidenceItem]) -> list[Claim]:
        """Auto-generate basic claims when LLM fails to produce any.
        
        Creates simple claims based on evidence summaries.
        
        Args:
            evidence: Evidence items
            
        Returns:
            List of basic claims
        """
        claims = []
        
        for e in evidence:
            self.claim_counter += 1
            
            # Generate a claim from the evidence summary
            if e.entities:
                statement = f"The codebase contains: {', '.join(e.entities[:5])}"
                if len(e.entities) > 5:
                    statement += f" (and {len(e.entities) - 5} more)"
            else:
                statement = e.summary
            
            try:
                claim = Claim(
                    id=f"C{self.claim_counter}",
                    statement=statement,
                    evidence_ids=[e.id],
                    confidence=1,
                    assumptions=["Auto-generated from evidence"],
                )
                claims.append(claim)
            except ValueError:
                pass
        
        return claims[:10]  # Limit auto-generated claims
    
    def identify_gaps(
        self,
        plan: ResearchPlan,
        claims: list[Claim],
        evidence: list[EvidenceItem],
    ) -> list[Gap]:
        """Identify questions that couldn't be answered.
        
        Args:
            plan: Research plan with questions
            claims: Claims proposed
            evidence: Evidence collected
            
        Returns:
            List of Gaps
        """
        gaps: list[Gap] = []
        
        # Check each P0/P1 question
        for question in plan.questions:
            if question.priority == "P2":
                continue
            
            # Check if any claim addresses this question
            question_addressed = False
            question_lower = question.question.lower()
            
            for claim in claims:
                # Simple heuristic: check if claim relates to question
                claim_lower = claim.statement.lower()
                keywords = self._extract_keywords(question_lower)
                
                if any(kw in claim_lower for kw in keywords):
                    question_addressed = True
                    break
            
            if not question_addressed:
                # Check if we have any relevant evidence
                relevant_evidence = self._find_relevant_evidence(question, evidence)
                
                if not relevant_evidence:
                    gaps.append(Gap(
                        question=question.question,
                        reason="No relevant evidence found",
                        suggested_tools=question.suggested_tools or ["semantic_search"],
                    ))
                else:
                    gaps.append(Gap(
                        question=question.question,
                        reason="Evidence exists but no clear answer emerged",
                        suggested_tools=["expand_node", "get_file_history"],
                    ))
        
        return gaps
    
    def _format_evidence_for_llm(self, evidence: list[EvidenceItem]) -> str:
        """Format evidence for LLM consumption.
        
        Args:
            evidence: Evidence items
            
        Returns:
            Formatted string
        """
        lines = []
        for e in evidence:
            lines.append(f"[{e.id}] Tool: {e.tool}")
            lines.append(f"    Query/Input: {e.input}")
            lines.append(f"    Result: {e.summary}")
            if e.entities:
                entities_str = ", ".join(e.entities[:10])
                if len(e.entities) > 10:
                    entities_str += f" ... (+{len(e.entities) - 10} more)"
                lines.append(f"    Found entities: {entities_str}")
            lines.append("")  # Blank line between evidence items
        
        return "\n".join(lines)
    
    def _parse_claims(
        self,
        content: str,
        evidence: list[EvidenceItem],
    ) -> list[Claim]:
        """Parse claims from LLM response.
        
        Args:
            content: LLM response content
            evidence: Available evidence (for validation)
            
        Returns:
            List of valid Claims
        """
        claims: list[Claim] = []
        evidence_ids = {e.id for e in evidence}
        
        try:
            # Try to extract JSON
            json_str = content
            if "```" in content:
                start = content.find("```")
                end = content.find("```", start + 3)
                if end > start:
                    json_str = content[start + 3:end]
                    if json_str.startswith("json"):
                        json_str = json_str[4:]
            
            data = json.loads(json_str.strip())
            
            for c in data.get("claims", []):
                # Validate evidence references
                claim_evidence = c.get("evidence_ids", [])
                valid_evidence = [eid for eid in claim_evidence if eid in evidence_ids]
                
                if not valid_evidence:
                    # Skip claims without valid evidence (V1: Truth over fluency)
                    continue
                
                self.claim_counter += 1
                claim_id = c.get("id", f"C{self.claim_counter}")
                
                # Ensure confidence rules are followed
                confidence = c.get("confidence", 1)
                assumptions = c.get("assumptions", [])
                
                # Adjust confidence based on evidence count
                if confidence >= 2 and len(valid_evidence) < 2:
                    confidence = 1
                
                # Cap confidence if assumptions exist
                if assumptions and confidence > 1:
                    confidence = 1
                
                try:
                    claim = Claim(
                        id=claim_id,
                        statement=c["statement"],
                        evidence_ids=valid_evidence,
                        confidence=confidence,
                        assumptions=assumptions,
                        counterevidence_ids=c.get("counterevidence_ids", []),
                    )
                    claims.append(claim)
                except ValueError as e:
                    # Claim validation failed
                    if self.verbose:
                        self.console.print(f"[yellow]Skipped invalid claim: {e}[/yellow]")
            
        except (json.JSONDecodeError, KeyError, TypeError) as e:
            if self.verbose:
                self.console.print(f"[yellow]Failed to parse claims: {e}[/yellow]")
        
        return claims
    
    def _extract_topic(self, question: str) -> str:
        """Extract the main topic from a question.
        
        Args:
            question: The question text
            
        Returns:
            Extracted topic
        """
        # Remove common question patterns
        topic = question.lower()
        patterns = [
            "what is the feature/behavior of ",
            "where is ",
            "what depends on ",
            "who owns or touches ",
            "what's risky about ",
            "what tests/ci validate ",
            "what should a reviewer check for ",
            "how does ",
            "what is ",
        ]
        
        for pattern in patterns:
            if topic.startswith(pattern):
                topic = topic[len(pattern):]
                break
        
        # Clean up
        topic = topic.rstrip("?!.")
        
        return topic.strip() or question
    
    def _extract_keywords(self, text: str) -> list[str]:
        """Extract keywords from text for matching.
        
        Args:
            text: Input text
            
        Returns:
            List of keywords
        """
        # Simple keyword extraction
        stopwords = {
            "the", "a", "an", "is", "are", "was", "were", "be", "been",
            "being", "have", "has", "had", "do", "does", "did", "will",
            "would", "could", "should", "may", "might", "must", "shall",
            "can", "need", "dare", "ought", "used", "to", "of", "in",
            "for", "on", "with", "at", "by", "from", "as", "into",
            "through", "during", "before", "after", "above", "below",
            "between", "under", "again", "further", "then", "once",
            "what", "where", "who", "how", "why", "when", "which",
        }
        
        words = text.lower().split()
        keywords = [w.strip("?!.,;:'\"") for w in words if w not in stopwords and len(w) > 2]
        
        return keywords
    
    def _find_relevant_evidence(
        self,
        question: ResearchQuestion,
        evidence: list[EvidenceItem],
    ) -> list[EvidenceItem]:
        """Find evidence relevant to a question.
        
        Args:
            question: The research question
            evidence: Available evidence
            
        Returns:
            List of relevant evidence items
        """
        keywords = self._extract_keywords(question.question)
        relevant = []
        
        for e in evidence:
            # Check if evidence relates to question
            e_text = f"{e.summary} {' '.join(e.entities)}".lower()
            
            if any(kw in e_text for kw in keywords):
                relevant.append(e)
        
        return relevant
