"""Step executor using sub-agents for isolated context.

Each plan step is executed in a fresh Claude conversation (sub-agent),
which:
1. Preserves context in the main agent
2. Allows each step to have focused context
3. Prevents context pollution between steps
4. Enables parallel step execution when dependencies allow
"""

from __future__ import annotations

import asyncio
from dataclasses import dataclass, field
from typing import Any

from promethian.config import PromethianConfig
from promethian.core.types import (
    Plan, PlanStep, Skill, Tool,
    StepResult, ExecutionStatus,
)
from promethian.core.tool_runner import ToolRunner, ToolExecutor, ToolExecutionError
from promethian.core.progress import ProgressTracker, NullProgressTracker, ProgressStage
from promethian.utils.claude import ClaudeClient, UsageStats
from promethian.utils.logging import get_logger


class ToolExecutionFailure(Exception):
    """Raised when a custom tool fails and should trigger healing."""
    def __init__(self, message: str, tool_name: str, error_details: str):
        super().__init__(message)
        self.tool_name = tool_name
        self.error_details = error_details

# Import built-in tools
from promethian.tools.bash import get_bash, BashTool
from promethian.tools.web_search import get_web_search, WebSearchTool
from promethian.tools.web_fetch import get_web_fetch, WebFetchTool
from promethian.tools.file_tools import get_file_tools, FileTools
from promethian.tools.todo import get_todo_tool, TodoWriteTool
from promethian.tools.ask_user import get_ask_user_tool, AskUserTool

try:
    from promethian.tools.browser import get_browser, BrowserTool
    # Also check if playwright is actually installed (not just our wrapper)
    from playwright.async_api import async_playwright  # noqa: F401
    BROWSER_AVAILABLE = True
except ImportError:
    BROWSER_AVAILABLE = False


def _estimate_tokens(text: str) -> int:
    """Rough estimate of token count (4 chars per token on average)."""
    return len(text) // 4


def _truncate_output(output: Any, max_chars: int = 8000) -> Any:
    """
    Truncate output to prevent context overflow.

    Handles strings, dicts with tool_results, and nested structures.
    Returns a truncated version that preserves structure but limits size.
    """
    import json

    if output is None:
        return None

    if isinstance(output, str):
        if len(output) > max_chars:
            return output[:max_chars] + "\n... (truncated)"
        return output

    if isinstance(output, dict):
        # Handle tool_results specially - these are often the culprit
        if "tool_results" in output:
            truncated = dict(output)
            tool_results = output["tool_results"]

            if isinstance(tool_results, list):
                # Keep only essential info from each result
                truncated_results = []
                total_chars = 0
                max_per_result = max_chars // max(len(tool_results), 1)

                for result in tool_results:
                    if isinstance(result, dict):
                        # Extract key info
                        summary = {
                            "success": result.get("success", True),
                        }

                        # Include error if present
                        if "error" in result:
                            summary["error"] = str(result["error"])[:500]

                        # Include truncated result
                        if "result" in result:
                            result_str = str(result["result"])
                            if len(result_str) > max_per_result:
                                summary["result"] = result_str[:max_per_result] + "...(truncated)"
                            else:
                                summary["result"] = result["result"]

                        result_json = json.dumps(summary, default=str)
                        total_chars += len(result_json)

                        if total_chars < max_chars:
                            truncated_results.append(summary)
                        else:
                            truncated_results.append({"truncated": True, "remaining": len(tool_results) - len(truncated_results)})
                            break
                    else:
                        truncated_results.append(result)

                truncated["tool_results"] = truncated_results

            # Also truncate text if present
            if "text" in truncated and isinstance(truncated["text"], str):
                if len(truncated["text"]) > max_chars // 2:
                    truncated["text"] = truncated["text"][:max_chars // 2] + "\n... (truncated)"

            return truncated

        # For other dicts, convert to string and truncate
        try:
            output_str = json.dumps(output, default=str)
            if len(output_str) > max_chars:
                return {"summary": output_str[:max_chars] + "...(truncated)"}
            return output
        except Exception:
            return {"summary": str(output)[:max_chars]}

    # For lists or other types, convert to string
    output_str = str(output)
    if len(output_str) > max_chars:
        return output_str[:max_chars] + "...(truncated)"
    return output


@dataclass
class SubAgentResult:
    """Result from a sub-agent execution."""
    success: bool
    output: Any = None
    error: str | None = None
    tool_calls_made: list[dict] = field(default_factory=list)
    tokens_used: int = 0
    usage_stats: UsageStats | None = None


SUB_AGENT_SYSTEM_PROMPT = """You are a focused coding agent handling a single step of a larger plan.

Your job is to complete THIS SPECIFIC STEP using the tools and guidance provided.

## Core Guidelines

1. Focus ONLY on this step - don't try to do future steps
2. Use tools to accomplish the task efficiently
3. Follow skill guidance - it contains proven approaches
4. Report your output clearly

## Tool Usage

- Call multiple tools in parallel when they're independent for faster execution.
- Use specialized file tools (read, write, edit, glob, grep) instead of bash for file operations.
- Never use bash for: cat, head, tail, sed, awk, echo, find, grep - use dedicated tools.

## File Operations

- ALWAYS read a file before editing. Never modify code you haven't read.
- Use edit for precise string replacements. old_string must match exactly.
- Include enough context in edits to make matches unique.
- Use glob for file patterns ("**/*.py"), grep for content search.

## Code Quality

- Only make changes directly requested. Don't over-engineer.
- Don't add unnecessary comments, docstrings, or type annotations.
- Keep solutions simple and focused on the task.

## Python Scripts

- Only use Python standard library modules - DO NOT import external packages like requests, beautifulsoup4, pandas, etc.
- Use the web_fetch tool instead of requests/urllib for HTTP requests.
- Use built-in json, re, html.parser for parsing - not external libraries.
- If you need data that web_fetch already retrieved, use that data directly.

When done, provide clear output the orchestrating agent can use."""


class StepSubAgent:
    """
    A sub-agent that executes a single plan step in isolated context.

    Each sub-agent:
    - Gets a fresh Claude conversation
    - Has access to relevant tools for just this step
    - Returns a structured result
    - Doesn't pollute the main agent's context

    Uses sub_agent_model (defaults to main model) which can be
    configured to use a faster/cheaper model for step execution.
    """

    def __init__(
        self,
        config: PromethianConfig,
        step: PlanStep,
        plan: Plan,
        previous_outputs: dict[str, Any],
        tool_executor: ToolExecutor,
        healing_context: list[str] | None = None,
    ):
        self.config = config
        self.step = step
        self.plan = plan
        self.previous_outputs = previous_outputs
        self.tool_executor = tool_executor
        self.healing_context = healing_context or []

        # Create a fresh Claude client for this sub-agent
        # Uses sub_agent_model which can be different from main model
        self.claude = ClaudeClient(
            api_key=config.anthropic_api_key,
            model=config.get_sub_agent_model(),
            max_tokens=config.sub_agent_max_tokens,
        )

        # Built-in tools (lazy init)
        self._bash: BashTool | None = None
        self._web_search: WebSearchTool | None = None
        self._web_fetch: WebFetchTool | None = None
        self._browser: BrowserTool | None = None
        self._file_tools: FileTools | None = None
        self._todo_tool: TodoWriteTool | None = None
        self._ask_user_tool: AskUserTool | None = None

        # Verbose logger
        self._logger = get_logger()

    def _get_todo_tool(self) -> TodoWriteTool:
        if self._todo_tool is None:
            self._todo_tool = get_todo_tool()
        return self._todo_tool

    def _get_ask_user_tool(self) -> AskUserTool:
        if self._ask_user_tool is None:
            self._ask_user_tool = get_ask_user_tool()
        return self._ask_user_tool

    def _get_file_tools(self) -> FileTools:
        if self._file_tools is None:
            self._file_tools = get_file_tools(working_dir=self.config.working_dir)
        return self._file_tools

    def _get_bash(self) -> BashTool:
        if self._bash is None:
            self._bash = get_bash(
                timeout=self.config.tool_timeout,
                working_dir=self.config.working_dir,
            )
        return self._bash

    def _get_web_search(self) -> WebSearchTool:
        if self._web_search is None:
            self._web_search = get_web_search()
        return self._web_search

    def _get_web_fetch(self) -> WebFetchTool:
        if self._web_fetch is None:
            self._web_fetch = get_web_fetch()
        return self._web_fetch

    async def _get_browser(self) -> BrowserTool | None:
        if self._browser is None and BROWSER_AVAILABLE:
            self._browser = await get_browser()
        return self._browser

    async def execute(self) -> SubAgentResult:
        """
        Execute the step in this sub-agent's isolated context.

        Returns:
            SubAgentResult with the outcome
        """
        tool_calls_made = []

        try:
            # Build the prompt with context for just this step
            # Separates cacheable (skills) from dynamic (step desc, outputs)
            cacheable_context, dynamic_prompt = self._build_step_prompt()

            # Gather tools needed for this step
            tools = await self._gather_tools()

            if tools:
                # Execute with tool use in a loop (with caching for efficiency)
                result = await self._execute_with_tools(
                    cacheable_context, dynamic_prompt, tools, tool_calls_made
                )
            else:
                # No tools needed - single completion
                # Combine prompts for simple case
                full_prompt = f"{cacheable_context}\n\n{dynamic_prompt}" if cacheable_context else dynamic_prompt
                result = await self.claude.complete(
                    prompt=full_prompt,
                    system=SUB_AGENT_SYSTEM_PROMPT,
                )

            # Truncate output to prevent context overflow in subsequent steps
            truncated_output = _truncate_output(result, max_chars=10000)

            return SubAgentResult(
                success=True,
                output=truncated_output,
                tool_calls_made=tool_calls_made,
                usage_stats=self.claude.usage,
            )

        except Exception as e:
            return SubAgentResult(
                success=False,
                error=str(e),
                tool_calls_made=tool_calls_made,
                usage_stats=self.claude.usage,
            )

    async def _execute_with_tools(
        self,
        cacheable_context: str,
        dynamic_prompt: str,
        tools: list[dict],
        tool_calls_made: list[dict],
    ) -> Any:
        """
        Execute the step with tool use support in a multi-turn loop.

        Uses prompt caching for efficiency:
        - System prompt is cached
        - cacheable_context (skills, static info) is cached
        - dynamic_prompt changes each call

        Continues calling tools until Claude indicates the step is complete.
        """
        import json

        MAX_TURNS = 10  # Prevent infinite loops
        all_results = []
        final_text = ""

        # Build conversation history for multi-turn
        messages = []

        # Build initial user message with caching
        if cacheable_context:
            initial_content = [
                {"type": "text", "text": cacheable_context, "cache_control": {"type": "ephemeral"}},
                {"type": "text", "text": dynamic_prompt},
            ]
        else:
            initial_content = dynamic_prompt

        messages.append({"role": "user", "content": initial_content})

        pending_tool_results = []  # Track results for continue_with_tool_result

        for turn in range(MAX_TURNS):
            if turn == 0:
                # Initial completion with caching
                text, tool_calls = await self.claude.complete_with_tools(
                    prompt=dynamic_prompt,
                    tools=tools,
                    system=SUB_AGENT_SYSTEM_PROMPT,
                    cache_system=True,
                    cacheable_context=cacheable_context if cacheable_context else None,
                )
            else:
                # Continue conversation with tool results from previous turn
                # Note: continue_with_tool_result creates a new list with tool_results added
                text, tool_calls = await self.claude.continue_with_tool_result(
                    messages=messages,
                    tool_results=pending_tool_results,
                    tools=tools,
                    system=SUB_AGENT_SYSTEM_PROMPT,
                    cache_system=True,
                )
                # Update our messages to include the tool_results that were sent
                tool_result_content = [
                    {"type": "tool_result", "tool_use_id": tr["tool_use_id"], "content": tr["content"]}
                    for tr in pending_tool_results
                ]
                messages.append({"role": "user", "content": tool_result_content})

            final_text = text or final_text

            # If no tool calls, step is complete
            if not tool_calls:
                break

            # Process tool calls
            assistant_content = []
            pending_tool_results = []  # Reset for this turn's results

            if text:
                assistant_content.append({"type": "text", "text": text})

            for call in tool_calls:
                tool_name = call["name"]
                tool_input = call["input"]
                tool_use_id = call.get("id", f"tool_{len(tool_calls_made)}")

                tool_calls_made.append({
                    "tool": tool_name,
                    "input": tool_input,
                })

                # Add tool_use to assistant content
                assistant_content.append({
                    "type": "tool_use",
                    "id": tool_use_id,
                    "name": tool_name,
                    "input": tool_input,
                })

                # Execute tool call
                tool_start_time = self._logger.tool_call_start(tool_name, tool_input)
                result = await self._execute_tool_call(tool_name, tool_input)
                self._logger.tool_call_complete(
                    tool_name,
                    result.get("success", False),
                    tool_start_time,
                    result=result.get("result"),
                    error=result.get("error"),
                )

                all_results.append(result)
                tool_calls_made[-1]["result"] = result

                # Prepare tool result for next turn's continuation
                result_content = json.dumps(result, default=str)[:4000]  # Truncate large results
                pending_tool_results.append({
                    "tool_use_id": tool_use_id,
                    "content": result_content,
                })

            # Add assistant response to messages (tool_results added at start of next turn)
            messages.append({"role": "assistant", "content": assistant_content})

        # Return combined result
        return {
            "text": final_text,
            "tool_results": all_results,
        }

    async def _execute_tool_call(
        self,
        tool_name: str,
        tool_input: dict,
    ) -> dict:
        """Execute a single tool call."""
        # Browser tools
        if tool_name.startswith("browser_"):
            browser = await self._get_browser()
            if browser:
                result = await browser.execute_tool(tool_name, tool_input)
                return {"success": result.success, "result": result.data, "error": result.error}
            return {"success": False, "error": "Browser not available"}

        # Bash tools
        if tool_name in ("bash", "bash_script"):
            bash = self._get_bash()
            result = await bash.execute_tool(tool_name, tool_input)
            return {
                "success": result.success,
                "result": result.stdout,
                "stderr": result.stderr,
                "exit_code": result.exit_code,
            }

        # Web search tools
        if tool_name in ("web_search", "web_news_search"):
            ws = self._get_web_search()
            result = await ws.execute_tool(tool_name, tool_input)
            if result.success:
                return {
                    "success": True,
                    "results": [
                        {"title": r.title, "url": r.url, "snippet": r.snippet}
                        for r in (result.results or [])
                    ],
                }
            return {"success": False, "error": result.error}

        # Web fetch tool (lightweight HTTP fetch, no browser required)
        if tool_name == "web_fetch":
            wf = self._get_web_fetch()
            result = await wf.fetch(tool_input.get("url", ""))
            if result.success:
                return {
                    "success": True,
                    "url": result.url,
                    "content": result.content,
                    "content_type": result.content_type,
                    "status_code": result.status_code,
                    "file_path": result.file_path,  # For binary content
                }
            return {"success": False, "error": result.error}

        # File tools (read, write, edit, glob, grep)
        if tool_name in ("read", "write", "edit", "glob", "grep"):
            file_tools = self._get_file_tools()
            result = await file_tools.execute_tool(tool_name, tool_input)

            if tool_name == "read":
                if result.success:
                    return {
                        "success": True,
                        "content": result.content,
                        "file_path": result.file_path,
                        "total_lines": result.total_lines,
                    }
                return {"success": False, "error": result.error}

            elif tool_name == "write":
                if result.success:
                    return {
                        "success": True,
                        "file_path": result.file_path,
                        "bytes_written": result.bytes_written,
                    }
                return {"success": False, "error": result.error}

            elif tool_name == "edit":
                if result.success:
                    return {
                        "success": True,
                        "file_path": result.file_path,
                        "replacements_made": result.replacements_made,
                    }
                return {"success": False, "error": result.error}

            elif tool_name == "glob":
                if result.success:
                    return {
                        "success": True,
                        "files": result.files,
                        "count": len(result.files),
                    }
                return {"success": False, "error": result.error}

            elif tool_name == "grep":
                if result.success:
                    return {
                        "success": True,
                        "matches": [
                            {"file": m.file, "line": m.line_number, "content": m.content}
                            for m in result.matches
                        ],
                        "files_searched": result.files_searched,
                    }
                return {"success": False, "error": result.error}

        # Todo tool
        if tool_name == "todo_write":
            todo_tool = self._get_todo_tool()
            result = await todo_tool.execute_tool(tool_name, tool_input)
            if result.success:
                return {
                    "success": True,
                    "todos": [t.to_dict() for t in result.todos],
                    "count": len(result.todos),
                }
            return {"success": False, "error": result.error}

        # Ask user tool
        if tool_name == "ask_user":
            ask_user_tool = self._get_ask_user_tool()
            result = ask_user_tool.execute(**tool_input)
            if result.success:
                return {
                    "success": True,
                    "response": result.response,
                }
            return {
                "success": False,
                "cancelled": result.cancelled,
                "error": result.error or "User cancelled input",
            }

        # Custom tools from plan
        try:
            result = await self.tool_executor.execute(
                tool_name=tool_name,
                tool_input=tool_input,
            )
            return {"success": True, "result": result}
        except ToolExecutionError as e:
            return {
                "success": False,
                "error": str(e),
                "stdout": e.stdout,
                "stderr": e.stderr,
            }
        except Exception as e:
            return {"success": False, "error": str(e)}

    def _build_step_prompt(self) -> tuple[str, str]:
        """
        Build the prompt for this step, separating cacheable and dynamic parts.

        Returns:
            Tuple of (cacheable_context, dynamic_prompt)
            - cacheable_context: Static content (skills, tool docs) that can be cached
            - dynamic_prompt: Step-specific content that changes each step
        """
        # Cacheable parts: skills, gathered context, and tool docs (static across steps)
        cacheable_parts = []

        # Add gathered context (URLs, file contents, etc.) - IMPORTANT for accuracy
        if self.plan.gathered_context:
            cacheable_parts.append("# Pre-gathered Context (USE THESE URLs AND INFORMATION)")
            cacheable_parts.append(self.plan.gathered_context)

        # Add relevant skill guidance (cacheable - same skills used across steps)
        skill_content = []
        for skill_id in self.step.skills_to_use:
            for skill in self.plan.relevant_skills:
                if skill.id == skill_id:
                    skill_content.append(f"## {skill.name}\n{skill.content}")

        if skill_content:
            cacheable_parts.append("# Guidance from Skills")
            cacheable_parts.extend(skill_content)

        # Dynamic parts: step description, previous outputs, healing context
        dynamic_parts = [
            f"# Your Task\n{self.step.description}",
            f"\n# Expected Output\n{self.step.expected_output}",
        ]

        # Add outputs from dependencies (dynamic - changes each step)
        if self.step.dependencies and self.previous_outputs:
            dynamic_parts.append("\n# Context from Previous Steps")
            for dep_id in self.step.dependencies:
                if dep_id in self.previous_outputs:
                    output = self.previous_outputs[dep_id]
                    # Truncate all output types to prevent context overflow
                    # Use smaller limit per dependency to leave room for multiple deps
                    max_per_dep = 4000 // max(len(self.step.dependencies), 1)
                    output = _truncate_output(output, max_chars=max(max_per_dep, 2000))
                    # Convert to string for prompt
                    if isinstance(output, dict):
                        import json
                        output = json.dumps(output, indent=2, default=str)
                    dynamic_parts.append(f"\n## From {dep_id}:\n{output}")

        # Add healing context from previous failed attempts (dynamic)
        if self.healing_context:
            dynamic_parts.append("\n# Previous Attempts (IMPORTANT - learn from these failures)")
            for entry in self.healing_context:
                dynamic_parts.append(f"\n{entry}")
            dynamic_parts.append("\nAvoid the same mistakes. Try a different approach if a tool keeps failing.")

        dynamic_parts.append("\nComplete this step and provide your output.")

        cacheable = "\n".join(cacheable_parts) if cacheable_parts else ""
        dynamic = "\n".join(dynamic_parts)

        return cacheable, dynamic

    async def _gather_tools(self) -> list[dict]:
        """Gather all available tools for this step.

        All built-in tools are provided unconditionally so the sub-agent can
        choose the right tool for the job. Keyword-based filtering was too
        fragile and caused failures when step descriptions didn't match
        expected keywords (e.g., "read the blog post" wouldn't get web_fetch).
        """
        tools = []
        seen_names: set[str] = set()

        def add_tool(schema: dict) -> None:
            """Add tool if name not already seen."""
            name = schema.get("name")
            if name and name not in seen_names:
                tools.append(schema)
                seen_names.add(name)

        # Add custom tools from plan first (they take priority)
        for tool_id in self.step.tools_to_use:
            for tool in self.plan.relevant_tools:
                if tool.id == tool_id:
                    add_tool(tool.get_schema())

        # File tools - read, write, edit, glob, grep
        file_tools = self._get_file_tools()
        for schema in file_tools.get_tool_schemas():
            add_tool(schema)

        # Bash - shell command execution
        bash = self._get_bash()
        for schema in bash.get_tool_schemas():
            add_tool(schema)

        # Web search - search the web
        ws = self._get_web_search()
        for schema in ws.get_tool_schemas():
            add_tool(schema)

        # Web fetch - lightweight HTTP fetch for URLs
        wf = self._get_web_fetch()
        for schema in wf.get_tool_schemas():
            add_tool(schema)

        # Browser - full browser automation (only if Playwright available)
        if BROWSER_AVAILABLE:
            browser = await self._get_browser()
            if browser:
                for schema in browser.get_tool_schemas():
                    add_tool(schema)

        # Todo tool - task tracking
        todo_tool = self._get_todo_tool()
        for schema in todo_tool.get_tool_schemas():
            add_tool(schema)

        # Ask user tool - for credentials, input, or confirmation
        ask_user_tool = self._get_ask_user_tool()
        add_tool(ask_user_tool.schema)

        return tools


class SubAgentExecutor:
    """
    Executes plan steps using sub-agents.

    Each step gets its own sub-agent with isolated context.
    Steps can be executed in parallel when their dependencies allow.
    """

    def __init__(
        self,
        config: PromethianConfig,
        tool_runner: ToolRunner,
    ):
        self.config = config
        self.tool_runner = tool_runner
        self._usage_stats = UsageStats()  # Accumulated usage from all sub-agents
        self._logger = get_logger()
        # Track URLs that repeatedly fail (e.g., 403 Forbidden)
        # Key: URL, Value: (failure_count, last_error)
        self._failed_urls: dict[str, tuple[int, str]] = {}

    @property
    def usage_stats(self) -> UsageStats:
        """Get accumulated usage statistics from all sub-agent executions."""
        return self._usage_stats

    async def execute_step(
        self,
        step: PlanStep,
        plan: Plan,
        previous_outputs: dict[str, Any],
        progress: ProgressTracker | None = None,
    ) -> StepResult:
        """
        Execute a single step using a sub-agent with self-healing retries.

        Args:
            step: The step to execute
            plan: The full plan (for context)
            previous_outputs: Outputs from completed steps
            progress: Optional progress tracker for verbose output

        Returns:
            StepResult with the outcome
        """
        # Create tool executor for custom tools
        tool_executor = ToolExecutor(self.tool_runner, plan.relevant_tools)
        progress = progress or NullProgressTracker()

        healing_log: list[str] = []
        last_error = None
        max_attempts = self.config.max_self_heal_attempts

        for attempt in range(max_attempts):
            # Create and run sub-agent
            sub_agent = StepSubAgent(
                config=self.config,
                step=step,
                plan=plan,
                previous_outputs=previous_outputs,
                tool_executor=tool_executor,
                healing_context=healing_log,
            )

            result = await sub_agent.execute()

            # Merge usage stats from this sub-agent execution
            if result.usage_stats:
                self._usage_stats.merge(result.usage_stats)

            if result.success:
                # Check for ANY tool failures (not just custom tools)
                # A step should fail if critical tools failed
                all_tool_failures = self._extract_all_tool_failures(result.tool_calls_made)

                if all_tool_failures:
                    # Check if this is a recoverable failure or fundamental issue
                    tool_name, error_details, is_recoverable = all_tool_failures[0]
                    last_error = f"Tool '{tool_name}' failed: {error_details}"

                    # Non-recoverable failures (e.g., Playwright not installed) should fail immediately
                    if not is_recoverable:
                        return StepResult(
                            step_id=step.id,
                            status=ExecutionStatus.FAILURE,
                            error=f"Critical tool failure: {last_error}",
                            output=result.output,
                            attempts=attempt + 1,
                            skills_used=step.skills_to_use,
                            tools_used=step.tools_to_use + [tc["tool"] for tc in result.tool_calls_made],
                            healing_log=healing_log if healing_log else None,
                        )

                    # For web_fetch failures: if we have enough successful fetches, consider it SUCCESS
                    # Some URLs will always fail (403s from anti-bot sites) - that's OK
                    if tool_name == "web_fetch" and self._has_enough_successful_fetches(result.tool_calls_made):
                        self._logger.log(
                            "partial_success",
                            f"Step has enough successful web fetches despite some failures",
                            successful_fetches=sum(
                                1 for tc in result.tool_calls_made
                                if tc.get("tool") == "web_fetch" and tc.get("result", {}).get("success", False)
                            ),
                            failed_urls=list(self._extract_url_failures(result.tool_calls_made).keys()),
                        )
                        # Success with partial web fetch results
                        return StepResult(
                            step_id=step.id,
                            status=ExecutionStatus.SUCCESS,
                            output=result.output,
                            attempts=attempt + 1,
                            skills_used=step.skills_to_use,
                            tools_used=step.tools_to_use + [tc["tool"] for tc in result.tool_calls_made],
                            healing_log=healing_log if healing_log else None,
                        )

                    # Recoverable failures can be retried
                    if attempt < max_attempts - 1:
                        # Log and emit self-healing progress
                        self._logger.self_heal_start(
                            step_id=step.id,
                            attempt=attempt + 1,
                            max_attempts=max_attempts,
                            error=last_error,
                        )
                        await progress.emit(
                            ProgressStage.SELF_HEALING,
                            f"Self-healing: tool '{tool_name}' failed, analyzing...",
                            attempt=attempt + 1,
                            max_attempts=max_attempts,
                            tool_name=tool_name,
                            error=error_details[:500],  # Show more error context
                        )

                        # Get healing suggestion with blocked URLs context
                        blocked_context = self._get_blocked_urls_context()
                        suggestion = await self._get_healing_suggestion(
                            step=step,
                            tool_name=tool_name,
                            error=error_details + blocked_context,
                            attempt=attempt,
                        )

                        # Log healing suggestion
                        self._logger.self_heal_suggestion(
                            step_id=step.id,
                            attempt=attempt + 1,
                            suggestion=suggestion,
                        )

                        healing_log.append(
                            f"Attempt {attempt + 1} - Tool '{tool_name}' failed: {error_details}\nFix: {suggestion}"
                        )

                        await progress.emit(
                            ProgressStage.HEAL_ATTEMPT,
                            f"Retrying with fix (attempt {attempt + 2}/{max_attempts})...",
                            attempt=attempt + 2,
                            max_attempts=max_attempts,
                            suggestion=suggestion[:500],  # Show more of the fix
                        )
                        continue  # Retry
                    else:
                        # Max attempts reached - but for web_fetch, check if we have ANY useful output
                        if tool_name == "web_fetch" and result.output:
                            output_text = str(result.output)
                            # If we generated meaningful content, consider it a success
                            if len(output_text) > 500 and "tool_results" in output_text:
                                self._logger.log(
                                    "graceful_degradation",
                                    "Step produced output despite web fetch failures, marking as success",
                                    output_length=len(output_text),
                                )
                                return StepResult(
                                    step_id=step.id,
                                    status=ExecutionStatus.SUCCESS,
                                    output=result.output,
                                    attempts=attempt + 1,
                                    skills_used=step.skills_to_use,
                                    tools_used=step.tools_to_use + [tc["tool"] for tc in result.tool_calls_made],
                                    healing_log=healing_log if healing_log else None,
                                )

                        # Max attempts reached for recoverable failure
                        return StepResult(
                            step_id=step.id,
                            status=ExecutionStatus.FAILURE,
                            error=f"Failed after {max_attempts} attempts: {last_error}",
                            output=result.output,
                            attempts=attempt + 1,
                            skills_used=step.skills_to_use,
                            tools_used=step.tools_to_use + [tc["tool"] for tc in result.tool_calls_made],
                            healing_log=healing_log if healing_log else None,
                        )

                # Success with no tool failures
                return StepResult(
                    step_id=step.id,
                    status=ExecutionStatus.SUCCESS,
                    output=result.output,
                    attempts=attempt + 1,
                    skills_used=step.skills_to_use,
                    tools_used=step.tools_to_use + [tc["tool"] for tc in result.tool_calls_made],
                    healing_log=healing_log if healing_log else None,
                )
            else:
                # Execution failed entirely
                last_error = result.error

                if attempt < max_attempts - 1:
                    # Emit self-healing progress
                    await progress.emit(
                        ProgressStage.SELF_HEALING,
                        f"Self-healing: step failed, analyzing error...",
                        attempt=attempt + 1,
                        max_attempts=max_attempts,
                        error=result.error[:500] if result.error else "Unknown error",
                    )

                    # Get healing suggestion
                    suggestion = await self._get_healing_suggestion(
                        step=step,
                        tool_name=None,
                        error=result.error or "Unknown error",
                        attempt=attempt,
                    )
                    healing_log.append(
                        f"Attempt {attempt + 1} failed: {result.error}\nFix: {suggestion}"
                    )

                    await progress.emit(
                        ProgressStage.HEAL_ATTEMPT,
                        f"Retrying with fix (attempt {attempt + 2}/{max_attempts})...",
                        attempt=attempt + 2,
                        max_attempts=max_attempts,
                        suggestion=suggestion[:500],
                    )

        # All attempts failed
        return StepResult(
            step_id=step.id,
            status=ExecutionStatus.FAILURE,
            error=last_error,
            attempts=max_attempts,
            skills_used=step.skills_to_use,
            tools_used=step.tools_to_use,
            healing_log=healing_log if healing_log else None,
        )

    def _extract_all_tool_failures(
        self,
        tool_calls_made: list[dict],
    ) -> list[tuple[str, str, bool]]:
        """
        Extract all tool failures (both built-in and custom).

        Returns:
            List of (tool_name, error_details, is_recoverable) tuples.

            is_recoverable=False for fundamental issues like:
            - Playwright not installed
            - Missing system dependencies
            - Invalid URL formats that can't be fixed

            is_recoverable=True for issues that might be fixed by retry:
            - Network timeouts
            - Temporary API errors
            - Code bugs in custom tools
        """
        # Errors that indicate fundamental issues (not recoverable by retry)
        # Note: "module not found" and "no module named" are intentionally NOT here
        # because self-healing can suggest using built-in tools instead
        NON_RECOVERABLE_PATTERNS = [
            "playwright not installed",
            "playwright install",
            "browser not available",
            "permission denied",
            "access denied",
        ]

        # HTTP errors that are common for anti-bot sites - these URLs should be skipped
        # rather than causing step failure if we have other successful results
        HTTP_SKIP_PATTERNS = [
            "http 403",
            "forbidden",
            "http 401",
            "unauthorized",
            "http 429",
            "rate limit",
        ]

        failures = []

        for tc in tool_calls_made:
            tool_name = tc.get("tool")
            result = tc.get("result", {})
            tool_input = tc.get("input", {})

            if not result.get("success", True):
                error = result.get("error", "Unknown error")
                stderr = result.get("stderr", "")
                full_error = f"{error}\n{stderr}".strip().lower()

                # Track URL failures for web_fetch
                if tool_name == "web_fetch":
                    url = tool_input.get("url", "")
                    if url and any(p in full_error for p in HTTP_SKIP_PATTERNS):
                        # Track this URL as failing
                        count, _ = self._failed_urls.get(url, (0, ""))
                        self._failed_urls[url] = (count + 1, error)

                # Check if this is a non-recoverable failure
                is_recoverable = not any(
                    pattern in full_error for pattern in NON_RECOVERABLE_PATTERNS
                )

                failures.append((tool_name, f"{error}\n{stderr}".strip(), is_recoverable))

        return failures

    def _extract_url_failures(
        self,
        tool_calls_made: list[dict],
    ) -> dict[str, str]:
        """
        Extract URL-specific failures from web_fetch calls.

        Returns:
            Dict mapping failed URL -> error message
        """
        url_failures = {}

        for tc in tool_calls_made:
            if tc.get("tool") == "web_fetch":
                result = tc.get("result", {})
                tool_input = tc.get("input", {})
                url = tool_input.get("url", "")

                if url and not result.get("success", True):
                    url_failures[url] = result.get("error", "Unknown error")

        return url_failures

    def _has_enough_successful_fetches(
        self,
        tool_calls_made: list[dict],
        min_successes: int = 2,
    ) -> bool:
        """
        Check if we have enough successful web fetches to consider the step successful.

        For web research tasks, having SOME successful fetches is often enough,
        even if other URLs failed (e.g., due to anti-bot protection).
        """
        successful_fetches = 0
        for tc in tool_calls_made:
            if tc.get("tool") == "web_fetch":
                result = tc.get("result", {})
                if result.get("success", False):
                    successful_fetches += 1

        return successful_fetches >= min_successes

    def _get_blocked_urls_context(self) -> str:
        """
        Get context about blocked URLs to include in healing suggestions.
        """
        if not self._failed_urls:
            return ""

        blocked = []
        for url, (count, error) in self._failed_urls.items():
            if count >= 2:  # Only mention URLs that failed multiple times
                blocked.append(f"  - {url} (failed {count}x: {error})")

        if not blocked:
            return ""

        return (
            "\n\n## Blocked URLs (DO NOT retry these - they will fail again)\n" +
            "\n".join(blocked) +
            "\nUse alternative URLs from the gathered context or skip these sources."
        )

    async def _get_healing_suggestion(
        self,
        step: PlanStep,
        tool_name: str | None,
        error: str,
        attempt: int,
    ) -> str:
        """Get a healing suggestion from Claude."""
        # Create a temporary Claude client for healing
        claude = ClaudeClient(
            api_key=self.config.anthropic_api_key,
            model=self.config.get_sub_agent_model(),  # Use sub-agent model for healing
            max_tokens=1024,
        )

        if tool_name:
            prompt = f"""A custom tool failed during step execution.

Tool Name: {tool_name}
Step: {step.description}
Error: {error}

This was attempt {attempt + 1}. The tool has a bug.
Analyze the error and suggest a SPECIFIC fix for the tool.
Focus on what code change would fix this bug."""
        else:
            prompt = f"""A step in our plan failed.

Step: {step.description}
Error: {error}

This was attempt {attempt + 1}. Analyze what went wrong and suggest how to fix it."""

        return await claude.complete(prompt=prompt)

    async def execute_steps_parallel(
        self,
        steps: list[PlanStep],
        plan: Plan,
        previous_outputs: dict[str, Any],
    ) -> list[StepResult]:
        """
        Execute multiple steps in parallel (when dependencies allow).

        Args:
            steps: Steps to execute (must have no inter-dependencies)
            plan: The full plan
            previous_outputs: Outputs from completed steps

        Returns:
            List of StepResults
        """
        tasks = [
            self.execute_step(step, plan, previous_outputs)
            for step in steps
        ]
        return await asyncio.gather(*tasks)
