#!/usr/bin/env python

import asyncio
import base64
import hashlib
import json
import locale
import math
import mimetypes
import os
import platform
import re
import sys
import threading
import time
import traceback
import weakref
from collections import defaultdict
from datetime import datetime

# Optional dependency: used to convert locale codes (eg ``en_US``)
# into human-readable language names (eg ``English``).
try:
    from babel import Locale  # type: ignore
except ImportError:  # Babel not installed – we will fall back to a small mapping
    Locale = None
from json.decoder import JSONDecodeError
from pathlib import Path
from typing import List

import httpx
from litellm import experimental_mcp_client
from litellm.types.utils import ModelResponse
from prompt_toolkit.patch_stdout import patch_stdout
from rich.console import Console

import aider.prompts.utils.system as prompts
from aider import __version__, models, urls, utils
from aider.commands import Commands, SwitchCoder
from aider.exceptions import LiteLLMExceptions
from aider.helpers import coroutines
from aider.helpers.profiler import TokenProfiler
from aider.history import ChatSummary
from aider.io import ConfirmGroup, InputOutput
from aider.linter import Linter
from aider.llm import litellm
from aider.mcp.server import LocalServer
from aider.models import RETRY_TIMEOUT
from aider.reasoning_tags import (
    REASONING_TAG,
    format_reasoning_content,
    remove_reasoning_content,
    replace_reasoning_tags,
)
from aider.repo import ANY_GIT_ERROR, GitRepo
from aider.repomap import RepoMap
from aider.run_cmd import run_cmd
from aider.sessions import SessionManager
from aider.tools.utils.output import print_tool_response
from aider.utils import format_tokens, is_image_file

from ..dump import dump  # noqa: F401
from ..prompts.utils.prompt_registry import registry
from .chat_chunks import ChatChunks


class UnknownEditFormat(ValueError):
    def __init__(self, edit_format, valid_formats):
        self.edit_format = edit_format
        self.valid_formats = valid_formats
        super().__init__(
            f"Unknown edit format {edit_format}. Valid formats are: {', '.join(valid_formats)}"
        )


class MissingAPIKeyError(ValueError):
    pass


class FinishReasonLength(Exception):
    pass


def wrap_fence(name):
    return f"<{name}>", f"</{name}>"


all_fences = [
    ("`" * 3, "`" * 3),
    ("`" * 4, "`" * 4),  # LLMs ignore and revert to triple-backtick, causing #2879
    wrap_fence("source"),
    wrap_fence("code"),
    wrap_fence("pre"),
    wrap_fence("codeblock"),
    wrap_fence("sourcecode"),
]


class Coder:
    abs_fnames = None
    abs_read_only_fnames = None
    abs_read_only_stubs_fnames = None
    repo = None
    last_aider_commit_hash = None
    aider_edited_files = None
    last_asked_for_commit_time = 0
    repo_map = None
    functions = None
    num_exhausted_context_windows = 0
    num_malformed_responses = 0
    last_keyboard_interrupt = None
    num_reflections = 0
    max_reflections = 3
    num_tool_calls = 0
    max_tool_calls = 25
    edit_format = None
    yield_stream = False
    temperature = None
    auto_lint = True
    auto_test = False
    test_cmd = None
    lint_outcome = None
    test_outcome = None
    multi_response_content = ""
    partial_response_content = ""
    partial_response_reasoning_content = ""
    partial_response_chunks = []
    partial_response_tool_calls = []
    commit_before_message = []
    message_cost = 0.0
    add_cache_headers = False
    cache_warming_thread = None
    num_cache_warming_pings = 0
    suggest_shell_commands = True
    detect_urls = True
    ignore_mentions = None
    chat_language = None
    commit_language = None
    file_watcher = None
    mcp_servers = None
    mcp_tools = None
    run_one_completed = True
    compact_context_completed = True
    suppress_announcements_for_next_prompt = False
    tool_reflection = False
    # Task coordination state variables
    input_running = False
    output_running = False

    # Context management settings (for all modes)
    context_management_enabled = False  # Disabled by default except for agent mode
    large_file_token_threshold = (
        25000  # Files larger than this will be truncated when context management is enabled
    )

    ok_to_warm_cache = False

    # Weak reference to TUI app instance (when running in TUI mode)
    tui = None

    _prompt_cache = {}

    @classmethod
    async def create(
        self,
        main_model=None,
        edit_format=None,
        io=None,
        from_coder=None,
        summarize_from_coder=True,
        args=None,
        **kwargs,
    ):
        import aider.coders as coders

        if not main_model:
            if from_coder:
                main_model = from_coder.main_model
            else:
                main_model = models.Model(models.DEFAULT_MODEL_NAME, io=io)

        if edit_format == "code":
            edit_format = None
        if edit_format is None:
            if from_coder:
                edit_format = from_coder.edit_format
            else:
                edit_format = main_model.edit_format

        if not io and from_coder:
            io = from_coder.io

        if from_coder:
            if not args:
                args = from_coder.args

            use_kwargs = dict(from_coder.original_kwargs)  # copy orig kwargs

            # If the edit format changes, we can't leave old ASSISTANT
            # messages in the chat history. The old edit format will
            # confused the new LLM. It may try and imitate it, disobeying
            # the system prompt.
            done_messages = from_coder.done_messages
            if edit_format != from_coder.edit_format and done_messages and summarize_from_coder:
                try:
                    io.tool_warning("Summarizing messages, please wait...")
                    done_messages = await from_coder.summarizer.summarize_all(done_messages)
                except (KeyboardInterrupt, ValueError):
                    # If summarization fails, keep the original messages and warn the user
                    io.tool_warning(
                        "Chat history summarization failed, continuing with full history"
                    )

            # Bring along context from the old Coder
            update = dict(
                fnames=list(from_coder.abs_fnames),
                read_only_fnames=list(from_coder.abs_read_only_fnames),  # Copy read-only files
                read_only_stubs_fnames=list(
                    from_coder.abs_read_only_stubs_fnames
                ),  # Copy read-only stubs
                done_messages=done_messages,
                cur_messages=from_coder.cur_messages,
                aider_commit_hashes=from_coder.aider_commit_hashes,
                commands=from_coder.commands.clone(),
                total_cost=from_coder.total_cost,
                ignore_mentions=from_coder.ignore_mentions,
                total_tokens_sent=from_coder.total_tokens_sent,
                total_tokens_received=from_coder.total_tokens_received,
                file_watcher=from_coder.file_watcher,
            )
            use_kwargs.update(update)  # override to complete the switch
            use_kwargs.update(kwargs)  # override passed kwargs

            kwargs = use_kwargs
            from_coder.ok_to_warm_cache = False

        res = None
        if (
            getattr(main_model, "copy_paste_mode", False)
            and getattr(main_model, "copy_paste_transport", "api") == "clipboard"
        ):
            res = coders.CopyPasteCoder(main_model, io, args=args, **kwargs)

        if not res:
            for coder in coders.__all__:
                if hasattr(coder, "edit_format") and coder.edit_format == edit_format:
                    res = coder(main_model, io, args=args, **kwargs)

        if res is not None:
            if from_coder:
                if from_coder.mcp_servers and kwargs.get("mcp_servers", False):
                    res.mcp_servers = from_coder.mcp_servers
                    res.mcp_tools = from_coder.mcp_tools

                # Transfer TUI app weak reference
                res.tui = from_coder.tui

            await res.initialize_mcp_tools()

            res.original_kwargs = dict(kwargs)
            return res

        valid_formats = [
            str(c.edit_format)
            for c in coders.__all__
            if hasattr(c, "edit_format") and c.edit_format is not None
        ]
        raise UnknownEditFormat(edit_format, valid_formats)

    async def clone(self, **kwargs):
        new_coder = await Coder.create(from_coder=self, **kwargs)
        return new_coder

    def __init__(
        self,
        main_model,
        io,
        args=None,
        repo=None,
        fnames=None,
        add_gitignore_files=False,
        read_only_fnames=None,
        read_only_stubs_fnames=None,
        show_diffs=False,
        auto_commits=True,
        dirty_commits=True,
        dry_run=False,
        map_tokens=1024,
        verbose=False,
        stream=True,
        use_git=True,
        cur_messages=None,
        done_messages=None,
        restore_chat_history=False,
        auto_lint=True,
        auto_test=False,
        lint_cmds=None,
        test_cmd=None,
        aider_commit_hashes=None,
        map_mul_no_files=8,
        map_max_line_length=100,
        commands=None,
        summarizer=None,
        total_cost=0.0,
        map_refresh="auto",
        cache_prompts=False,
        num_cache_warming_pings=0,
        suggest_shell_commands=True,
        chat_language=None,
        commit_language=None,
        detect_urls=True,
        ignore_mentions=None,
        total_tokens_sent=0,
        total_tokens_received=0,
        file_watcher=None,
        auto_copy_context=False,
        auto_accept_architect=True,
        mcp_servers=None,
        enable_context_compaction=False,
        context_compaction_max_tokens=None,
        context_compaction_summary_tokens=8192,
        map_cache_dir=".",
        repomap_in_memory=False,
        linear_output=False,
    ):
        # initialize from args.map_cache_dir
        self.map_cache_dir = map_cache_dir

        self.chat_language = chat_language
        self.commit_language = commit_language
        self.commit_before_message = []
        self.aider_commit_hashes = set()
        self.rejected_urls = set()
        self.abs_root_path_cache = {}

        self.auto_copy_context = auto_copy_context
        self.auto_accept_architect = auto_accept_architect

        self.ignore_mentions = ignore_mentions
        if not self.ignore_mentions:
            self.ignore_mentions = set()

        self.file_watcher = file_watcher
        if self.file_watcher:
            self.file_watcher.coder = self

        self.suggest_shell_commands = suggest_shell_commands
        self.detect_urls = detect_urls
        self.args = args

        self.num_cache_warming_pings = num_cache_warming_pings
        self.mcp_servers = mcp_servers
        self.enable_context_compaction = enable_context_compaction

        self.context_compaction_max_tokens = context_compaction_max_tokens
        self.context_compaction_summary_tokens = context_compaction_summary_tokens

        if not fnames:
            fnames = []

        if io is None:
            io = InputOutput()

        if aider_commit_hashes:
            self.aider_commit_hashes = aider_commit_hashes
        else:
            self.aider_commit_hashes = set()

        self.chat_completion_call_hashes = []
        self.chat_completion_response_hashes = []
        self.need_commit_before_edits = set()

        self.total_cost = total_cost
        self.total_tokens_sent = total_tokens_sent
        self.total_tokens_received = total_tokens_received
        self.message_tokens_sent = 0
        self.message_tokens_received = 0

        self.token_profiler = TokenProfiler(
            enable_printing=getattr(args, "show_speed", False) if args else False
        )
        self.verbose = verbose
        self.abs_fnames = set()
        self.abs_read_only_fnames = set()
        self.add_gitignore_files = add_gitignore_files
        self.abs_read_only_stubs_fnames = set()

        if cur_messages:
            self.cur_messages = cur_messages
        else:
            self.cur_messages = []

        if done_messages:
            self.done_messages = done_messages
        else:
            self.done_messages = []

        self.io = io
        self.io.coder = weakref.ref(self)

        self.manual_copy_paste = getattr(main_model, "copy_paste_transport", "api") == "clipboard"
        self.copy_paste_mode = getattr(main_model, "copy_paste_mode", False) or auto_copy_context

        self.shell_commands = []
        self.partial_response_tool_calls = []

        if not auto_commits:
            dirty_commits = False

        self.auto_commits = auto_commits
        self.dirty_commits = dirty_commits

        self.dry_run = dry_run
        self.pretty = self.io.pretty
        self.linear_output = linear_output
        self.io.linear = linear_output
        self.main_model = main_model

        # Set the reasoning tag name based on model settings or default
        self.reasoning_tag_name = (
            self.main_model.reasoning_tag if self.main_model.reasoning_tag else REASONING_TAG
        )

        self.stream = stream and main_model.streaming and not self.manual_copy_paste

        if cache_prompts and self.main_model.cache_control:
            self.add_cache_headers = True

        self.show_diffs = show_diffs

        self.commands = commands or Commands(self.io, self)
        self.commands.coder = self

        self.data_cache = {
            "repo": {"last_key": "", "read_only_count": None},
            "relative_files": None,
        }

        self.repo = repo
        if use_git and self.repo is None:
            try:
                self.repo = GitRepo(
                    self.io,
                    fnames,
                    None,
                    models=main_model.commit_message_models(),
                )
            except FileNotFoundError:
                pass

        if self.repo:
            self.root = self.repo.root

        for fname in fnames:
            fname = Path(fname)
            if self.repo and self.repo.git_ignored_file(fname) and not self.add_gitignore_files:
                self.io.tool_warning(f"Skipping {fname} that matches gitignore spec.")
                continue

            if self.repo and self.repo.ignored_file(fname):
                self.io.tool_warning(f"Skipping {fname} that matches aiderignore spec.")
                continue

            if not fname.exists():
                if utils.touch_file(fname):
                    self.io.tool_output(f"Creating empty file {fname}")
                else:
                    self.io.tool_warning(f"Can not create {fname}, skipping.")
                    continue

            if not fname.is_file():
                self.io.tool_warning(f"Skipping {fname} that is not a normal file.")
                continue

            fname = str(fname.resolve())

            self.abs_fnames.add(fname)
            self.check_added_files()

        if not self.repo:
            self.root = utils.find_common_root(self.abs_fnames)

        if read_only_fnames:
            self.abs_read_only_fnames = set()
            for fname in read_only_fnames:
                abs_fname = self.abs_root_path(fname)
                if os.path.exists(abs_fname):
                    self.abs_read_only_fnames.add(abs_fname)
                else:
                    self.io.tool_warning(f"Error: Read-only file {fname} does not exist. Skipping.")

        if read_only_stubs_fnames:
            self.abs_read_only_stubs_fnames = set()
            for fname in read_only_stubs_fnames:
                abs_fname = self.abs_root_path(fname)
                if os.path.exists(abs_fname):
                    self.abs_read_only_stubs_fnames.add(abs_fname)
                else:
                    self.io.tool_warning(
                        f"Error: Read-only (stub) file {fname} does not exist. Skipping."
                    )

        if map_tokens is None:
            use_repo_map = main_model.use_repo_map
            map_tokens = 1024
        else:
            use_repo_map = map_tokens > 0

        max_inp_tokens = self.main_model.info.get("max_input_tokens") or 0

        has_map_prompt = hasattr(self, "gpt_prompts") and self.gpt_prompts.repo_content_prefix

        if use_repo_map and self.repo and has_map_prompt:
            self.repo_map = RepoMap(
                map_tokens,
                self.map_cache_dir,
                self.main_model,
                io,
                self.gpt_prompts.repo_content_prefix,
                self.verbose,
                max_inp_tokens,
                map_mul_no_files=map_mul_no_files,
                refresh=map_refresh,
                max_code_line_length=map_max_line_length,
                repo_root=self.root,
                use_memory_cache=repomap_in_memory,
                use_enhanced_map=False if not self.args or self.args.use_enhanced_map else True,
            )

        self.summarizer = summarizer or ChatSummary(
            [self.main_model.weak_model, self.main_model],
            self.main_model.max_chat_history_tokens,
        )

        self.summarizer_thread = None
        self.summarized_done_messages = []
        self.summarizing_messages = None

        self.files_edited_by_tools = set()

        if not self.done_messages and restore_chat_history:
            history_md = self.io.read_text(self.io.chat_history_file)
            if history_md:
                self.done_messages = utils.split_chat_history_markdown(history_md)
                self.summarize_start()

        # Linting and testing
        self.linter = Linter(root=self.root, encoding=io.encoding)
        self.auto_lint = auto_lint
        self.setup_lint_cmds(lint_cmds)
        self.lint_cmds = lint_cmds
        self.auto_test = auto_test
        self.test_cmd = test_cmd

        # Clean up todo list file on startup; sessions will restore it when needed
        todo_file_path = ".aider.todo.txt"
        abs_path = self.abs_root_path(todo_file_path)
        if os.path.isfile(abs_path):
            try:
                os.remove(abs_path)
                if self.verbose:
                    self.io.tool_output(f"Removed existing todo list file: {todo_file_path}")
            except Exception as e:
                self.io.tool_warning(f"Could not remove todo list file {todo_file_path}: {e}")

        # validate the functions jsonschema
        if self.functions:
            from jsonschema import Draft7Validator

            for function in self.functions:
                Draft7Validator.check_schema(function)

            if self.verbose:
                self.io.tool_output("JSON Schema:")
                self.io.tool_output(json.dumps(self.functions, indent=4))

    @property
    def gpt_prompts(self):
        """Get prompts from the registry based on the coder type."""
        cls = self.__class__

        # Every coder class MUST have a prompt_format attribute
        if not hasattr(cls, "prompt_format"):
            raise AttributeError(
                f"Coder class {cls.__name__} must have a 'prompt_format' attribute. "
                "Add 'prompt_format = \"<format_name>\"' to the class definition."
            )

        if cls.prompt_format is None:
            raise AttributeError(
                f"Coder class {cls.__name__} has prompt_format=None. "
                "It must have a valid prompt format name."
            )

        prompt_name = cls.prompt_format

        # Check cache first
        if prompt_name in Coder._prompt_cache:
            return Coder._prompt_cache[prompt_name]

        # Get prompts from registry
        prompts = registry.get_prompt(prompt_name)

        # Create a simple object that allows attribute access
        class PromptObject:
            def __init__(self, prompts_dict):
                for key, value in prompts_dict.items():
                    setattr(self, key, value)

        # Cache the prompt object
        prompt_obj = PromptObject(prompts)
        Coder._prompt_cache[prompt_name] = prompt_obj

        return prompt_obj

    def get_announcements(self):
        lines = []
        lines.append(f"cecli v{__version__}")

        # Model
        main_model = self.main_model
        weak_model = main_model.weak_model

        if weak_model is not main_model:
            prefix = "Main model"
        else:
            prefix = "Model"

        output = f"{prefix}: {main_model.name} with {self.edit_format} edit format"

        # Check for thinking token budget
        thinking_tokens = main_model.get_thinking_tokens()
        if thinking_tokens:
            output += f", {thinking_tokens} think tokens"

        # Check for reasoning effort
        reasoning_effort = main_model.get_reasoning_effort()
        if reasoning_effort:
            output += f", reasoning {reasoning_effort}"

        if self.add_cache_headers or main_model.caches_by_default:
            output += ", prompt cache"
        if main_model.info.get("supports_assistant_prefill"):
            output += ", infinite output"
        if self.copy_paste_mode:
            output += ", copy/paste mode"

        lines.append(output)

        if self.edit_format == "architect":
            output = (
                f"Editor model: {main_model.editor_model.name} with"
                f" {main_model.editor_edit_format} edit format"
            )
            lines.append(output)

        if weak_model is not main_model:
            output = f"Weak model: {weak_model.name}"
            lines.append(output)

        # Repo
        if self.repo:
            rel_repo_dir = self.repo.get_rel_repo_dir()
            num_files = len(self.repo.get_tracked_files())

            lines.append(f"Git repo: {rel_repo_dir} with {num_files:,} files")
            if num_files > 1000:
                lines.append(
                    "Warning: For large repos, consider using --subtree-only and .aiderignore"
                )
                lines.append(f"See: {urls.large_repos}")
        else:
            lines.append("Git repo: none")

        # Repo-map
        if self.repo_map:
            map_tokens = self.repo_map.max_map_tokens
            if map_tokens > 0:
                refresh = self.repo_map.refresh
                lines.append(f"Repo-map: using {map_tokens} tokens, {refresh} refresh")
                max_map_tokens = self.main_model.get_repo_map_tokens() * 2
                if map_tokens > max_map_tokens:
                    lines.append(
                        f"Warning: map-tokens > {max_map_tokens} is not recommended. Too much"
                        " irrelevant code can confuse LLMs."
                    )
            else:
                lines.append("Repo-map: disabled because map_tokens == 0")
        else:
            lines.append("Repo-map: disabled")

        if self.mcp_tools:
            mcp_servers = []
            for server_name, server_tools in self.mcp_tools:
                mcp_servers.append(server_name)
            lines.append(f"MCP servers configured: {', '.join(mcp_servers)}")

        for fname in self.abs_read_only_stubs_fnames:
            rel_fname = self.get_rel_fname(fname)
            lines.append(f"Added {rel_fname} to the chat (read-only stub).")

        if self.done_messages:
            lines.append("Restored previous conversation history.")

        if self.io.multiline_mode and not self.args.tui:
            lines.append("Multiline mode: Enabled. Enter inserts newline, Alt-Enter submits text")

        return lines

    def show_announcements(self):
        bold = True
        for line in self.get_announcements():
            self.io.tool_output(line, bold=bold)
            bold = False

    def setup_lint_cmds(self, lint_cmds):
        if not lint_cmds:
            return
        for lang, cmd in lint_cmds.items():
            self.linter.set_linter(lang, cmd)

    def add_rel_fname(self, rel_fname):
        self.abs_fnames.add(self.abs_root_path(rel_fname))
        self.check_added_files()

    def drop_rel_fname(self, fname):
        abs_fname = self.abs_root_path(fname)
        if abs_fname in self.abs_fnames:
            self.abs_fnames.remove(abs_fname)
            return True

    def abs_root_path(self, path):
        key = path
        if key in self.abs_root_path_cache:
            return self.abs_root_path_cache[key]

        res = Path(self.root) / path
        res = utils.safe_abs_path(res)
        self.abs_root_path_cache[key] = res
        return res

    fences = all_fences
    fence = fences[0]

    def show_pretty(self):
        if not self.pretty:
            return False

        # only show pretty output if fences are the normal triple-backtick
        if self.fence[0][0] != "`":
            return False

        return True

    def get_abs_fnames_content(self):
        # Sort files by last modified time (earliest first, latest last)
        sorted_fnames = sorted(
            list(filter(lambda f: os.path.exists(f), self.abs_fnames)),
            key=lambda fname: os.path.getmtime(fname),
        )

        for fname in sorted_fnames:
            content = self.io.read_text(fname)

            if content is None:
                relative_fname = self.get_rel_fname(fname)
                self.io.tool_warning(f"Dropping {relative_fname} from the chat.")
                self.abs_fnames.remove(fname)
            else:
                yield fname, content

    def choose_fence(self):
        all_content = ""
        for _fname, content in self.get_abs_fnames_content():
            all_content += content + "\n"
        for _fname in self.abs_read_only_fnames:
            content = self.io.read_text(_fname)
            if content is not None:
                all_content += content + "\n"
        for _fname in self.abs_read_only_stubs_fnames:
            content = self.io.read_text(_fname)
            if content is not None:
                all_content += content + "\n"

        lines = all_content.splitlines()
        good = False
        for fence_open, fence_close in self.fences:
            if any(line.startswith(fence_open) or line.startswith(fence_close) for line in lines):
                continue
            good = True
            break

        if good:
            self.fence = (fence_open, fence_close)
        else:
            self.fence = self.fences[0]
            self.io.tool_warning(
                "Unable to find a fencing strategy! Falling back to:"
                f" {self.fence[0]}...{self.fence[1]}"
            )

        return

    def get_files_content(self, fnames=None):
        if not fnames:
            fnames = self.abs_fnames

        # If there are files, return a dictionary with chat_files and edit_files
        if fnames:
            # Get current time for comparison
            current_time = time.time()
            lookback = current_time - 30

            # Get file modification times and sort by most recent first
            file_times = []
            for fname in fnames:
                try:
                    if os.path.exists(fname):
                        mtime = os.path.getmtime(fname)
                        file_times.append((fname, mtime))
                except OSError:
                    # Skip files that can't be accessed
                    continue

            # Sort by modification time (most recent first)
            file_times.sort(key=lambda x: x[1], reverse=True)

            # Determine which files go to edit_files
            edit_files = set()
            if file_times:
                # Always include the most recently edited file
                most_recent_file, most_recent_time = file_times[0]
                edit_files.add(most_recent_file)

                # Include any files edited within the last minute
                for fname, mtime in file_times:
                    if mtime >= lookback:
                        edit_files.add(fname)

            # Build content for chat_files and edit_files
            chat_files_prompt = ""
            edit_files_prompt = ""
            chat_file_names = set()
            edit_file_names = set()

            for fname, content in self.get_abs_fnames_content():
                if not is_image_file(fname):
                    relative_fname = self.get_rel_fname(fname)
                    file_prompt = "\n"
                    file_prompt += relative_fname
                    file_prompt += f"\n{self.fence[0]}\n"

                    # Apply context management if enabled for large files
                    if self.context_management_enabled:
                        # Calculate tokens for this file
                        file_tokens = self.main_model.token_count(content)

                        if file_tokens > self.large_file_token_threshold:
                            # Instead of truncating, show the file's definitions/structure
                            file_stub = RepoMap.get_file_stub(fname, self.io)

                            # Add message about showing definitions instead of full content
                            self.io.tool_output(
                                f"⚠️ '{relative_fname}' is very large ({file_tokens} tokens). "
                                "Use /context-management to toggle truncation off if needed."
                            )

                            # Add a message in the content itself so the model knows it's truncated
                            truncation_note = (
                                f"\n... [File content truncated due to size ({file_tokens} tokens)."
                                " Showing structure/definitions only.] ...\n\n"
                            )
                            file_prompt += truncation_note + file_stub
                        else:
                            file_prompt += content
                    else:
                        file_prompt += content

                    file_prompt += f"{self.fence[1]}\n"

                    # Add to appropriate prompt based on edit time
                    if fname in edit_files:
                        edit_files_prompt += file_prompt
                        edit_file_names.add(relative_fname)
                    else:
                        chat_files_prompt += file_prompt
                        chat_file_names.add(relative_fname)

            return {
                "chat_files": chat_files_prompt,
                "edit_files": edit_files_prompt,
                "chat_file_names": chat_file_names,
                "edit_file_names": edit_file_names,
            }
        else:
            # Return empty dictionary when no files
            return {
                "chat_files": "",
                "edit_files": "",
                "chat_file_names": set(),
                "edit_file_names": set(),
            }

    def get_read_only_files_content(self):
        prompt = ""
        # Sort read-only files by last modified time (earliest first, latest last)
        sorted_fnames = sorted(
            list(filter(lambda f: os.path.exists(f), self.abs_read_only_fnames)),
            key=lambda fname: os.path.getmtime(fname),
        )

        # Handle regular read-only files
        for fname in sorted_fnames:
            content = self.io.read_text(fname)
            if content is not None and not is_image_file(fname):
                relative_fname = self.get_rel_fname(fname)
                prompt += "\n"
                prompt += relative_fname
                prompt += f"\n{self.fence[0]}\n"

                # Apply context management if enabled for large files (same as get_files_content)
                if self.context_management_enabled:
                    # Calculate tokens for this file
                    file_tokens = self.main_model.token_count(content)

                    if file_tokens > self.large_file_token_threshold:
                        # Instead of truncating, show the file's definitions/structure
                        file_stub = RepoMap.get_file_stub(fname, self.io)

                        # Add message about showing definitions instead of full content
                        self.io.tool_output(
                            f"⚠️ '{relative_fname}' is very large ({file_tokens} tokens). "
                            "Use /context-management to toggle truncation off if needed."
                        )

                        # Add a message in the content itself so the model knows it's truncated
                        truncation_note = (
                            f"\n... [File content truncated due to size ({file_tokens} tokens)."
                            " Showing structure/definitions only.] ...\n\n"
                        )
                        prompt += truncation_note + file_stub
                    else:
                        prompt += content
                else:
                    prompt += content

                prompt += f"{self.fence[1]}\n"

        # Sort stub files by last modified time (earliest first, latest last)
        sorted_stub_fnames = sorted(
            list(filter(lambda f: os.path.exists(f), self.abs_read_only_stubs_fnames)),
            key=lambda fname: os.path.getmtime(fname),
        )

        # Handle stub files
        for fname in sorted_stub_fnames:
            if not is_image_file(fname):
                relative_fname = self.get_rel_fname(fname)
                prompt += "\n"
                prompt += f"{relative_fname} (stub)"
                prompt += f"\n{self.fence[0]}\n"
                stub = self.get_file_stub(fname)
                prompt += stub
                prompt += f"{self.fence[1]}\n"
        return prompt

    def get_cur_message_text(self):
        text = ""
        for msg in self.cur_messages:
            # For some models the content is None if the message
            # contains tool calls.
            content = msg["content"] or ""
            text += content + "\n"
        return text

    def get_ident_mentions(self, text):
        # Split the string on any character that is not alphanumeric
        # \W+ matches one or more non-word characters (equivalent to [^a-zA-Z0-9_]+)
        words = set(re.split(r"\W+", text))
        return words

    def get_ident_filename_matches(self, idents):
        all_fnames = defaultdict(set)
        for fname in self.get_all_relative_files():
            # Skip empty paths or just '.'
            if not fname or fname == ".":
                continue

            try:
                # Handle dotfiles properly
                path = Path(fname)
                base = path.stem.lower()  # Use stem instead of with_suffix("").name
                if len(base) >= 5:
                    all_fnames[base].add(fname)
            except ValueError:
                # Skip paths that can't be processed
                continue

        matches = set()
        for ident in idents:
            if len(ident) < 5:
                continue
            matches.update(all_fnames[ident.lower()])

        return matches

    def get_repo_map(self, force_refresh=False):
        if not self.repo_map or not self.repo:
            return

        self.io.update_spinner("Updating repo map")

        cur_msg_text = self.get_cur_message_text()
        try:
            staged_files_hash = hash(
                str([item.a_path for item in self.repo.repo.index.diff("HEAD")])
            )
        except ANY_GIT_ERROR as err:
            # Handle git errors gracefully - use a fallback hash
            if self.verbose:
                self.io.tool_warning(f"Git error while checking staged files for repo map: {err}")
            staged_files_hash = hash(str(time.time()))  # Use timestamp as fallback

        read_only_count = len(set(self.abs_read_only_fnames)) + len(
            set(self.abs_read_only_stubs_fnames)
        )
        self.data_cache["repo"]["mentioned_idents"] = self.get_ident_mentions(cur_msg_text)

        if (
            staged_files_hash != self.data_cache["repo"]["last_key"]
            or read_only_count != self.data_cache["repo"]["read_only_count"]
        ):
            self.data_cache["repo"]["last_key"] = staged_files_hash
            mentioned_idents = self.data_cache["repo"]["mentioned_idents"]
            mentioned_fnames = self.get_file_mentions(cur_msg_text)
            mentioned_fnames.update(self.get_ident_filename_matches(mentioned_idents))

            all_abs_files = set(self.get_all_abs_files())

            # Exclude metadata/docs from repo map inputs to reduce parsing overhead
            def _include_in_map(abs_path):
                try:
                    rel = self.get_rel_fname(abs_path)
                except Exception:
                    rel = str(abs_path)
                parts = Path(rel).parts
                if ".meta" in parts or ".docs" in parts:
                    return False
                if ".min." in parts[-1]:
                    return False
                if self.repo.ignored_file(abs_path):
                    return False
                return True

            all_abs_files = {p for p in all_abs_files if _include_in_map(p)}
            repo_abs_read_only_fnames = set(self.abs_read_only_fnames) & all_abs_files
            repo_abs_read_only_stubs_fnames = set(self.abs_read_only_stubs_fnames) & all_abs_files
            chat_files = (
                set(self.abs_fnames) | repo_abs_read_only_fnames | repo_abs_read_only_stubs_fnames
            )
            other_files = all_abs_files - chat_files

            self.data_cache["repo"].update(
                {
                    "chat_files": chat_files,
                    "other_files": other_files,
                    "mentioned_fnames": mentioned_fnames,
                    "all_abs_files": all_abs_files,
                    "read_only_count": len(set(self.abs_read_only_fnames)) + len(
                        set(self.abs_read_only_stubs_fnames)
                    ),
                }
            )

        repo_content = self.repo_map.get_repo_map(
            self.data_cache["repo"]["chat_files"],
            self.data_cache["repo"]["other_files"],
            mentioned_fnames=self.data_cache["repo"]["mentioned_fnames"],
            mentioned_idents=self.data_cache["repo"]["mentioned_idents"],
            force_refresh=force_refresh,
        )

        # fall back to global repo map if files in chat are disjoint from rest of repo
        if not repo_content:
            repo_content = self.repo_map.get_repo_map(
                set(),
                self.data_cache["repo"]["all_abs_files"],
                mentioned_fnames=self.data_cache["repo"]["mentioned_fnames"],
                mentioned_idents=self.data_cache["repo"]["mentioned_idents"],
            )

        # fall back to completely unhinted repo
        if not repo_content:
            repo_content = self.repo_map.get_repo_map(
                set(),
                self.data_cache["repo"]["all_abs_files"],
            )

        self.io.update_spinner(self.io.last_spinner_text)
        return repo_content

    def get_repo_messages(self):
        repo_messages = []
        repo_content = self.get_repo_map()
        if repo_content:
            repo_messages += [
                dict(role="user", content=repo_content),
                dict(
                    role="assistant",
                    content="Ok, I won't try and edit those files without asking first.",
                ),
            ]
        return repo_messages

    def get_readonly_files_messages(self):
        readonly_messages = []

        # Handle non-image files
        read_only_content = self.get_read_only_files_content()
        if read_only_content:
            readonly_messages += [
                dict(
                    role="user", content=self.gpt_prompts.read_only_files_prefix + read_only_content
                ),
                dict(
                    role="assistant",
                    content="Ok, I will use these files as references.",
                ),
            ]

        # Handle image files
        images_message = self.get_images_message(
            list(self.abs_read_only_fnames) + list(self.abs_read_only_stubs_fnames)
        )
        if images_message is not None:
            readonly_messages += [
                images_message,
                dict(role="assistant", content="Ok, I will use these images as references."),
            ]

        return readonly_messages

    def get_chat_files_messages(self):
        chat_files_messages = []
        edit_files_messages = []
        chat_file_names = set()
        edit_file_names = set()

        if self.abs_fnames:
            files_content_result = self.get_files_content()

            # Get content and file names from dictionary
            chat_files_content = files_content_result.get("chat_files", "")
            edit_files_content = files_content_result.get("edit_files", "")
            chat_file_names = files_content_result.get("chat_file_names", set())
            edit_file_names = files_content_result.get("edit_file_names", set())

            files_reply = self.gpt_prompts.files_content_assistant_reply

            if chat_files_content:
                chat_files_messages += [
                    dict(
                        role="user",
                        content=self.gpt_prompts.files_content_prefix + chat_files_content,
                    ),
                    dict(role="assistant", content=files_reply),
                ]

            if edit_files_content:
                edit_files_messages += [
                    dict(
                        role="user",
                        content=self.gpt_prompts.files_content_prefix + edit_files_content,
                    ),
                    dict(role="assistant", content=files_reply),
                ]
        elif self.gpt_prompts.files_no_full_files_with_repo_map:
            files_content = self.gpt_prompts.files_no_full_files_with_repo_map
            files_reply = self.gpt_prompts.files_no_full_files_with_repo_map_reply

            if files_content:
                chat_files_messages += [
                    dict(role="user", content=files_content),
                    dict(role="assistant", content=files_reply),
                ]
        else:
            files_content = self.gpt_prompts.files_no_full_files
            files_reply = "Ok."

            if files_content:
                chat_files_messages += [
                    dict(role="user", content=files_content),
                    dict(role="assistant", content=files_reply),
                ]

        images_message = self.get_images_message(self.abs_fnames)
        if images_message is not None:
            chat_files_messages += [
                images_message,
                dict(role="assistant", content="Ok."),
            ]

        return {
            "chat_files": chat_files_messages,
            "edit_files": edit_files_messages,
            "chat_file_names": chat_file_names,
            "edit_file_names": edit_file_names,
        }

    def get_images_message(self, fnames):
        supports_images = self.main_model.info.get("supports_vision")
        supports_pdfs = self.main_model.info.get("supports_pdf_input") or self.main_model.info.get(
            "max_pdf_size_mb"
        )

        # https://github.com/BerriAI/litellm/pull/6928
        supports_pdfs = supports_pdfs or "claude-3-5-sonnet-20241022" in self.main_model.name

        if not (supports_images or supports_pdfs):
            return None

        image_messages = []
        for fname in fnames:
            if not is_image_file(fname):
                continue

            mime_type, _ = mimetypes.guess_type(fname)
            if not mime_type:
                continue

            with open(fname, "rb") as image_file:
                encoded_string = base64.b64encode(image_file.read()).decode("utf-8")
            image_url = f"data:{mime_type};base64,{encoded_string}"
            rel_fname = self.get_rel_fname(fname)

            if mime_type.startswith("image/") and supports_images:
                image_messages += [
                    {"type": "text", "text": f"Image file: {rel_fname}"},
                    {"type": "image_url", "image_url": {"url": image_url, "detail": "high"}},
                ]
            elif mime_type == "application/pdf" and supports_pdfs:
                image_messages += [
                    {"type": "text", "text": f"PDF file: {rel_fname}"},
                    {"type": "image_url", "image_url": image_url},
                ]

        if not image_messages:
            return None

        return {"role": "user", "content": image_messages}

    async def run_stream(self, user_message):
        self.io.user_input(user_message)
        self.init_before_message()
        async for chunk in self.send_message(user_message):
            yield chunk

    def init_before_message(self):
        self.aider_edited_files = set()
        self.reflected_message = None
        self.num_reflections = 0
        self.lint_outcome = None
        self.test_outcome = None
        self.shell_commands = []
        self.message_cost = 0

        if self.repo:
            self.commit_before_message.append(self.repo.get_head_commit_sha())

    async def run(self, with_message=None, preproc=True):
        # Wait for confirmation to finish if in progress
        if not self.io.confirmation_in_progress_event.is_set():
            await self.io.confirmation_in_progress_event.wait()

        if self.linear_output:
            return await self._run_linear(with_message, preproc)

        if self.io.prompt_session:
            with patch_stdout(raw=True):
                return await self._run_parallel(with_message, preproc)
        else:
            return await self._run_parallel(with_message, preproc)

    async def _run_linear(self, with_message=None, preproc=True):
        try:
            if with_message:
                self.io.user_input(with_message)
                await self.run_one(with_message, preproc)
                return self.partial_response_content

            user_message = None
            await self.io.stop_task_streams()

            while True:
                try:
                    # Wait for commands to finish
                    if not self.commands.cmd_running_event.is_set():
                        await self.commands.cmd_running_event.wait()
                        continue

                    if not self.suppress_announcements_for_next_prompt:
                        self.show_announcements()
                    self.suppress_announcements_for_next_prompt = True

                    await self.io.recreate_input()
                    await self.io.input_task
                    user_message = self.io.input_task.result()

                    if self.args and not self.args.tui:
                        self.io.tool_output("Processing...\n")

                    self.io.output_task = asyncio.create_task(self.generate(user_message, preproc))

                    await self.io.output_task

                    if self.args and not self.args.tui:
                        self.io.tool_output("Finished.")

                    self.io.ring_bell()
                    user_message = None
                    await self.auto_save_session()

                except KeyboardInterrupt:
                    self.io.set_placeholder("")
                    self.io.stop_spinner()
                    self.keyboard_interrupt()
                    await self.io.stop_task_streams()
                except (asyncio.CancelledError, IndexError):
                    pass

        except EOFError:
            return
        finally:
            await self.io.stop_task_streams()

    async def _run_parallel(self, with_message=None, preproc=True):
        try:
            if with_message:
                self.io.user_input(with_message)
                await self.run_one(with_message, preproc)
                return self.partial_response_content

            # Initialize state for task coordination
            self.input_running = True
            self.output_running = True
            self.user_message = ""

            # Cancel any existing tasks
            await self.io.stop_task_streams()

            # Start the input and output tasks
            input_task = asyncio.create_task(self.input_task(preproc))
            output_task = asyncio.create_task(self.output_task(preproc))

            try:
                # Wait for both tasks to complete or for one to raise an exception
                done, pending = await asyncio.wait(
                    [input_task, output_task], return_when=asyncio.FIRST_EXCEPTION
                )

                # Check for exceptions
                for task in done:
                    if task.exception():
                        raise task.exception()

            except (SwitchCoder, SystemExit):
                # Re-raise SwitchCoder to be handled by outer try block
                raise
            except KeyboardInterrupt:
                # Handle keyboard interrupt gracefully
                self.io.set_placeholder("")
                self.io.stop_spinner()
                self.keyboard_interrupt()
            finally:
                # Signal tasks to stop
                self.input_running = False
                self.output_running = False

                # Cancel tasks
                input_task.cancel()
                output_task.cancel()

                # Wait for tasks to finish
                try:
                    await asyncio.gather(input_task, output_task, return_exceptions=True)
                except (asyncio.CancelledError, KeyboardInterrupt):
                    pass

                # Ensure IO tasks are properly cancelled
                await self.io.stop_task_streams()

            await self.auto_save_session()
        except EOFError:
            return
        finally:
            await self.io.stop_task_streams()

    async def input_task(self, preproc):
        """
        Handles input creation/recreation and user message processing.
        This task manages the input loop and coordinates with output_task.
        """
        while self.input_running:
            try:
                # Wait for commands to finish
                if not self.commands.cmd_running_event.is_set():
                    await self.commands.cmd_running_event.wait()
                    continue

                # Wait for input task completion
                if self.io.input_task and self.io.input_task.done():
                    try:
                        user_message = self.io.input_task.result()

                        # Defer to confirmation handler to fix Windows event loop race.
                        if not self.io.confirmation_in_progress_event.is_set():
                            pass
                        # Set user message for output task
                        elif not self.io.acknowledge_confirmation():
                            if user_message:
                                self.user_message = user_message
                                await self.auto_save_session()
                            else:
                                self.user_message = ""
                                await self.io.stop_task_streams()

                    except (asyncio.CancelledError, KeyboardInterrupt):
                        self.user_message = ""
                        await self.io.stop_task_streams()

                # Check if we should show announcements
                if (
                    self.io.confirmation_in_progress_event.is_set()
                    and not self.user_message
                    and not coroutines.is_active(self.io.input_task)
                    and (not coroutines.is_active(self.io.output_task) or not self.io.placeholder)
                ):
                    if not self.suppress_announcements_for_next_prompt:
                        self.show_announcements()
                    self.suppress_announcements_for_next_prompt = True

                    # Stop spinner before showing announcements or getting input
                    self.io.stop_spinner()
                    self.copy_context()

                # Check if we should recreate input
                if not coroutines.is_active(self.io.input_task):
                    self.io.ring_bell()
                    await self.io.recreate_input()

                await asyncio.sleep(0.1)  # Small yield to prevent tight loop

            except KeyboardInterrupt:
                self.io.set_placeholder("")
                self.keyboard_interrupt()
                await self.io.stop_task_streams()
            except (SwitchCoder, SystemExit):
                raise
            except Exception as e:
                if self.verbose or self.args.debug:
                    print(e)

    async def output_task(self, preproc):
        """
        Handles output task generation and monitoring.
        This task manages the output loop and coordinates with input_task.
        """
        while self.output_running:
            try:
                # Wait for commands to finish
                if not self.commands.cmd_running_event.is_set():
                    await self.commands.cmd_running_event.wait()
                    continue

                # Check if we have a user message to process
                if self.user_message and not self.io.get_confirmation_acknowledgement():
                    user_message = self.user_message
                    self.user_message = ""

                    # Create output task for processing
                    self.io.output_task = asyncio.create_task(self.generate(user_message, preproc))

                    # Start spinner for output task
                    self.io.start_spinner("Processing...")
                    await self.io.recreate_input()

                # Monitor output task
                if self.io.output_task:
                    if self.io.output_task.done():
                        exception = self.io.output_task.exception()
                        if exception:
                            if isinstance(exception, SwitchCoder):
                                await self.io.output_task
                                raise exception

                            self.io.tool_error(f"Error during generation: {exception}")
                            if self.verbose:
                                traceback.print_exception(
                                    type(exception), exception, exception.__traceback__
                                )

                        # Stop spinner when processing task completes
                        self.io.stop_spinner()

                        # And stop monitoring the output task
                        await self.io.stop_output_task()

                await self.auto_save_session()
                await asyncio.sleep(0.1)  # Small yield to prevent tight loop

            except KeyboardInterrupt:
                self.io.stop_spinner()
                self.keyboard_interrupt()
                await self.io.stop_task_streams()
            except (SwitchCoder, SystemExit):
                raise
            except Exception as e:
                if self.verbose or self.args.debug:
                    print(e)

    async def generate(self, user_message, preproc):
        await asyncio.sleep(0.1)

        try:
            if not self.enable_context_compaction:
                self.compact_context_completed = False
                await self.compact_context_if_needed()
                self.compact_context_completed = True

            self.run_one_completed = False
            await self.run_one(user_message, preproc)
            self.show_undo_hint()
        except asyncio.CancelledError:
            # Don't show undo hint if cancelled
            raise
        finally:
            self.run_one_completed = True
            self.compact_context_completed = True
            self.io.stop_spinner()

    def copy_context(self):
        if self.auto_copy_context:
            self.commands.cmd_copy_context()

    async def get_input(self):
        inchat_files = self.get_inchat_relative_files()
        all_read_only_fnames = self.abs_read_only_fnames | self.abs_read_only_stubs_fnames
        all_read_only_files = [self.get_rel_fname(fname) for fname in all_read_only_fnames]
        all_files = sorted(set(inchat_files + all_read_only_files))
        edit_format = "" if self.edit_format == self.main_model.edit_format else self.edit_format

        return await self.io.get_input(
            self.root,
            all_files,
            self.get_addable_relative_files(),
            self.commands,
            abs_read_only_fnames=self.abs_read_only_fnames,
            abs_read_only_stubs_fnames=self.abs_read_only_stubs_fnames,
            edit_format=edit_format,
        )

    async def preproc_user_input(self, inp):
        if not inp:
            return

        # Strip whitespace from beginning and end
        inp = inp.strip()

        if self.commands.is_command(inp):
            if inp[0] in "!":
                inp = f"/run {inp[1:]}"

            if self.commands.is_run_command(inp):
                self.commands.cmd_running_event.clear()  # Command is running

            return await self.commands.run(inp)

        await self.check_for_file_mentions(inp)
        inp = await self.check_for_urls(inp)

        return inp

    async def run_one(self, user_message, preproc):
        self.init_before_message()

        if preproc:
            message = await self.preproc_user_input(user_message)
        else:
            message = user_message

        if self.commands.is_command(user_message) and not self.commands.is_test_command(
            user_message
        ):
            return

        while True:
            self.reflected_message = None
            self.tool_reflection = False

            async for _ in self.send_message(message):
                pass

            if not self.reflected_message:
                break

            if self.num_reflections >= self.max_reflections:
                self.io.tool_warning(f"Only {self.max_reflections} reflections allowed, stopping.")
                return

            self.num_reflections += 1

            if self.tool_reflection:
                self.num_reflections -= 1

            if self.reflected_message is True:
                message = None
            else:
                message = self.reflected_message

            if self.enable_context_compaction:
                await self.compact_context_if_needed()

    async def check_and_open_urls(self, exc, friendly_msg=None):
        """Check exception for URLs, offer to open in a browser, with user-friendly error msgs."""
        text = str(exc)

        if friendly_msg:
            self.io.tool_warning(text)
            self.io.tool_error(f"{friendly_msg}")
        else:
            self.io.tool_error(text)

        # Exclude double quotes from the matched URL characters
        url_pattern = re.compile(r'(https?://[^\s/$.?#].[^\s"]*)')
        # Use set to remove duplicates
        urls = list(set(url_pattern.findall(text)))
        for url in urls:
            url = url.rstrip(".',\"}")  # Added } to the characters to strip
            await self.io.offer_url(url)
        return urls

    async def check_for_urls(self, inp: str) -> List[str]:
        """Check input for URLs and offer to add them to the chat."""
        if not self.detect_urls or self.args.disable_scraping:
            return inp

        # Exclude double quotes from the matched URL characters
        url_pattern = re.compile(r'(https?://[^\s/$.?#].[^\s"]*[^\s,.])')
        # Use set to remove duplicates
        urls = list(set(url_pattern.findall(inp)))
        group = ConfirmGroup(urls)
        for url in urls:
            if url not in self.rejected_urls:
                url = url.rstrip(".',\"")
                if await self.io.confirm_ask(
                    "Add URL to the chat?",
                    subject=url,
                    group=group,
                    allow_never=True,
                    explicit_yes_required=self.args.yes_always_commands,
                ):
                    inp += "\n\n"
                    inp += await self.commands.do_run("web", url, return_content=True)
                else:
                    self.rejected_urls.add(url)

        return inp

    def keyboard_interrupt(self):
        # Ensure cursor is visible on exit
        Console().show_cursor(True)

        self.io.tool_warning("\n\n^C KeyboardInterrupt")

        self.last_keyboard_interrupt = time.time()

    def summarize_start(self):
        if not self.summarizer.check_max_tokens(self.done_messages):
            return

        self.summarize_end()

        if self.verbose:
            self.io.tool_output("Starting to summarize chat history.")

        self.summarizer_thread = threading.Thread(target=self.summarize_worker)
        self.summarizer_thread.start()

    def summarize_worker(self):
        self.summarizing_messages = list(self.done_messages)
        try:
            self.summarized_done_messages = asyncio.run(
                self.summarizer.summarize(self.summarizing_messages)
            )
        except ValueError as err:
            self.io.tool_warning(err.args[0])
            self.summarized_done_messages = self.summarizing_messages

        if self.verbose:
            self.io.tool_output("Finished summarizing chat history.")

    def summarize_end(self):
        if self.summarizer_thread is None:
            return

        self.summarizer_thread.join()
        self.summarizer_thread = None

        if self.summarizing_messages == self.done_messages:
            self.done_messages = self.summarized_done_messages
        self.summarizing_messages = None
        self.summarized_done_messages = []

    async def compact_context_if_needed(self):
        if not self.enable_context_compaction:
            self.summarize_start()
            return

        # Check if combined messages exceed the token limit,
        # Exclude first cur_message since that's the user's initial input
        done_tokens = self.summarizer.count_tokens(self.done_messages)
        cur_tokens = self.summarizer.count_tokens(self.cur_messages[1:])
        combined_tokens = done_tokens + cur_tokens

        if combined_tokens < self.context_compaction_max_tokens:
            return

        self.io.tool_output("Compacting chat history to make room for new messages...")
        self.io.update_spinner("Compacting...")

        try:
            # Check if done_messages alone exceed the limit
            if done_tokens > self.context_compaction_max_tokens or done_tokens > cur_tokens:
                # Create a summary of the done_messages
                summary_text = await self.summarizer.summarize_all_as_text(
                    self.done_messages,
                    self.gpt_prompts.compaction_prompt,
                    self.context_compaction_summary_tokens,
                )

                if not summary_text:
                    raise ValueError("Summarization returned an empty result.")

                # Replace old messages with the summary
                self.done_messages = [
                    {
                        "role": "user",
                        "content": summary_text,
                    },
                    {
                        "role": "assistant",
                        "content": (
                            "Ok, I will use this summary as the context for our conversation going"
                            " forward."
                        ),
                    },
                ]

            # Check if cur_messages alone exceed the limit (after potentially compacting done_messages)
            if cur_tokens > self.context_compaction_max_tokens or cur_tokens > done_tokens:
                # Create a summary of the cur_messages
                cur_summary_text = await self.summarizer.summarize_all_as_text(
                    self.cur_messages,
                    self.gpt_prompts.compaction_prompt,
                    self.context_compaction_summary_tokens,
                )

                if not cur_summary_text:
                    raise ValueError("Summarization of current messages returned an empty result.")

                # Replace current messages with the summary
                self.cur_messages = [
                    self.cur_messages[0],
                    {
                        "role": "assistant",
                        "content": "Ok. I am awaiting your summary of our goals to proceed.",
                    },
                    {
                        "role": "user",
                        "content": f"Here is a summary of our current goals:\n{cur_summary_text}",
                    },
                    {
                        "role": "assistant",
                        "content": (
                            "Ok, I will use this summary and proceed with our task."
                            " I will first apply any changes in the summary and then"
                            " continue exploration as necessary."
                        ),
                    },
                ]

            self.io.tool_output("...chat history compacted.")
            self.io.update_spinner(self.io.last_spinner_text)
        except Exception as e:
            self.io.tool_warning(f"Context compaction failed: {e}")
            self.io.tool_warning("Proceeding with full history for now.")
            self.summarize_start()
            return

    def move_back_cur_messages(self, message):
        self.done_messages += self.cur_messages

        # TODO check for impact on image messages
        if message:
            self.done_messages += [
                dict(role="user", content=message),
                dict(role="assistant", content="Ok."),
            ]
        self.cur_messages = []

    def normalize_language(self, lang_code):
        """
        Convert a locale code such as ``en_US`` or ``fr`` into a readable
        language name (e.g. ``English`` or ``French``).  If Babel is
        available it is used for reliable conversion; otherwise a small
        built-in fallback map handles common languages.
        """
        if not lang_code:
            return None

        if lang_code.upper() in ("C", "POSIX"):
            return None

        # Probably already a language name
        if (
            len(lang_code) > 3
            and "_" not in lang_code
            and "-" not in lang_code
            and lang_code[0].isupper()
        ):
            return lang_code

        # Preferred: Babel
        if Locale is not None:
            try:
                loc = Locale.parse(lang_code.replace("-", "_"))
                return loc.get_display_name("en").capitalize()
            except Exception:
                pass  # Fall back to manual mapping

        # Simple fallback for common languages
        fallback = {
            "en": "English",
            "fr": "French",
            "es": "Spanish",
            "de": "German",
            "it": "Italian",
            "pt": "Portuguese",
            "zh": "Chinese",
            "ja": "Japanese",
            "ko": "Korean",
            "ru": "Russian",
        }
        primary_lang_code = lang_code.replace("-", "_").split("_")[0].lower()
        return fallback.get(primary_lang_code, lang_code)

    def get_user_language(self):
        """
        Detect the user's language preference and return a human-readable
        language name such as ``English``. Detection order:

        1. ``self.chat_language`` if explicitly set
        2. ``locale.getlocale()``
        3. ``LANG`` / ``LANGUAGE`` / ``LC_ALL`` / ``LC_MESSAGES`` environment variables
        """

        # Explicit override
        if self.chat_language:
            return self.normalize_language(self.chat_language)

        # System locale
        try:
            lang = locale.getlocale()[0]
            if lang:
                lang = self.normalize_language(lang)
            if lang:
                return lang
        except Exception:
            pass

        # Environment variables
        for env_var in ("LANG", "LANGUAGE", "LC_ALL", "LC_MESSAGES"):
            lang = os.environ.get(env_var)
            if lang:
                lang = lang.split(".")[0]  # Strip encoding if present
                return self.normalize_language(lang)

        return None

    def get_platform_info(self):
        platform_text = ""
        try:
            platform_text = f"- Platform: {platform.platform()}\n"
        except KeyError:
            # Skip platform info if it can't be retrieved
            platform_text = "- Platform information unavailable\n"

        shell_var = "COMSPEC" if os.name == "nt" else "SHELL"
        shell_val = os.getenv(shell_var)
        platform_text += f"- Shell: {shell_var}={shell_val}\n"

        user_lang = self.get_user_language()
        if user_lang:
            platform_text += f"- Language: {user_lang}\n"

        dt = datetime.now().astimezone().strftime("%Y-%m-%d")
        platform_text += f"- Current date: {dt}\n"

        if self.repo:
            platform_text += "- The user is operating inside a git repository\n"

        if self.lint_cmds:
            if self.auto_lint:
                platform_text += (
                    "- The user's pre-commit runs these lint commands, don't suggest running"
                    " them:\n"
                )
            else:
                platform_text += "- The user prefers these lint commands:\n"
            for lang, cmd in self.lint_cmds.items():
                if lang is None:
                    platform_text += f"  - {cmd}\n"
                else:
                    platform_text += f"  - {lang}: {cmd}\n"

        if self.test_cmd:
            if self.auto_test:
                platform_text += (
                    "- The user's pre-commit runs this test command, don't suggest running them: "
                )
            else:
                platform_text += "- The user prefers this test command: "
            platform_text += self.test_cmd + "\n"

        return platform_text

    def fmt_system_prompt(self, prompt):
        final_reminders = []

        lazy_prompt = ""
        if self.main_model.lazy:
            lazy_prompt = self.gpt_prompts.lazy_prompt
            final_reminders.append(lazy_prompt)

        overeager_prompt = ""
        if self.main_model.overeager:
            overeager_prompt = self.gpt_prompts.overeager_prompt
            final_reminders.append(overeager_prompt)

        user_lang = self.get_user_language()
        if user_lang:
            final_reminders.append(f"Reply in {user_lang}.\n")

        platform_text = self.get_platform_info()

        if self.suggest_shell_commands:
            shell_cmd_prompt = self.gpt_prompts.shell_cmd_prompt.format(platform=platform_text)
            shell_cmd_reminder = self.gpt_prompts.shell_cmd_reminder.format(platform=platform_text)
            rename_with_shell = self.gpt_prompts.rename_with_shell
        else:
            shell_cmd_prompt = self.gpt_prompts.no_shell_cmd_prompt.format(platform=platform_text)
            shell_cmd_reminder = self.gpt_prompts.no_shell_cmd_reminder.format(
                platform=platform_text
            )
            rename_with_shell = ""

        if user_lang:  # user_lang is the result of self.get_user_language()
            language = user_lang
        else:
            # Default if no specific lang detected
            language = "the same language they are using"

        if self.fence[0] == "`" * 4:
            quad_backtick_reminder = (
                "\nIMPORTANT: Use *quadruple* backticks ```` as fences, not triple backticks!\n"
            )
        else:
            quad_backtick_reminder = ""

        if self.mcp_tools and len(self.mcp_tools) > 0:
            final_reminders.append(self.gpt_prompts.tool_prompt)

        final_reminders = "\n\n".join(final_reminders)

        prompt = prompt.format(
            fence=self.fence,
            quad_backtick_reminder=quad_backtick_reminder,
            final_reminders=final_reminders,
            platform=platform_text,
            shell_cmd_prompt=shell_cmd_prompt,
            rename_with_shell=rename_with_shell,
            shell_cmd_reminder=shell_cmd_reminder,
            go_ahead_tip=self.gpt_prompts.go_ahead_tip,
            language=language,
            lazy_prompt=lazy_prompt,
            overeager_prompt=overeager_prompt,
        )

        return prompt

    def format_chat_chunks(self):
        self.choose_fence()
        main_sys = self.fmt_system_prompt(self.gpt_prompts.main_system)
        if self.main_model.system_prompt_prefix:
            main_sys = self.main_model.system_prompt_prefix + "\n" + main_sys

        example_messages = []
        if self.main_model.examples_as_sys_msg:
            if self.gpt_prompts.example_messages:
                main_sys += "\n# Example conversations:\n\n"
            for msg in self.gpt_prompts.example_messages:
                role = msg["role"]
                content = self.fmt_system_prompt(msg["content"])
                main_sys += f"## {role.upper()}: {content}\n\n"
            main_sys = main_sys.strip()
        else:
            for msg in self.gpt_prompts.example_messages:
                example_messages.append(
                    dict(
                        role=msg["role"],
                        content=self.fmt_system_prompt(msg["content"]),
                    )
                )
            if self.gpt_prompts.example_messages:
                example_messages += [
                    dict(
                        role="user",
                        content=(
                            "I switched to a new code base. Please don't consider the above files"
                            " or try to edit them any longer."
                        ),
                    ),
                    dict(role="assistant", content="Ok."),
                ]

        if self.gpt_prompts.system_reminder:
            main_sys += "\n" + self.fmt_system_prompt(self.gpt_prompts.system_reminder)

        chunks = ChatChunks()

        if self.main_model.use_system_prompt:
            chunks.system = [
                dict(role="system", content=main_sys),
            ]
        else:
            chunks.system = [
                dict(role="user", content=main_sys),
                dict(role="assistant", content="Ok."),
            ]

        chunks.examples = example_messages

        self.summarize_end()
        chunks.done = self.done_messages

        chunks.repo = self.get_repo_messages()
        chunks.readonly_files = self.get_readonly_files_messages()

        # Handle the dictionary structure from get_chat_files_messages()
        chat_files_result = self.get_chat_files_messages()
        chunks.chat_files = chat_files_result.get("chat_files", [])
        chunks.edit_files = chat_files_result.get("edit_files", [])

        if self.gpt_prompts.system_reminder:
            reminder_message = [
                dict(
                    role="system", content=self.fmt_system_prompt(self.gpt_prompts.system_reminder)
                ),
            ]
        else:
            reminder_message = []

        chunks.cur = list(self.cur_messages)
        chunks.reminder = []

        # TODO review impact of token count on image messages
        messages_tokens = self.main_model.token_count(chunks.all_messages())
        reminder_tokens = self.main_model.token_count(reminder_message)
        cur_tokens = self.main_model.token_count(chunks.cur)

        if None not in (messages_tokens, reminder_tokens, cur_tokens):
            total_tokens = messages_tokens
            # Only add tokens for reminder and cur if they're not already included
            # in the messages_tokens calculation
            if not chunks.reminder:
                total_tokens += reminder_tokens
            if not chunks.cur:
                total_tokens += cur_tokens
        else:
            # add the reminder anyway
            total_tokens = 0

        if chunks.cur:
            final = chunks.cur[-1]
        else:
            final = None

        max_input_tokens = self.main_model.info.get("max_input_tokens") or 0
        # Add the reminder prompt if we still have room to include it.
        if (
            not max_input_tokens
            or total_tokens < max_input_tokens
            and self.gpt_prompts.system_reminder
        ):
            if self.main_model.reminder == "sys":
                chunks.reminder = reminder_message
            elif self.main_model.reminder == "user" and final and final["role"] == "user":
                # stuff it into the user message
                new_content = (
                    final["content"]
                    + "\n\n"
                    + self.fmt_system_prompt(self.gpt_prompts.system_reminder)
                )
                chunks.cur[-1] = dict(role=final["role"], content=new_content)

        return chunks

    def format_messages(self):
        chunks = self.format_chat_chunks()
        if self.add_cache_headers:
            chunks.add_cache_control_headers()

        return chunks

    def warm_cache(self, chunks):
        if not self.add_cache_headers:
            return
        if not self.num_cache_warming_pings:
            return
        if not self.ok_to_warm_cache:
            return

        delay = 5 * 60 - 5
        delay = float(os.environ.get("AIDER_CACHE_KEEPALIVE_DELAY", delay))
        self.next_cache_warm = time.time() + delay
        self.warming_pings_left = self.num_cache_warming_pings
        self.cache_warming_chunks = chunks

        if self.cache_warming_thread:
            return

        def warm_cache_worker():
            while self.ok_to_warm_cache:
                time.sleep(1)
                if self.warming_pings_left <= 0:
                    continue
                now = time.time()
                if now < self.next_cache_warm:
                    continue

                self.warming_pings_left -= 1
                self.next_cache_warm = time.time() + delay

                kwargs = dict(self.main_model.extra_params) or dict()
                kwargs["max_tokens"] = 1

                try:
                    completion = litellm.completion(
                        model=self.main_model.name,
                        messages=self.cache_warming_chunks.cacheable_messages(),
                        stream=False,
                        **kwargs,
                    )
                except Exception as err:
                    self.io.tool_warning(f"Cache warming error: {str(err)}")
                    continue

                cache_hit_tokens = getattr(
                    completion.usage, "prompt_cache_hit_tokens", 0
                ) or getattr(completion.usage, "cache_read_input_tokens", 0)

                if self.verbose:
                    self.io.tool_output(f"Warmed {format_tokens(cache_hit_tokens)} cached tokens.")

        self.cache_warming_thread = threading.Timer(0, warm_cache_worker)
        self.cache_warming_thread.daemon = True
        self.cache_warming_thread.start()

        return chunks

    async def check_tokens(self, messages):
        """Check if the messages will fit within the model's token limits."""
        input_tokens = self.main_model.token_count(messages)
        max_input_tokens = self.main_model.info.get("max_input_tokens") or 0

        if max_input_tokens and input_tokens >= max_input_tokens:
            self.io.tool_error(
                f"Your estimated chat context of {input_tokens:,} tokens exceeds the"
                f" {max_input_tokens:,} token limit for {self.main_model.name}!"
            )
            self.io.tool_output("To reduce the chat context:")
            self.io.tool_output("- Use /drop to remove unneeded files from the chat")
            self.io.tool_output("- Use /clear to clear the chat history")
            self.io.tool_output("- Break your code into smaller files")
            self.io.tool_output(
                "It's probably safe to try and send the request, most providers won't charge if"
                " the context limit is exceeded."
            )

            if not await self.io.confirm_ask("Try to proceed anyway?"):
                return False
        return True

    async def send_message(self, inp):
        # Notify IO that LLM processing is starting
        self.io.llm_started()

        if inp:
            self.cur_messages += [
                dict(role="user", content=inp),
            ]

        loop = asyncio.get_running_loop()
        chunks = await loop.run_in_executor(None, self.format_messages)
        messages = chunks.all_messages()

        if not await self.check_tokens(messages):
            return
        self.warm_cache(chunks)

        if self.verbose:
            utils.show_messages(messages, functions=self.functions)

        self.multi_response_content = ""
        if self.show_pretty():
            spinner_text = (
                f"Waiting for {self.main_model.name} • ${self.format_cost(self.total_cost)} session"
            )
            self.io.start_spinner(spinner_text)

            if self.stream:
                self.mdstream = True
            else:
                self.mdstream = None
        else:
            self.mdstream = None

        retry_delay = 0.125

        litellm_ex = LiteLLMExceptions()

        self.usage_report = None
        exhausted = False
        interrupted = False

        try:
            while True:
                try:
                    async for chunk in self.send(messages, tools=self.get_tool_list()):
                        yield chunk
                    break
                except litellm_ex.exceptions_tuple() as err:
                    ex_info = litellm_ex.get_ex_info(err)

                    if ex_info.name == "ContextWindowExceededError":
                        exhausted = True
                        break

                    should_retry = ex_info.retry
                    if should_retry:
                        retry_delay *= 2
                        if retry_delay > RETRY_TIMEOUT:
                            should_retry = False

                    if not should_retry:
                        self.mdstream = None
                        await self.check_and_open_urls(err, ex_info.description)
                        break

                    err_msg = str(err)
                    if ex_info.description:
                        self.io.tool_warning(err_msg)
                        self.io.tool_error(ex_info.description)
                    else:
                        self.io.tool_error(err_msg)

                    self.io.tool_output(f"Retrying in {retry_delay:.1f} seconds...")
                    await asyncio.sleep(retry_delay)
                    continue
                except KeyboardInterrupt:
                    interrupted = True
                    break
                except FinishReasonLength:
                    # We hit the output limit!
                    if not self.main_model.info.get("supports_assistant_prefill"):
                        exhausted = True
                        break

                    self.multi_response_content = self.get_multi_response_content_in_progress()

                    if messages[-1]["role"] == "assistant":
                        messages[-1]["content"] = self.multi_response_content
                    else:
                        messages.append(
                            dict(role="assistant", content=self.multi_response_content, prefix=True)
                        )
                except Exception as err:
                    self.mdstream = None
                    lines = traceback.format_exception(type(err), err, err.__traceback__)
                    self.io.tool_warning("".join(lines))
                    self.io.tool_error(str(err))
                    return
        finally:
            if self.mdstream:
                content_to_show = self.live_incremental_response(True)
                self.stream_wrapper(content_to_show, final=True)
            self.mdstream = None

            # Ensure any waiting spinner is stopped
            self.io.start_spinner("Processing Answer...")

            self.partial_response_content = self.get_multi_response_content_in_progress(True)
            self.remove_reasoning_content()
            self.multi_response_content = ""

        self.io.tool_output()
        self.show_usage_report()
        self.add_assistant_reply_to_cur_messages()

        if exhausted:
            if self.cur_messages and self.cur_messages[-1]["role"] == "user":
                self.cur_messages += [
                    dict(
                        role="assistant",
                        content="FinishReasonLength exception: you sent too many tokens",
                    ),
                ]

            await self.show_exhausted_error()
            self.num_exhausted_context_windows += 1
            return

        if self.partial_response_function_call:
            args = self.parse_partial_args()
            if args:
                content = args.get("explanation") or ""
            else:
                content = ""
        elif self.partial_response_content:
            content = self.partial_response_content
        else:
            content = ""

        if interrupted:
            if self.cur_messages and self.cur_messages[-1]["role"] == "user":
                self.cur_messages[-1]["content"] += "\n^C KeyboardInterrupt"
            else:
                self.cur_messages += [dict(role="user", content="^C KeyboardInterrupt")]
            self.cur_messages += [
                dict(role="assistant", content="I see that you interrupted my previous reply.")
            ]
            return

        edited = await self.apply_updates()

        if edited:
            self.aider_edited_files.update(edited)
            saved_message = await self.auto_commit(edited)

            if not saved_message and hasattr(self.gpt_prompts, "files_content_gpt_edits_no_repo"):
                saved_message = self.gpt_prompts.files_content_gpt_edits_no_repo

            self.move_back_cur_messages(saved_message)

        if not interrupted:
            add_rel_files_message = await self.check_for_file_mentions(content)
            if add_rel_files_message:
                if self.reflected_message:
                    self.reflected_message += "\n\n" + add_rel_files_message
                else:
                    self.reflected_message = add_rel_files_message
                return

            # Process any tools using MCP servers
            try:
                if self.partial_response_tool_calls:
                    tool_call_response, a, b = self.consolidate_chunks()
                    if await self.process_tool_calls(tool_call_response):
                        self.num_tool_calls += 1
                        self.reflected_message = True
                        return
            except Exception as e:
                self.io.tool_error(f"Error processing tool calls: {str(e)}")
                self.reflected_message = True
                return
                # Continue without tool processing

            self.num_tool_calls = 0

            try:
                if await self.reply_completed():
                    return
            except KeyboardInterrupt:
                interrupted = True

        if self.reflected_message:
            return

        if edited and self.auto_lint:
            lint_errors = self.lint_edited(edited)
            await self.auto_commit(edited, context="Ran the linter")
            self.lint_outcome = not lint_errors
            if lint_errors:
                ok = await self.io.confirm_ask("Attempt to fix lint errors?")
                if ok:
                    self.reflected_message = lint_errors
                    return

        shared_output = await self.run_shell_commands()
        if shared_output:
            self.cur_messages += [
                dict(role="user", content=shared_output),
                dict(role="assistant", content="Ok"),
            ]

        if edited and self.auto_test:
            test_errors = await self.commands.cmd_test(self.test_cmd)
            self.test_outcome = not test_errors
            if test_errors:
                ok = await self.io.confirm_ask("Attempt to fix test errors?")
                if ok:
                    self.reflected_message = test_errors
                    return

    async def process_tool_calls(self, tool_call_response):
        if tool_call_response is None:
            return False

        # Handle different response structures
        try:
            # Try to get tool calls from the standard OpenAI response format
            if hasattr(tool_call_response, "choices") and tool_call_response.choices:
                message = tool_call_response.choices[0].message
                if hasattr(message, "tool_calls") and message.tool_calls:
                    original_tool_calls = message.tool_calls
                else:
                    return False
            else:
                # Handle other response formats
                return False
        except (AttributeError, IndexError):
            return False

        if not original_tool_calls:
            return False

        # Expand any tool calls that have concatenated JSON in their arguments.
        # This is necessary because some models (like Gemini) will serialize
        # multiple tool calls in this way.
        expanded_tool_calls = []
        for tool_call in original_tool_calls:
            args_string = tool_call.function.arguments.strip()

            # If there are no arguments, or it's not a string that looks like it could
            # be concatenated JSON, just add it and continue.
            if not args_string or not (args_string.startswith("{") or args_string.startswith("[")):
                expanded_tool_calls.append(tool_call)
                continue

            json_chunks = utils.split_concatenated_json(args_string)

            # If it's just a single JSON object, there's nothing to expand.
            if len(json_chunks) <= 1:
                expanded_tool_calls.append(tool_call)
                continue

            # We have concatenated JSON, so expand it into multiple tool calls.
            for i, chunk in enumerate(json_chunks):
                if not chunk.strip():
                    continue

                # Create a new tool call for each JSON chunk, with a unique ID.
                new_function = tool_call.function.model_copy(update={"arguments": chunk})
                new_tool_call = tool_call.model_copy(
                    update={"id": f"{tool_call.id}-{i}", "function": new_function}
                )
                expanded_tool_calls.append(new_tool_call)

        # Collect all tool calls grouped by server
        server_tool_calls = self._gather_server_tool_calls(expanded_tool_calls)

        if server_tool_calls and self.num_tool_calls < self.max_tool_calls:
            self._print_tool_call_info(server_tool_calls)

            if await self.io.confirm_ask("Run tools?", group_response="Run MCP Tools"):
                tool_responses = await self._execute_tool_calls(server_tool_calls)

                # Add all tool responses
                for tool_response in tool_responses:
                    self.cur_messages.append(tool_response)

                return True
        elif self.num_tool_calls >= self.max_tool_calls:
            self.io.tool_warning(f"Only {self.max_tool_calls} tool calls allowed, stopping.")

        return False

    def _print_tool_call_info(self, server_tool_calls):
        """Print information about an MCP tool call."""
        # self.io.tool_output("Preparing to run MCP tools", bold=False)

        for server, tool_calls in server_tool_calls.items():
            for tool_call in tool_calls:
                if hasattr(self, "tool_registry") and self.tool_registry.get(
                    tool_call.function.name.lower(), None
                ):
                    self.tool_registry.get(tool_call.function.name.lower()).format_output(
                        coder=self, mcp_server=server, tool_response=tool_call
                    )
                else:
                    print_tool_response(coder=self, mcp_server=server, tool_response=tool_call)

    def _gather_server_tool_calls(self, tool_calls):
        """Collect all tool calls grouped by server.
        Args:
            tool_calls: List of tool calls from the LLM response

        Returns:
            dict: Dictionary mapping servers to their respective tool calls
        """
        if not self.mcp_tools or len(self.mcp_tools) == 0:
            return None

        server_tool_calls = {}
        tool_id_set = set()

        for tool_call in tool_calls:
            # LLM APIs sometimes return duplicates and that's annoying part 3
            if tool_call.get("id") in tool_id_set:
                continue

            tool_id_set.add(tool_call.get("id"))

            # Check if this tool_call matches any MCP tool
            for server_name, server_tools in self.mcp_tools:
                for tool in server_tools:
                    tool_name_from_schema = tool.get("function", {}).get("name")
                    if (
                        tool_name_from_schema
                        and tool_name_from_schema.lower() == tool_call.function.name.lower()
                    ):
                        # Find the McpServer instance that will be used for communication
                        for server in self.mcp_servers:
                            if server.name == server_name:
                                if server not in server_tool_calls:
                                    server_tool_calls[server] = []
                                server_tool_calls[server].append(tool_call)
                                break

        return server_tool_calls

    async def _execute_tool_calls(self, tool_calls):
        """Process tool calls from the response and execute them if they match MCP tools.
        Returns a list of tool response messages."""
        tool_responses = []

        # Define the coroutine to execute all tool calls for a single server
        async def _exec_server_tools(server, tool_calls_list):
            if isinstance(server, LocalServer):
                if hasattr(self, "_execute_local_tool_calls"):
                    return await self._execute_local_tool_calls(tool_calls_list)
                else:
                    # This coder doesn't support local tools, return errors for all calls
                    error_responses = []
                    for tool_call in tool_calls_list:
                        error_responses.append(
                            {
                                "role": "tool",
                                "tool_call_id": tool_call.id,
                                "content": (
                                    f"Coder does not support local tool: {tool_call.function.name}"
                                ),
                            }
                        )
                    return error_responses

            tool_responses = []
            try:
                # Connect to the server once
                session = await server.connect()
                tool_id_set = set()

                # Execute all tool calls for this server
                for tool_call in tool_calls_list:
                    # LLM APIs sometimes return duplicates and that's annoying part 4
                    if tool_call.id in tool_id_set:
                        continue

                    tool_id_set.add(tool_call.id)

                    try:
                        # Arguments can be a stream of JSON objects.
                        # We need to parse them and run a tool call for each.
                        args_string = tool_call.function.arguments.strip()
                        parsed_args_list = []
                        if args_string:
                            json_chunks = utils.split_concatenated_json(args_string)
                            for chunk in json_chunks:
                                try:
                                    parsed_args_list.append(json.loads(chunk))
                                except json.JSONDecodeError:
                                    self.io.tool_warning(
                                        "Could not parse JSON chunk for tool"
                                        f" {tool_call.function.name}: {chunk}"
                                    )
                                    continue

                        if not parsed_args_list and not args_string:
                            parsed_args_list.append({})  # For tool calls with no arguments

                        all_results_content = []
                        for args in parsed_args_list:
                            new_tool_call = tool_call.model_copy(deep=True)
                            new_tool_call.function.arguments = json.dumps(args)

                            call_result = await experimental_mcp_client.call_openai_tool(
                                session=session,
                                openai_tool=new_tool_call,
                            )

                            content_parts = []
                            if call_result.content:
                                for item in call_result.content:
                                    if hasattr(item, "resource"):  # EmbeddedResource
                                        resource = item.resource
                                        if hasattr(resource, "text"):  # TextResourceContents
                                            content_parts.append(resource.text)
                                        elif hasattr(resource, "blob"):  # BlobResourceContents
                                            try:
                                                decoded_blob = base64.b64decode(
                                                    resource.blob
                                                ).decode("utf-8")
                                                content_parts.append(decoded_blob)
                                            except (UnicodeDecodeError, TypeError):
                                                # Handle non-text blobs gracefully
                                                name = getattr(resource, "name", "unnamed")
                                                mime_type = getattr(
                                                    resource, "mimeType", "unknown mime type"
                                                )
                                                content_parts.append(
                                                    "[embedded binary resource:"
                                                    f" {name} ({mime_type})]"
                                                )
                                    elif hasattr(item, "text"):  # TextContent
                                        content_parts.append(item.text)

                            result_text = "".join(content_parts)
                            all_results_content.append(result_text)

                        tool_responses.append(
                            {
                                "role": "tool",
                                "tool_call_id": tool_call.id,
                                "content": "\n\n".join(all_results_content),
                            }
                        )

                    except Exception as e:
                        tool_error = f"Error executing tool call {tool_call.function.name}: \n{e}"
                        self.io.tool_warning(
                            f"Executing {tool_call.function.name} on {server.name} failed: \n "
                            f" Error: {e}\n"
                        )
                        tool_responses.append(
                            {"role": "tool", "tool_call_id": tool_call.id, "content": tool_error}
                        )
            except httpx.RemoteProtocolError as e:
                connection_error = f"Server {server.name} disconnected unexpectedly: {e}"
                self.io.tool_warning(connection_error)
                for tool_call in tool_calls_list:
                    tool_responses.append(
                        {"role": "tool", "tool_call_id": tool_call.id, "content": connection_error}
                    )
            except Exception as e:
                connection_error = f"Could not connect to server {server.name}\n{e}"
                self.io.tool_warning(connection_error)
                for tool_call in tool_calls_list:
                    tool_responses.append(
                        {"role": "tool", "tool_call_id": tool_call.id, "content": connection_error}
                    )

            return tool_responses

        # Execute all tool calls concurrently
        async def _execute_all_tool_calls():
            tasks = []
            for server, tool_calls_list in tool_calls.items():
                tasks.append(_exec_server_tools(server, tool_calls_list))
            # Wait for all tasks to complete
            results = await asyncio.gather(*tasks)
            return results

        # Run the async execution and collect results
        if tool_calls:
            all_results = []
            max_retries = 3
            for i in range(max_retries):
                try:
                    all_results = await _execute_all_tool_calls()
                    break
                except asyncio.exceptions.CancelledError:
                    if i < max_retries - 1:
                        await asyncio.sleep(0.1)  # Brief pause before retrying
                    else:
                        self.io.tool_warning(
                            "MCP tool execution failed after multiple retries due to cancellation."
                        )
                        all_results = []

            # Flatten the results from all servers
            for server_results in all_results:
                tool_responses.extend(server_results)

        return tool_responses

    async def initialize_mcp_tools(self):
        """
        Initialize tools from all configured MCP servers. MCP Servers that fail to be
        initialized will not be available to the Coder instance.
        """
        tools = []

        async def get_server_tools(server):
            # Check if we already have tools for this server in mcp_tools
            if self.mcp_tools:
                for server_name, server_tools in self.mcp_tools:
                    if server_name == server.name:
                        return (server.name, server_tools)

            try:
                session = await server.connect()
                server_tools = await experimental_mcp_client.load_mcp_tools(
                    session=session, format="openai"
                )
                return (server.name, server_tools)
            except Exception as e:
                if server.name != "unnamed-server" and server.name != "local_tools":
                    self.io.tool_warning(f"Error initializing MCP server {server.name}: {e}")
                return None

        async def get_all_server_tools():
            tasks = [get_server_tools(server) for server in self.mcp_servers]
            results = await asyncio.gather(*tasks)
            return [result for result in results if result is not None]

        if self.mcp_servers:
            # Retry initialization in case of CancelledError
            max_retries = 3
            for i in range(max_retries):
                try:
                    tools = await get_all_server_tools()
                    break
                except asyncio.exceptions.CancelledError:
                    if i < max_retries - 1:
                        await asyncio.sleep(0.1)  # Brief pause before retrying
                    else:
                        self.io.tool_warning(
                            "MCP tool initialization failed after multiple retries due to"
                            " cancellation."
                        )
                        tools = []

        if len(tools) > 0:
            if self.verbose:
                self.io.tool_output("MCP servers configured:")

                for server_name, server_tools in tools:
                    self.io.tool_output(f"  - {server_name}")

                    for tool in server_tools:
                        tool_name = tool.get("function", {}).get("name", "unknown")
                        tool_desc = tool.get("function", {}).get("description", "").split("\n")[0]
                        self.io.tool_output(f"    - {tool_name}: {tool_desc}")

        self.mcp_tools = tools

    def get_tool_list(self):
        """Get a flattened list of all MCP tools."""
        tool_list = []
        if self.mcp_tools:
            for _, server_tools in self.mcp_tools:
                tool_list.extend(server_tools)
        return tool_list

    async def reply_completed(self):
        pass

    async def show_exhausted_error(self):
        output_tokens = 0
        if self.partial_response_content:
            output_tokens = self.main_model.token_count(self.partial_response_content)
        max_output_tokens = self.main_model.info.get("max_output_tokens") or 0

        input_tokens = self.main_model.token_count(self.format_messages().all_messages())
        max_input_tokens = self.main_model.info.get("max_input_tokens") or 0

        total_tokens = input_tokens + output_tokens

        fudge = 0.7

        out_err = ""
        if output_tokens >= max_output_tokens * fudge:
            out_err = " -- possibly exceeded output limit!"

        inp_err = ""
        if input_tokens >= max_input_tokens * fudge:
            inp_err = " -- possibly exhausted context window!"

        tot_err = ""
        if total_tokens >= max_input_tokens * fudge:
            tot_err = " -- possibly exhausted context window!"

        res = ["", ""]
        res.append(f"Model {self.main_model.name} has hit a token limit!")
        res.append("Token counts below are approximate.")
        res.append("")
        res.append(f"Input tokens: ~{input_tokens:,} of {max_input_tokens:,}{inp_err}")
        res.append(f"Output tokens: ~{output_tokens:,} of {max_output_tokens:,}{out_err}")
        res.append(f"Total tokens: ~{total_tokens:,} of {max_input_tokens:,}{tot_err}")

        if output_tokens >= max_output_tokens:
            res.append("")
            res.append("To reduce output tokens:")
            res.append("- Ask for smaller changes in each request.")
            res.append("- Break your code into smaller source files.")
            if "diff" not in self.main_model.edit_format:
                res.append("- Use a stronger model that can return diffs.")

        if input_tokens >= max_input_tokens or total_tokens >= max_input_tokens:
            res.append("")
            res.append("To reduce input tokens:")
            res.append("- Use /tokens to see token usage.")
            res.append("- Use /drop to remove unneeded files from the chat session.")
            res.append("- Use /clear to clear the chat history.")
            res.append("- Break your code into smaller source files.")

        res = "".join([line + "\n" for line in res])
        self.io.tool_error(res)
        await self.io.offer_url(urls.token_limits)

    def lint_edited(self, fnames):
        res = ""
        for fname in fnames:
            if not fname:
                continue
            errors = self.linter.lint(self.abs_root_path(fname))

            if errors:
                res += "\n"
                res += errors
                res += "\n"

        if res:
            self.io.tool_warning(res)

        return res

    def __del__(self):
        """Cleanup when the Coder object is destroyed."""
        self.ok_to_warm_cache = False

    def add_assistant_reply_to_cur_messages(self):
        """
        Add the assistant's reply to `cur_messages`.
        Handles model-specific quirks, like Deepseek which requires `content`
        to be `None` when `tool_calls` are present.
        """
        msg = dict(role="assistant")
        response = (
            self.partial_response_chunks[0]
            if not self.stream
            else litellm.stream_chunk_builder(self.partial_response_chunks)
        )

        try:
            # Use response_dict as a regular dictionary
            response_dict = response.model_dump()
        except AttributeError:
            # Option 2: Fall back to dict() or response.dict() (Pydantic V1 style)
            try:
                # Note: calling dict(response) works in both V1 and V2 for raw fields,
                # but response.dict() is the Pydantic V1 method name.
                response_dict = dict(response)
            except TypeError:
                print("Response parsing error.")
                return

        msg = response_dict["choices"][0]["message"]

        if self.partial_response_tool_calls:
            msg["tool_calls"] = self.partial_response_tool_calls
        elif self.partial_response_function_call:
            msg["function_call"] = self.partial_response_function_call

        if "reasoning_content" not in msg:
            msg["reasoning_content"] = self.partial_response_reasoning_content

        # Only add a message if it's not empty.
        if msg is not None and (
            msg.get("content", None)
            or msg.get("tool_calls", None)
            or msg.get("function_call", None)
        ):
            self.cur_messages.append(msg)

    def get_file_mentions(self, content, ignore_current=False):
        words = set(word for word in content.split())

        # drop sentence punctuation from the end
        words = set(word.rstrip(",.!;:?") for word in words)

        # strip away all kinds of quotes
        quotes = "\"'`*_"
        words = set(word.strip(quotes) for word in words)

        if ignore_current:
            addable_rel_fnames = self.get_all_relative_files()
            existing_basenames = {}
        else:
            addable_rel_fnames = self.get_addable_relative_files()

            # Get basenames of files already in chat or read-only
            existing_basenames = {os.path.basename(f) for f in self.get_inchat_relative_files()} | {
                os.path.basename(self.get_rel_fname(f))
                for f in self.abs_read_only_fnames | self.abs_read_only_stubs_fnames
            }

        mentioned_rel_fnames = set()
        fname_to_rel_fnames = {}
        for rel_fname in addable_rel_fnames:
            normalized_rel_fname = rel_fname.replace("\\", "/")
            normalized_words = set(word.replace("\\", "/") for word in words)
            if normalized_rel_fname in normalized_words:
                mentioned_rel_fnames.add(rel_fname)

            fname = os.path.basename(rel_fname)

            # Don't add basenames that could be plain words like "run" or "make"
            if "/" in fname or "\\" in fname or "." in fname or "_" in fname or "-" in fname:
                if fname not in fname_to_rel_fnames:
                    fname_to_rel_fnames[fname] = []
                fname_to_rel_fnames[fname].append(rel_fname)

        for fname, rel_fnames in fname_to_rel_fnames.items():
            # If the basename is already in chat, don't add based on a basename mention
            if fname in existing_basenames:
                continue
            # If the basename mention is unique among addable files and present in the text
            if len(rel_fnames) == 1 and fname in words:
                mentioned_rel_fnames.add(rel_fnames[0])

        return mentioned_rel_fnames

    async def check_for_file_mentions(self, content):
        mentioned_rel_fnames = self.get_file_mentions(content)

        new_mentions = mentioned_rel_fnames - self.ignore_mentions

        if not new_mentions:
            return

        added_fnames = []
        group = ConfirmGroup(new_mentions)
        for rel_fname in sorted(new_mentions):
            if self.args.tui:
                self.io.tool_output(rel_fname)

            if await self.io.confirm_ask(
                "Add file to the chat?", subject=rel_fname, group=group, allow_never=True
            ):
                self.add_rel_fname(rel_fname)
                added_fnames.append(rel_fname)
            else:
                self.ignore_mentions.add(rel_fname)

        if added_fnames:
            return prompts.added_files.format(fnames=", ".join(added_fnames))

    async def send(self, messages, model=None, functions=None, tools=None):
        self.got_reasoning_content = False
        self.ended_reasoning_content = False

        self._streaming_buffer_length = 0
        self.io.reset_streaming_response()

        if not model:
            model = self.main_model

        self.partial_response_content = ""
        self.partial_response_reasoning_content = ""
        self.partial_response_chunks = []
        self.partial_response_tool_calls = []
        self.partial_response_function_call = dict()

        completion = None
        self.token_profiler.start()

        try:
            hash_object, completion = await model.send_completion(
                messages,
                functions,
                self.stream,
                self.temperature,
                # This could include any tools, but for now it is just MCP tools
                tools=tools,
            )
            self.chat_completion_call_hashes.append(hash_object.hexdigest())

            if not isinstance(completion, ModelResponse):
                async for chunk in self.show_send_output_stream(completion):
                    yield chunk
            else:
                self.show_send_output(completion)

            # Calculate costs for successful responses
            self.calculate_and_show_tokens_and_cost(messages, completion)

        except LiteLLMExceptions().exceptions_tuple() as err:
            ex_info = LiteLLMExceptions().get_ex_info(err)
            if ex_info.name == "ContextWindowExceededError":
                # Still calculate costs for context window errors
                self.token_profiler.on_error()
                self.calculate_and_show_tokens_and_cost(messages, completion)
            raise
        except KeyboardInterrupt as kbi:
            self.keyboard_interrupt()
            raise kbi
        finally:
            self.preprocess_response()

            if self.partial_response_content:
                self.io.ai_output(self.partial_response_content)
            elif self.partial_response_function_call:
                # TODO: push this into subclasses
                args = self.parse_partial_args()
                if args:
                    self.io.ai_output(json.dumps(args, indent=4))

    def show_send_output(self, completion):
        if self.verbose:
            print(completion)

        if not isinstance(completion, ModelResponse):
            self.io.tool_error(str(completion))
            return

        if not completion.choices:
            self.io.tool_error(str(completion))
            return

        self.partial_response_chunks.append(completion)

        response, func_err, content_err = self.consolidate_chunks()

        resp_hash = dict(
            function_call=str(self.partial_response_function_call),
            content=self.partial_response_content,
        )
        resp_hash = hashlib.sha1(json.dumps(resp_hash, sort_keys=True).encode())
        self.chat_completion_response_hashes.append(resp_hash.hexdigest())

        if func_err and content_err:
            self.io.tool_error(func_err)
            self.io.tool_error(content_err)
            raise Exception("No data found in LLM response!")

        show_resp = self.render_incremental_response(True)

        if self.partial_response_reasoning_content:
            formatted_reasoning = format_reasoning_content(
                self.partial_response_reasoning_content, self.reasoning_tag_name
            )
            show_resp = formatted_reasoning + show_resp

        show_resp = replace_reasoning_tags(show_resp, self.reasoning_tag_name)

        self.io.assistant_output(show_resp, pretty=self.show_pretty())

        if (
            hasattr(completion.choices[0], "finish_reason")
            and completion.choices[0].finish_reason == "length"
        ):
            raise FinishReasonLength()

    async def show_send_output_stream(self, completion):
        received_content = False

        async for chunk in completion:
            if self.args.debug:
                with open(".aider/logs/chunks.log", "a") as f:
                    print(chunk, file=f)

            # Check if confirmation is in progress and wait if needed
            if not self.io.confirmation_in_progress_event.is_set():
                await self.io.confirmation_in_progress_event.wait()

            if isinstance(chunk, str):
                self.io.tool_error(chunk)
                continue
            else:
                if len(chunk.choices) == 0:
                    continue

                if (
                    hasattr(chunk.choices[0], "finish_reason")
                    and chunk.choices[0].finish_reason == "length"
                ):
                    raise FinishReasonLength()

                try:
                    if chunk.choices[0].delta.tool_calls:
                        received_content = True
                        self.token_profiler.on_token()
                        for tool_call_chunk in chunk.choices[0].delta.tool_calls:
                            self.tool_reflection = True

                            if tool_call_chunk.type:
                                self.io.update_spinner_suffix(tool_call_chunk.type)

                            if tool_call_chunk.function:
                                if tool_call_chunk.function.name:
                                    self.io.update_spinner_suffix(tool_call_chunk.function.name)

                                if tool_call_chunk.function.arguments:
                                    self.io.update_spinner_suffix(
                                        tool_call_chunk.function.arguments
                                    )

                except (AttributeError, IndexError):
                    # Handle cases where the response structure doesn't match expectations
                    pass

                try:
                    func = chunk.choices[0].delta.function_call
                    # dump(func)
                    for k, v in func.items():
                        self.tool_reflection = True
                        self.io.update_spinner_suffix(v)

                    received_content = True
                    self.token_profiler.on_token()
                except AttributeError:
                    pass

                text = ""

                try:
                    reasoning_content = chunk.choices[0].delta.reasoning_content
                except AttributeError:
                    try:
                        reasoning_content = chunk.choices[0].delta.reasoning
                    except AttributeError:
                        reasoning_content = None

                if reasoning_content:
                    if not self.got_reasoning_content:
                        text += f"<{REASONING_TAG}>\n\n"
                    text += reasoning_content
                    self.got_reasoning_content = True
                    received_content = True
                    self.token_profiler.on_token()
                    self.io.update_spinner_suffix(reasoning_content)
                    self.partial_response_reasoning_content += reasoning_content

                try:
                    content = chunk.choices[0].delta.content
                    if content:
                        if self.got_reasoning_content and not self.ended_reasoning_content:
                            text += f"\n\n</{self.reasoning_tag_name}>\n\n"
                            self.ended_reasoning_content = True

                        text += content
                        received_content = True
                        self.token_profiler.on_token()
                        self.io.update_spinner_suffix(content)
                except AttributeError:
                    pass

            self.partial_response_content += text

            self.partial_response_chunks.append(chunk)

            if self.show_pretty():
                # Use simplified streaming - just call the method with full content
                content_to_show = self.live_incremental_response(False)
                self.stream_wrapper(content_to_show, final=False)
            elif text:
                # Apply reasoning tag formatting for non-pretty output
                text = replace_reasoning_tags(text, self.reasoning_tag_name)
                try:
                    self.stream_wrapper(text, final=False)
                except UnicodeEncodeError:
                    # Safely encode and decode the text
                    safe_text = text.encode(sys.stdout.encoding, errors="backslashreplace").decode(
                        sys.stdout.encoding
                    )
                    self.stream_wrapper(safe_text, final=False)
                yield text

        # The Part Doing the Heavy Lifting Now
        self.consolidate_chunks()

        if not received_content and len(self.partial_response_tool_calls) == 0:
            self.io.tool_warning("Empty response received from LLM. Check your provider account?")

    def consolidate_chunks(self):
        response = (
            self.partial_response_chunks[0]
            if not self.stream
            else litellm.stream_chunk_builder(self.partial_response_chunks)
        )
        func_err = None
        content_err = None

        # Collect provider-specific fields from chunks to preserve them
        # We need to track both by ID (primary) and index (fallback) since
        # early chunks might not have IDs established yet
        provider_specific_fields_by_id = {}
        provider_specific_fields_by_index = {}

        for chunk in self.partial_response_chunks:
            try:
                if chunk.choices and chunk.choices[0].delta and chunk.choices[0].delta.tool_calls:
                    for tool_call in chunk.choices[0].delta.tool_calls:
                        if (
                            hasattr(tool_call, "provider_specific_fields")
                            and tool_call.provider_specific_fields
                        ):
                            # Ensure provider_specific_fields is a dictionary
                            psf = tool_call.provider_specific_fields
                            if not isinstance(psf, dict):
                                continue

                            # Try to use ID first
                            if hasattr(tool_call, "id") and tool_call.id:
                                tool_id = tool_call.id
                                if tool_id not in provider_specific_fields_by_id:
                                    provider_specific_fields_by_id[tool_id] = {}
                                # Merge provider-specific fields for this tool ID
                                provider_specific_fields_by_id[tool_id].update(psf)
                            # Also track by index as fallback
                            elif hasattr(tool_call, "index"):
                                tool_index = tool_call.index
                                if tool_index not in provider_specific_fields_by_index:
                                    provider_specific_fields_by_index[tool_index] = {}
                                provider_specific_fields_by_index[tool_index].update(psf)
            except (AttributeError, IndexError):
                continue

        try:
            if response.choices[0].message.tool_calls:
                for i, tool_call in enumerate(response.choices[0].message.tool_calls):
                    # Add provider-specific fields if we collected any for this tool
                    tool_id = tool_call.id

                    # Try ID first
                    if tool_id in provider_specific_fields_by_id:
                        # Add provider-specific fields directly to the tool call object
                        tool_call.provider_specific_fields = provider_specific_fields_by_id[tool_id]
                    # Fall back to index
                    elif i in provider_specific_fields_by_index:
                        # Add provider-specific fields directly to the tool call object
                        tool_call.provider_specific_fields = provider_specific_fields_by_index[i]

                    # Create dictionary version with provider-specific fields
                    tool_call_dict = tool_call.model_dump()

                    # Add provider-specific fields to the dictionary too (in case model_dump() doesn't include them)
                    if tool_id in provider_specific_fields_by_id:
                        tool_call_dict["provider_specific_fields"] = provider_specific_fields_by_id[
                            tool_id
                        ]
                    elif i in provider_specific_fields_by_index:
                        tool_call_dict["provider_specific_fields"] = (
                            provider_specific_fields_by_index[i]
                        )

                    # Only append to partial_response_tool_calls if it's empty
                    if len(self.partial_response_tool_calls) == 0:
                        self.partial_response_tool_calls.append(tool_call_dict)

                self.partial_response_function_call = (
                    response.choices[0].message.tool_calls[0].function
                )
        except AttributeError as e:
            func_err = e

        try:
            reasoning_content = response.choices[0].message.reasoning_content
        except AttributeError:
            try:
                reasoning_content = response.choices[0].message.reasoning
            except AttributeError:
                reasoning_content = None

        self.partial_response_reasoning_content = reasoning_content or ""

        try:
            content = response.choices[0].message.content
            if isinstance(content, list):
                # OpenAI-compatible APIs sometimes return content as a list
                # of blocks; join the textual pieces for display.
                content = "".join(
                    block.get("text", "")
                    for block in content
                    if isinstance(block, dict) and block.get("type") == "output_text"
                ) or "".join(
                    block.get("text", "")
                    for block in content
                    if isinstance(block, dict) and block.get("type") == "text"
                )
            self.partial_response_content = content or ""
        except AttributeError as e:
            content_err = e

        return response, func_err, content_err

    def stream_wrapper(self, content, final):
        if not hasattr(self, "_streaming_buffer_length"):
            self._streaming_buffer_length = 0

        if final:
            content += "\n\n"

        if isinstance(content, str):
            self._streaming_buffer_length += len(content)

            self.io.stream_output(content, final=final)

            if final:
                self._streaming_buffer_length = 0

    def live_incremental_response(self, final):
        show_resp = self.render_incremental_response(final)
        # Apply any reasoning tag formatting
        show_resp = replace_reasoning_tags(show_resp, self.reasoning_tag_name)

        # Track streaming state to avoid repetitive output
        if not hasattr(self, "_streaming_buffer_length"):
            self._streaming_buffer_length = 0

        # Only send new content that hasn't been streamed yet
        if len(show_resp) >= self._streaming_buffer_length:
            new_content = show_resp[self._streaming_buffer_length :]
            return new_content
        else:
            self._streaming_buffer_length = 0
            self.io.reset_streaming_response()
            return show_resp

    def render_incremental_response(self, final):
        # Just return the current content - the streaming logic will handle incremental updates
        return self.get_multi_response_content_in_progress()

    def preprocess_response(self):
        if len(self.partial_response_tool_calls):
            tool_list = []
            tool_id_set = set()

            for tool_call_dict in self.partial_response_tool_calls:
                # LLM APIs sometimes return duplicates and that's annoying part 2
                if tool_call_dict.get("id") in tool_id_set:
                    continue

                tool_id_set.add(tool_call_dict.get("id"))
                tool_list.append(tool_call_dict)

            self.partial_response_tool_calls = tool_list

    def remove_reasoning_content(self):
        """Remove reasoning content from the model's response."""

        self.partial_response_content = remove_reasoning_content(
            self.partial_response_content,
            self.reasoning_tag_name,
        )

    def calculate_and_show_tokens_and_cost(self, messages, completion=None):
        prompt_tokens = 0
        completion_tokens = 0
        cache_hit_tokens = 0
        cache_write_tokens = 0

        if completion and hasattr(completion, "usage") and completion.usage is not None:
            prompt_tokens = completion.usage.prompt_tokens
            completion_tokens = completion.usage.completion_tokens
            cache_hit_tokens = getattr(completion.usage, "prompt_cache_hit_tokens", 0) or getattr(
                completion.usage, "cache_read_input_tokens", 0
            )
            cache_write_tokens = getattr(completion.usage, "cache_creation_input_tokens", 0)

            if hasattr(completion.usage, "cache_read_input_tokens") or hasattr(
                completion.usage, "cache_creation_input_tokens"
            ):
                self.message_tokens_sent += prompt_tokens
                self.message_tokens_sent += cache_write_tokens
            else:
                self.message_tokens_sent += prompt_tokens

        else:
            prompt_tokens = self.main_model.token_count(messages)
            completion_tokens = self.main_model.token_count(self.partial_response_content)
            self.message_tokens_sent += prompt_tokens

        self.message_tokens_received += completion_tokens

        tokens_report = f"Tokens: {format_tokens(self.message_tokens_sent)} sent"

        if cache_write_tokens:
            tokens_report += f", {format_tokens(cache_write_tokens)} cache write"
        if cache_hit_tokens:
            tokens_report += f", {format_tokens(cache_hit_tokens)} cache hit"
        tokens_report += f", {format_tokens(self.message_tokens_received)} received."
        tokens_report = self.token_profiler.add_to_usage_report(
            tokens_report, self.message_tokens_sent, self.message_tokens_received
        )

        if not self.main_model.info.get("input_cost_per_token"):
            self.usage_report = tokens_report
            return

        try:
            # Try and use litellm's built in cost calculator. Seems to work for non-streaming only?
            cost = litellm.completion_cost(completion_response=completion)
        except Exception:
            cost = 0

        if not cost:
            cost = self.compute_costs_from_tokens(
                prompt_tokens, completion_tokens, cache_write_tokens, cache_hit_tokens
            )

        self.total_cost += cost
        self.message_cost += cost

        cost_report = (
            f"Cost: ${self.format_cost(self.message_cost)} message,"
            f" ${self.format_cost(self.total_cost)} session."
        )

        if cache_hit_tokens and cache_write_tokens:
            sep = "\n"
        else:
            sep = " "

        self.usage_report = tokens_report + sep + cost_report

    def format_cost(self, value):
        if value == 0:
            return "0.00"
        magnitude = abs(value)
        if magnitude >= 0.01:
            return f"{value:.2f}"
        else:
            return f"{value:.{max(2, 2 - int(math.log10(magnitude)))}f}"

    def compute_costs_from_tokens(
        self, prompt_tokens, completion_tokens, cache_write_tokens, cache_hit_tokens
    ):
        cost = 0

        input_cost_per_token = self.main_model.info.get("input_cost_per_token") or 0
        output_cost_per_token = self.main_model.info.get("output_cost_per_token") or 0
        input_cost_per_token_cache_hit = (
            self.main_model.info.get("input_cost_per_token_cache_hit") or 0
        )

        # deepseek
        # prompt_cache_hit_tokens + prompt_cache_miss_tokens
        #    == prompt_tokens == total tokens that were sent
        #
        # Anthropic
        # cache_creation_input_tokens + cache_read_input_tokens + prompt
        #    == total tokens that were

        if input_cost_per_token_cache_hit:
            # must be deepseek
            cost += input_cost_per_token_cache_hit * cache_hit_tokens
            cost += (prompt_tokens - input_cost_per_token_cache_hit) * input_cost_per_token
        else:
            # hard code the anthropic adjustments, no-ops for other models since cache_x_tokens==0
            cost += cache_write_tokens * input_cost_per_token * 1.25
            cost += cache_hit_tokens * input_cost_per_token * 0.10
            cost += prompt_tokens * input_cost_per_token

        cost += completion_tokens * output_cost_per_token
        return cost

    def show_usage_report(self):
        if not self.usage_report:
            return

        self.total_tokens_sent += self.message_tokens_sent
        self.total_tokens_received += self.message_tokens_received

        self.io.tool_output(self.usage_report)
        self.io.rule()

        self.message_cost = 0.0
        self.message_tokens_sent = 0
        self.message_tokens_received = 0

    def get_multi_response_content_in_progress(self, final=False):
        cur = self.multi_response_content or ""
        new = self.partial_response_content or ""

        if new.rstrip() != new and not final:
            new = new.rstrip()

        return cur + new

    def get_file_stub(self, fname):
        return RepoMap.get_file_stub(fname, self.io)

    def get_rel_fname(self, fname):
        try:
            return os.path.relpath(fname, self.root)
        except ValueError:
            return fname

    def get_inchat_relative_files(self):
        files = [self.get_rel_fname(fname) for fname in self.abs_fnames]
        return sorted(set(files))

    def is_file_safe(self, fname):
        try:
            return Path(self.abs_root_path(fname)).is_file()
        except OSError:
            return

    def get_all_relative_files(self):
        if self.repo_map and self.repo:
            try:
                staged_files_hash = hash(
                    str([item.a_path for item in self.repo.repo.index.diff("HEAD")])
                )
                if (
                    staged_files_hash == self.data_cache["repo"]["last_key"]
                    and self.data_cache["relative_files"]
                ):
                    return self.data_cache["relative_files"]
            except ANY_GIT_ERROR as err:
                # Handle git errors gracefully - fall back to getting tracked files
                if self.verbose:
                    self.io.tool_warning(f"Git error while checking staged files: {err}")
                # Continue to get tracked files normally

        if self.repo:
            files = self.repo.get_tracked_files()
        else:
            files = self.get_inchat_relative_files()

        # This is quite slow in large repos
        # files = [fname for fname in files if self.is_file_safe(fname)]

        self.data_cache["relative_files"] = sorted(set(files))

        return self.data_cache["relative_files"]

    def get_all_abs_files(self):
        files = self.get_all_relative_files()
        files = [self.abs_root_path(path) for path in files]
        return files

    def get_addable_relative_files(self):
        all_files = set(self.get_all_relative_files())
        inchat_files = set(self.get_inchat_relative_files())
        read_only_files = set(self.get_rel_fname(fname) for fname in self.abs_read_only_fnames)
        stub_files = set(self.get_rel_fname(fname) for fname in self.abs_read_only_stubs_fnames)
        return all_files - inchat_files - read_only_files - stub_files

    def check_for_dirty_commit(self, path):
        if not self.repo:
            return
        if not self.dirty_commits:
            return
        if not self.repo.is_dirty(path):
            return

        # We need a committed copy of the file in order to /undo, so skip this
        # fullp = Path(self.abs_root_path(path))
        # if not fullp.stat().st_size:
        #     return

        self.io.tool_output(f"Committing {path} before applying edits.")
        self.need_commit_before_edits.add(path)

    async def allowed_to_edit(self, path):
        full_path = self.abs_root_path(path)
        if self.repo:
            need_to_add = not self.repo.path_in_repo(path)
        else:
            need_to_add = False

        if full_path in self.abs_fnames:
            self.check_for_dirty_commit(path)
            return True

        if self.repo and self.repo.git_ignored_file(path):
            self.io.tool_warning(f"Skipping edits to {path} that matches gitignore spec.")
            return

        if not Path(full_path).exists():
            if not await self.io.confirm_ask("Create new file?", subject=path):
                self.io.tool_output(f"Skipping edits to {path}")
                return

            if not self.dry_run:
                if not utils.touch_file(full_path):
                    self.io.tool_error(f"Unable to create {path}, skipping edits.")
                    return

                # Seems unlikely that we needed to create the file, but it was
                # actually already part of the repo.
                # But let's only add if we need to, just to be safe.
                if need_to_add:
                    self.repo.repo.git.add(full_path)

            self.abs_fnames.add(full_path)
            self.check_added_files()
            return True

        if not await self.io.confirm_ask(
            "Allow edits to file that has not been added to the chat?",
            subject=path,
        ):
            self.io.tool_output(f"Skipping edits to {path}")
            return

        if need_to_add:
            self.repo.repo.git.add(full_path)

        self.abs_fnames.add(full_path)
        self.check_added_files()
        self.check_for_dirty_commit(path)

        return True

    warning_given = False

    def check_added_files(self):
        if self.warning_given:
            return

        warn_number_of_files = 4
        warn_number_of_tokens = 20 * 1024

        num_files = len(self.abs_fnames)
        if num_files < warn_number_of_files:
            return

        tokens = 0
        for fname in self.abs_fnames:
            if is_image_file(fname):
                continue
            content = self.io.read_text(fname)
            tokens += self.main_model.token_count(content)

        if tokens < warn_number_of_tokens:
            return

        self.io.tool_warning("Warning: it's best to only add files that need changes to the chat.")
        self.io.tool_warning(urls.edit_errors)
        self.warning_given = True

    async def prepare_to_edit(self, edits):
        res = []
        seen = dict()

        self.need_commit_before_edits = set()

        for edit in edits:
            path = edit[0]
            if path is None:
                res.append(edit)
                continue
            if path == "python":
                dump(edits)
            if path in seen:
                allowed = seen[path]
            else:
                allowed = await self.allowed_to_edit(path)
                seen[path] = allowed

            if allowed:
                res.append(edit)

        await self.dirty_commit()
        self.need_commit_before_edits = set()

        return res

    async def apply_updates(self):
        edited = set()
        try:
            if getattr(self.args, "tweak_responses", False):
                confirmation = await self.io.confirm_ask("Tweak Response?", allow_tweak=True)

                if confirmation or confirmation == "tweak":
                    self.partial_response_content = self.io.edit_in_editor(
                        self.partial_response_content
                    )

            await asyncio.sleep(0.1)

            edits = self.get_edits()
            edits = self.apply_edits_dry_run(edits)
            edits = await self.prepare_to_edit(edits)
            edited = set(edit[0] for edit in edits)

            self.apply_edits(edits)
        except ValueError as err:
            self.num_malformed_responses += 1

            err = err.args[0]

            self.io.tool_error("The LLM did not conform to the edit format.")
            self.io.tool_output(urls.edit_errors)
            self.io.tool_output()
            self.io.tool_output(str(err))

            self.reflected_message = str(err)
            return edited

        except ANY_GIT_ERROR as err:
            self.io.tool_error(str(err))
            return edited
        except Exception as err:
            self.io.tool_error("Exception while updating files:")
            self.io.tool_error(str(err), strip=False)
            self.io.tool_error(traceback.format_exc())
            self.reflected_message = str(err)
            return edited

        for path in edited:
            if self.dry_run:
                self.io.tool_output(f"Did not apply edit to {path} (--dry-run)")
            else:
                self.io.tool_output(f"Applied edit to {path}")

        return edited

    def parse_partial_args(self):
        # dump(self.partial_response_function_call)

        data = self.partial_response_function_call.get("arguments")
        if not data:
            return

        try:
            return json.loads(data)
        except JSONDecodeError:
            pass

        try:
            return json.loads(data + "]}")
        except JSONDecodeError:
            pass

        try:
            return json.loads(data + "}]}")
        except JSONDecodeError:
            pass

        try:
            return json.loads(data + '"}]}')
        except JSONDecodeError:
            pass

    def _find_occurrences(self, content, pattern, near_context=None):
        """Find all occurrences of pattern, optionally filtered by near_context."""
        occurrences = []
        start = 0
        while True:
            index = content.find(pattern, start)
            if index == -1:
                break

            if near_context:
                # Check if near_context is within a window around the match
                window_start = max(0, index - 200)
                window_end = min(len(content), index + len(pattern) + 200)
                window = content[window_start:window_end]
                if near_context in window:
                    occurrences.append(index)
            else:
                occurrences.append(index)

            start = index + 1  # Move past this occurrence's start
        return occurrences

    # commits...

    def get_context_from_history(self, history):
        context = ""
        if history:
            for msg in history:
                msg_content = msg.get("content") or ""
                context += "\n" + msg["role"].upper() + ": " + msg_content + "\n"

        return context

    async def auto_commit(self, edited, context=None):
        if not self.repo or not self.auto_commits or self.dry_run:
            return

        if not context:
            context = self.get_context_from_history(self.cur_messages)

        try:
            res = await self.repo.commit(
                fnames=edited, context=context, aider_edits=True, coder=self
            )
            if res:
                self.show_auto_commit_outcome(res)
                commit_hash, commit_message = res
                return self.gpt_prompts.files_content_gpt_edits.format(
                    hash=commit_hash,
                    message=commit_message,
                )

            return self.gpt_prompts.files_content_gpt_no_edits
        except ANY_GIT_ERROR as err:
            self.io.tool_error(f"Unable to commit: {str(err)}")
            return

    def show_auto_commit_outcome(self, res):
        commit_hash, commit_message = res
        self.last_aider_commit_hash = commit_hash
        self.aider_commit_hashes.add(commit_hash)
        self.last_aider_commit_message = commit_message
        if self.show_diffs:
            self.commands.cmd_diff()

    def show_undo_hint(self):
        if not self.commit_before_message:
            return
        if self.commit_before_message[-1] != self.repo.get_head_commit_sha():
            self.io.tool_output("You can use /undo to undo and discard each aider commit.")

    async def dirty_commit(self):
        if not self.need_commit_before_edits:
            return
        if not self.dirty_commits:
            return
        if not self.repo:
            return

        await self.repo.commit(fnames=self.need_commit_before_edits, coder=self)

        # files changed, move cur messages back behind the files messages
        # self.move_back_cur_messages(self.gpt_prompts.files_content_local_edits)
        return True

    def get_edits(self, mode="update"):
        return []

    def apply_edits(self, edits):
        return

    def apply_edits_dry_run(self, edits):
        return edits

    async def auto_save_session(self):
        """Automatically save the current session to {auto-save-session-name}.json."""
        if not getattr(self.args, "auto_save", False):
            return

        # Initialize last autosave time if not exists
        if not hasattr(self, "_last_autosave_time"):
            self._last_autosave_time = 0

        if not hasattr(self, "_autosave_future"):
            self._autosave_future = None

        if self._autosave_future and not self._autosave_future.done():
            return

        # Throttle autosave to run at most once every 15 seconds
        current_time = time.time()
        if current_time - self._last_autosave_time >= 15.0:
            try:
                self._last_autosave_time = current_time
                session_manager = SessionManager(self, self.io)
                loop = asyncio.get_running_loop()
                self._autosave_future = loop.run_in_executor(
                    None,
                    session_manager.save_session,
                    getattr(self.args, "auto_save_session_name", "auto-save"),
                    False,
                )
            except Exception:
                # Don't show errors for auto-save to avoid interrupting the user experience
                pass

    async def run_shell_commands(self):
        if not self.suggest_shell_commands:
            return ""

        done = set()
        group = ConfirmGroup(set(self.shell_commands))
        accumulated_output = ""

        try:
            self.commands.cmd_running_event.clear()  # Command is running

            for command in self.shell_commands:
                if command in done:
                    continue
                done.add(command)
                output = await self.handle_shell_commands(command, group)
                if output:
                    accumulated_output += output + "\n\n"

            return accumulated_output
        finally:
            self.commands.cmd_running_event.set()  # Command finished

    async def handle_shell_commands(self, commands_str, group):
        commands = commands_str.strip().split(";")
        command_count = sum(
            1 for cmd in commands if cmd.strip() and not cmd.strip().startswith("#")
        )
        prompt = "Run shell command?" if command_count == 1 else "Run shell commands?"
        if not await self.io.confirm_ask(
            prompt,
            subject="\n".join(commands),
            explicit_yes_required=self.args.yes_always_commands,
            group=group,
            allow_never=True,
        ):
            return

        accumulated_output = ""
        for command in commands:
            command = command.strip()
            if not command or command.startswith("#"):
                continue

            command = self.format_command_with_prefix(command)

            self.io.tool_output()
            self.io.tool_output(f"Running {command}")
            # Add the command to input history
            self.io.add_to_input_history(f"/run {command.strip()}")
            exit_status, output = await asyncio.to_thread(
                run_cmd, command, error_print=self.io.tool_error, cwd=self.root
            )
            if output:
                accumulated_output += f"Output from {command}\n{output}\n"

        if accumulated_output.strip() and await self.io.confirm_ask(
            "Add command output to the chat?", allow_never=True
        ):
            num_lines = len(accumulated_output.strip().splitlines())
            line_plural = "line" if num_lines == 1 else "lines"
            self.io.tool_output(f"Added {num_lines} {line_plural} of output to the chat.")
            return accumulated_output

    def format_command_with_prefix(self, command):
        """
        Format a command with a command prefix.

        If the command prefix contains a {} placeholder, replace it with the command.
        Otherwise, append the command to the prefix with a space.

        Args:
            command (str): The command to format

        Returns:
            str: The formatted command
        """
        command_prefix = None

        if command and getattr(self.args, "command_prefix", None):
            command_prefix = getattr(self.args, "command_prefix", None)

        if not command_prefix:
            return command

        # Check if the prefix contains a {} placeholder
        if "{}" in command_prefix:
            # Replace the {} placeholder with the command
            return command_prefix.replace("{}", command)
        else:
            # Append the command to the prefix with a space
            return f"{command_prefix} {command}"
